v0.8.1

.vscode/launch.json

@@ -12,14 +12,20 @@
                 //"-nw",
                 "-ed",
                 "-emp",
-                "-e2d",
+                "-e2dsa",
-                "-e2s",
                 "-a",
                 "ed:wark",
                 "-v",
                 "srv::r:aed:cnodupe"
             ]
         },
+        {
+            "name": "No debug",
+            "preLaunchTask": "no_dbg",
+            "type": "python",
+            //"request": "attach", "port": 42069
+            // fork: nc -l 42069 </dev/null
+        },
         {
             "name": "Run active unit test",
             "type": "python",

.vscode/settings.json

@@ -50,11 +50,9 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "editor.codeActionsOnSaveTimeout": 9001,
+    "python.formatting.blackArgs": [
-    "editor.formatOnSaveTimeout": 9001,
+        "-t",
-    //
+        "py27"
-    //  things you may wanna edit:
+    ],
-    //
+    "python.linting.enabled": true,
-    "python.pythonPath": "/usr/bin/python3",
-    //"python.linting.enabled": true,
 }

.vscode/tasks.json

@@ -5,6 +5,11 @@
             "label": "pre",
             "command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
             "type": "shell"
+        },
+        {
+            "label": "no_dbg",
+            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -a ed:wark -v srv::r:aed:cnodupe ;exit 1",
+            "type": "shell"
         }
     ]
 }

README.md

@@ -8,7 +8,7 @@
 
 turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
 
-* server runs on anything with `py2.7` or `py3.2+`
+* server runs on anything with `py2.7` or `py3.3+`
 * *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
 * code standard: `black`
 
@@ -36,24 +36,55 @@ you may also want these, especially on servers:
 
 ## status
 
-* [x] sanic multipart parser
+* backend stuff
-* [x] load balancer (multiprocessing)
+  * ☑ sanic multipart parser
-* [x] upload (plain multipart, ie6 support)
+  * ☑ load balancer (multiprocessing)
-* [x] upload (js, resumable, multithreaded)
+  * ☑ volumes (mountpoints)
-* [x] download
+  * ☑ accounts
-* [x] browser
+* upload
-* [x] media player
+  * ☑ basic: plain multipart, ie6 support
-* [ ] thumbnails
+  * ☑ up2k: js, resumable, multithreaded
-* [ ] download as zip
+  * ☑ stash: simple PUT filedropper
-* [x] volumes
+  * ☑ symlink/discard existing files (content-matching)
-* [x] accounts
+* download
-* [x] markdown viewer
+  * ☑ single files in browser
-* [x] markdown editor
+  * ✖ folders as zip files
-* [x] FUSE client (read-only)
+  * ☑ FUSE client (read-only)
+* browser
+  * ☑ tree-view
+  * ☑ media player
+  * ✖ thumbnails
+  * ✖ SPA (browse while uploading)
+    * currently safe using the file-tree on the left only, not folders in the file list
+* server indexing
+  * ☑ locate files by contents
+  * ☑ search by name/path/date/size
+  * ✖ search by ID3-tags etc.
+* markdown
+  * ☑ viewer
+  * ☑ editor (sure why not)
 
 summary: it works! you can use it! (but technically not even close to beta)
 
 
+# bugs
+
+* probably, pls let me know
+
+
+# searching
+
+when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
+* make search queries by `size`/`date`/`directory-path`/`filename`, or...
+* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+
+path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
+* path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
+* name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9)
+
+other metadata (like song tags etc) are not yet indexed for searching
+
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)
@@ -68,17 +99,16 @@ summary: it works! you can use it! (but technically not even close to beta)
 # dependencies
 
 * `jinja2`
-  * pulls in `markupsafe` as of v2.7; use jinja 2.6 on py3.2
 
-optional, enables thumbnails:
+optional, will eventually enable thumbnails:
 * `Pillow` (requires py2.7 or py3.5+)
 
 
 # sfx
 
 currently there are two self-contained binaries:
-* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
-* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
 
 launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
 

bin/copyparty-fuse-streaming.py

@@ -1067,7 +1067,7 @@ def main():
         dbg = null_log
 
     if WINDOWS:
-        os.system("")
+        os.system("rem")
 
         for ch in '<>:"\\|?*':
             # microsoft maps illegal characters to f0xx

bin/copyparty-fuse.py

@@ -33,6 +33,7 @@ import re
 import os
 import sys
 import time
+import json
 import stat
 import errno
 import struct
@@ -323,7 +324,7 @@ class Gateway(object):
         if bad_good:
             path = dewin(path)
 
-        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
+        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
         r = self.sendreq("GET", web_path)
         if r.status != 200:
             self.closeconn()
@@ -334,12 +335,17 @@ class Gateway(object):
             )
             raise FuseOSError(errno.ENOENT)
 
-        if not r.getheader("Content-Type", "").startswith("text/html"):
+        ctype = r.getheader("Content-Type", "")
+        if ctype == "application/json":
+            parser = self.parse_jls
+        elif ctype.startswith("text/html"):
+            parser = self.parse_html
+        else:
             log("listdir on file: {}".format(path))
             raise FuseOSError(errno.ENOENT)
 
         try:
-            return self.parse_html(r)
+            return parser(r)
         except:
             info(repr(path) + "\n" + traceback.format_exc())
             raise
@@ -367,6 +373,29 @@ class Gateway(object):
 
         return r.read()
 
+    def parse_jls(self, datasrc):
+        rsp = b""
+        while True:
+            buf = datasrc.read(1024 * 32)
+            if not buf:
+                break
+
+            rsp += buf
+
+        rsp = json.loads(rsp.decode("utf-8"))
+        ret = []
+        for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
+            for n in nodes:
+                fname = unquote(n["href"]).rstrip(b"/")
+                fname = fname.decode("wtf-8")
+                if bad_good:
+                    fname = enwin(fname)
+
+                fun = self.stat_dir if is_dir else self.stat_file
+                ret.append([fname, fun(n["ts"], n["sz"]), 0])
+
+        return ret
+
     def parse_html(self, datasrc):
         ret = []
         remainder = b""
@@ -818,9 +847,9 @@ class CPPF(Operations):
                 return cache_stat
 
         fun = info
-        if MACOS and path.split('/')[-1].startswith('._'):
+        if MACOS and path.split("/")[-1].startswith("._"):
             fun = dbg
-        
+
         fun("=ENOENT ({})".format(hexler(path)))
         raise FuseOSError(errno.ENOENT)
 
@@ -980,7 +1009,7 @@ def main():
         dbg = null_log
 
     if WINDOWS:
-        os.system("")
+        os.system("rem")
 
         for ch in '<>:"\\|?*':
             # microsoft maps illegal characters to f0xx

contrib/README.md

@@ -10,7 +10,12 @@
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
-disables thumbnails and folder-type detection in windows explorer, makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+* disables thumbnails and folder-type detection in windows explorer
+* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+
+### [`cfssl.sh`](cfssl.sh)
+* creates CA and server certificates using cfssl
+* give a 3rd argument to install it to your copyparty config
 
 # OS integration
 init-scripts to start copyparty as a service

contrib/cfssl.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e
+
+# ca-name and server-name
+ca_name="$1"
+srv_name="$2"
+
+[ -z "$srv_name" ] && {
+	echo "need arg 1: ca name"
+	echo "need arg 2: server name"
+	exit 1
+}
+
+
+gen_ca() {
+	(tee /dev/stderr <<EOF
+{"CN": "$ca_name ca",
+"CA": {"expiry":"87600h", "pathlen":0},
+"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name ca"}]}
+EOF
+	)|
+	cfssl gencert -initca - |
+	cfssljson -bare ca
+	
+	mv ca-key.pem ca.key
+	rm ca.csr
+}
+
+
+gen_srv() {
+	(tee /dev/stderr <<EOF
+{"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name - $srv_name"}]}
+EOF
+	)|
+	cfssl gencert -ca ca.pem -ca-key ca.key \
+		-profile=www -hostname="$srv_name.$ca_name" - |
+	cfssljson -bare "$srv_name"
+
+	mv "$srv_name-key.pem" "$srv_name.key"
+	rm "$srv_name.csr"
+}
+
+
+# create ca if not exist
+[ -e ca.key ] ||
+	gen_ca
+
+# always create server cert
+gen_srv
+
+
+# dump cert info
+show() {
+	openssl x509 -text -noout -in $1 |
+	awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
+}
+show ca.pem
+show "$srv_name.pem"
+
+
+# write cert into copyparty config
+[ -z "$3" ] || {
+	mkdir -p ~/.config/copyparty
+	cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
+}
+
+
+# rm *.key *.pem
+# cfssl print-defaults config
+# cfssl print-defaults csr

copyparty/__main__.py

@@ -8,7 +8,9 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
+import re
 import os
+import sys
 import time
 import shutil
 import filecmp
@@ -19,7 +21,13 @@ from textwrap import dedent
 from .__init__ import E, WINDOWS, VT100
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc
+from .util import py_desc, align_tab
+
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
 
 
 class RiceFormatter(argparse.HelpFormatter):
@@ -85,10 +93,77 @@ def ensure_cert():
     # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
 
 
+def configure_ssl_ver(al):
+    def terse_sslver(txt):
+        txt = txt.lower()
+        for c in ["_", "v", "."]:
+            txt = txt.replace(c, "")
+
+        return txt.replace("tls10", "tls1")
+
+    # oh man i love openssl
+    # check this out
+    # hold my beer
+    ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
+    sslver = terse_sslver(al.ssl_ver).split(",")
+    flags = [k for k in ssl.__dict__ if ptn.match(k)]
+    # SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
+    if "help" in sslver:
+        avail = [terse_sslver(x[6:]) for x in flags]
+        avail = " ".join(sorted(avail) + ["all"])
+        print("\navailable ssl/tls versions:\n  " + avail)
+        sys.exit(0)
+
+    al.ssl_flags_en = 0
+    al.ssl_flags_de = 0
+    for flag in sorted(flags):
+        ver = terse_sslver(flag[6:])
+        num = getattr(ssl, flag)
+        if ver in sslver:
+            al.ssl_flags_en |= num
+        else:
+            al.ssl_flags_de |= num
+
+    if sslver == ["all"]:
+        x = al.ssl_flags_en
+        al.ssl_flags_en = al.ssl_flags_de
+        al.ssl_flags_de = x
+
+    for k in ["ssl_flags_en", "ssl_flags_de"]:
+        num = getattr(al, k)
+        print("{}: {:8x} ({})".format(k, num, num))
+
+    # think i need that beer now
+
+
+def configure_ssl_ciphers(al):
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    if al.ssl_ver:
+        ctx.options &= ~al.ssl_flags_en
+        ctx.options |= al.ssl_flags_de
+
+    is_help = al.ciphers == "help"
+
+    if al.ciphers and not is_help:
+        try:
+            ctx.set_ciphers(al.ciphers)
+        except:
+            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+
+    if not hasattr(ctx, "get_ciphers"):
+        print("cannot read cipher list: openssl or python too old")
+    else:
+        ciphers = [x["description"] for x in ctx.get_ciphers()]
+        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+
+    if is_help:
+        sys.exit(0)
+
+
 def main():
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
-        os.system("")  # enables colors
+        os.system("rem")  # enables colors
 
     desc = py_desc().replace("[", "\033[1;30m[")
 
@@ -96,7 +171,20 @@ def main():
     print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
-    ensure_cert()
+    if HAVE_SSL:
+        ensure_cert()
+
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = sys.argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        sys.argv[idx] = nk
+        time.sleep(2)
 
     ap = argparse.ArgumentParser(
         formatter_class=RiceFormatter,
@@ -127,6 +215,16 @@ def main():
 
             consider the config file for more flexible account/volume management,
             including dynamic reload at runtime (and being more readable w)
+
+            values for --urlform:
+              "stash" dumps the data to file and returns length + checksum
+              "save,get" dumps to file and returns the page like a GET
+              "print,get" prints the data in the log and returns GET
+              (leave out the ",get" to return an error instead)
+
+            --ciphers help = available ssl/tls ciphers,
+            --ssl-ver help = available ssl/tls versions,
+              default is what python considers safe, usually >= TLS1
             """
         ),
     )
@@ -142,15 +240,32 @@ def main():
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-e2d", action="store_true", help="enable up2k database")
-    ap.add_argument("-e2s", action="store_true", help="enable up2k db-scanner")
+    ap.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
+    ap.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
+    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", type=str, help="ssl/tls versions to allow")
+    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     al = ap.parse_args()
     # fmt: on
 
+    if al.e2dsa:
+        al.e2ds = True
+
+    if al.e2ds:
+        al.e2d = True
+
     al.i = al.i.split(",")
     try:
         if "-" in al.p:
@@ -161,6 +276,15 @@ def main():
     except:
         raise Exception("invalid value for -p")
 
+    if HAVE_SSL:
+        if al.ssl_ver:
+            configure_ssl_ver(al)
+
+        if al.ciphers:
+            configure_ssl_ciphers(al)
+    else:
+        print("\033[33m  ssl module does not exist; cannot enable https\033[0m\n")
+
     SvcHub(al).run()
 
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 7, 3)
+VERSION = (0, 8, 1)
 CODENAME = "keeping track"
-BUILD_DT = (2021, 2, 3)
+BUILD_DT = (2021, 2, 22)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -19,6 +19,11 @@ class VFS(object):
         self.uwrite = uwrite  # users who can write this
         self.flags = flags  # config switches
         self.nodes = {}  # child nodes
+        self.all_vols = {vpath: self}  # flattened recursive
+
+    def _trk(self, vol):
+        self.all_vols[vol.vpath] = vol
+        return vol
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""
@@ -30,7 +35,7 @@ class VFS(object):
             name, dst = dst.split("/", 1)
             if name in self.nodes:
                 # exists; do not manipulate permissions
-                return self.nodes[name].add(src, dst)
+                return self._trk(self.nodes[name].add(src, dst))
 
             vn = VFS(
                 "{}/{}".format(self.realpath, name),
@@ -40,7 +45,7 @@ class VFS(object):
                 self.flags,
             )
             self.nodes[name] = vn
-            return vn.add(src, dst)
+            return self._trk(vn.add(src, dst))
 
         if dst in self.nodes:
             # leaf exists; return as-is
@@ -50,7 +55,7 @@ class VFS(object):
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
         self.nodes[dst] = vn
-        return vn
+        return self._trk(vn)
 
     def _find(self, vpath):
         """return [vfs,remainder]"""
@@ -257,7 +262,6 @@ class AuthSrv(object):
                 with open(cfg_fn, "rb") as f:
                     self._parse_config_file(f, user, mread, mwrite, mflags, mount)
 
-        self.all_writable = []
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
@@ -280,11 +284,6 @@ class AuthSrv(object):
             v.uread = mread[dst]
             v.uwrite = mwrite[dst]
             v.flags = mflags[dst]
-            if v.uwrite:
-                self.all_writable.append(v)
-
-        if vfs.uwrite and vfs not in self.all_writable:
-            self.all_writable.append(vfs)
 
         missing_users = {}
         for d in [mread, mwrite]:

copyparty/httpcli.py

@@ -5,6 +5,7 @@ import os
 import stat
 import gzip
 import time
+import copy
 import json
 import socket
 import ctypes
@@ -34,6 +35,7 @@ class HttpCli(object):
         self.auth = conn.auth
         self.log_func = conn.log_func
         self.log_src = conn.log_src
+        self.tls = hasattr(self.s, "cipher")
 
         self.bufsz = 1024 * 32
         self.absolute_urls = False
@@ -75,6 +77,8 @@ class HttpCli(object):
             self.loud_reply(str(ex), status=ex.code)
             return self.keepalive
 
+        # time.sleep(0.4)
+
         # normalize incoming headers to lowercase;
         # outgoing headers however are Correct-Case
         for header_line in headerlines[1:]:
@@ -124,15 +128,15 @@ class HttpCli(object):
                     k, v = k.split("=", 1)
                     uparam[k.lower()] = v.strip()
                 else:
-                    uparam[k.lower()] = True
+                    uparam[k.lower()] = False
 
         self.uparam = uparam
         self.vpath = unquotep(vpath)
 
         ua = self.headers.get("user-agent", "")
         if ua.startswith("rclone/"):
-            uparam["raw"] = True
+            uparam["raw"] = False
-            uparam["dots"] = True
+            uparam["dots"] = False
 
         try:
             if self.mode in ["GET", "HEAD"]:
@@ -236,12 +240,15 @@ class HttpCli(object):
         )
         if not self.readable and not self.writable:
             self.log("inaccessible: [{}]".format(self.vpath))
-            self.uparam = {"h": True}
+            self.uparam = {"h": False}
 
         if "h" in self.uparam:
             self.vpath = None
             return self.tx_mounts()
 
+        if "tree" in self.uparam:
+            return self.tx_tree()
+
         return self.tx_browser()
 
     def handle_options(self):
@@ -294,16 +301,37 @@ class HttpCli(object):
         if "application/octet-stream" in ctype:
             return self.handle_post_binary()
 
-        raise Pebkac(405, "don't know how to handle {} POST".format(ctype))
+        if "application/x-www-form-urlencoded" in ctype:
+            opt = self.args.urlform
+            if "stash" in opt:
+                return self.handle_stash()
 
-    def handle_stash(self):
+            if "save" in opt:
+                post_sz, _, _, path = self.dump_to_file()
+                self.log("urlform: {} bytes, {}".format(post_sz, path))
+            elif "print" in opt:
+                reader, _ = self.get_body_reader()
+                for buf in reader:
+                    buf = buf.decode("utf-8", "replace")
+                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
+
+            if "get" in opt:
+                return self.handle_get()
+
+            raise Pebkac(405, "POST({}) is disabled".format(ctype))
+
+        raise Pebkac(405, "don't know how to handle POST({})".format(ctype))
+
+    def get_body_reader(self):
         remains = int(self.headers.get("content-length", None))
         if remains is None:
-            reader = read_socket_unbounded(self.sr)
             self.keepalive = False
+            return read_socket_unbounded(self.sr), remains
         else:
-            reader = read_socket(self.sr, remains)
+            return read_socket(self.sr, remains), remains
 
+    def dump_to_file(self):
+        reader, remains = self.get_body_reader()
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
         fdir = os.path.join(vfs.realpath, rem)
 
@@ -314,6 +342,10 @@ class HttpCli(object):
         with open(path, "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
+        return post_sz, sha_b64, remains, path
+
+    def handle_stash(self):
+        post_sz, sha_b64, remains, path = self.dump_to_file()
         spd = self._spd(post_sz)
         self.log("{} wrote {}/{} bytes to {}".format(spd, post_sz, remains, path))
         self.reply("{}\n{}\n".format(post_sz, sha_b64).encode("utf-8"))
@@ -375,6 +407,9 @@ class HttpCli(object):
         except:
             raise Pebkac(422, "you POSTed invalid json")
 
+        if "srch" in self.uparam or "srch" in body:
+            return self.handle_search(body)
+
         # prefer this over undot; no reason to allow traversion
         if "/" in body["name"]:
             raise Pebkac(400, "folders verboten")
@@ -400,6 +435,30 @@ class HttpCli(object):
         self.reply(response.encode("utf-8"), mime="application/json")
         return True
 
+    def handle_search(self, body):
+        vols = []
+        for vtop in self.rvol:
+            vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
+            vols.append([vfs.vpath, vfs.realpath, vfs.flags])
+
+        idx = self.conn.get_u2idx()
+        if "srch" in body:
+            # search by up2k hashlist
+            vbody = copy.deepcopy(body)
+            vbody["hash"] = len(vbody["hash"])
+            self.log("qj: " + repr(vbody))
+            hits = idx.fsearch(vols, body)
+            self.log("q#: " + repr(hits))
+        else:
+            # search by query params
+            self.log("qj: " + repr(body))
+            hits = idx.search(vols, body)
+            self.log("q#: " + str(len(hits)))
+
+        r = json.dumps(hits).encode("utf-8")
+        self.reply(r, mime="application/json")
+        return True
+
     def handle_post_binary(self):
         try:
             remains = int(self.headers["content-length"])
@@ -461,7 +520,12 @@ class HttpCli(object):
                 self.log("clone {} done".format(cstart[0]))
 
         x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", ptop, wark, chash)
-        num_left, path = x.get()
+        x = x.get()
+        try:
+            num_left, path = x
+        except:
+            self.loud_reply(x, status=500)
+            return False
 
         if not WINDOWS and num_left == 0:
             times = (int(time.time()), int(lastmod))
@@ -517,10 +581,9 @@ class HttpCli(object):
                 raise Pebkac(500, "mkdir failed, check the logs")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
+        esc_paths = [quotep(vpath), html_escape(vpath)]
         html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">go to /{}</a>'.format(
+            h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
-                quotep(vpath), html_escape(vpath)
-            ),
             pre="aight",
             click=True,
         )
@@ -903,8 +966,11 @@ class HttpCli(object):
             open_func = open
             # 512 kB is optimal for huge files, use 64k
             open_args = [fsenc(fs_path), "rb", 64 * 1024]
-            if hasattr(os, "sendfile"):
+            use_sendfile = (
-                use_sendfile = not self.args.no_sendfile
+                not self.tls  #
+                and not self.args.no_sendfile
+                and hasattr(os, "sendfile")
+            )
 
         #
         # send reply
@@ -1004,6 +1070,60 @@ class HttpCli(object):
         self.reply(html.encode("utf-8"))
         return True
 
+    def tx_tree(self):
+        top = self.uparam["tree"] or ""
+        dst = self.vpath
+        if top in [".", ".."]:
+            top = undot(self.vpath + "/" + top)
+
+        if top == dst:
+            dst = ""
+        elif top:
+            if not dst.startswith(top + "/"):
+                raise Pebkac(400, "arg funk")
+
+            dst = dst[len(top) + 1 :]
+
+        ret = self.gen_tree(top, dst)
+        ret = json.dumps(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
+        return True
+
+    def gen_tree(self, top, target):
+        ret = {}
+        excl = None
+        if target:
+            excl, target = (target.split("/", 1) + [""])[:2]
+            ret["k" + excl] = self.gen_tree("/".join([top, excl]).strip("/"), target)
+
+        try:
+            vn, rem = self.auth.vfs.get(top, self.uname, True, False)
+            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)
+        except:
+            vfs_ls = []
+            vfs_virt = {}
+            for v in self.rvol:
+                d1, d2 = v.rsplit("/", 1) if "/" in v else ["", v]
+                if d1 == top:
+                    vfs_virt[d2] = 0
+
+        dirs = []
+
+        if not self.args.ed or "dots" not in self.uparam:
+            vfs_ls = exclude_dotfiles(vfs_ls)
+
+        for fn in [x for x in vfs_ls if x != excl]:
+            abspath = os.path.join(fsroot, fn)
+            if os.path.isdir(abspath):
+                dirs.append(fn)
+
+        for x in vfs_virt.keys():
+            if x != excl:
+                dirs.append(x)
+
+        ret["a"] = dirs
+        return ret
+
     def tx_browser(self):
         vpath = ""
         vpnodes = [["", "/"]]
@@ -1029,8 +1149,7 @@ class HttpCli(object):
             if abspath.endswith(".md") and "raw" not in self.uparam:
                 return self.tx_md(abspath)
 
-            bad = "{0}.hist{0}up2k.".format(os.sep)
+            if rem.startswith(".hist/up2k."):
-            if abspath.endswith(bad + "db") or abspath.endswith(bad + "snap"):
                 raise Pebkac(403)
 
             return self.tx_file(abspath)
@@ -1058,17 +1177,25 @@ class HttpCli(object):
         if not self.args.ed or "dots" not in self.uparam:
             vfs_ls = exclude_dotfiles(vfs_ls)
 
+        hidden = []
+        if rem == ".hist":
+            hidden = ["up2k."]
+
+        is_ls = "ls" in self.uparam
+
         dirs = []
         files = []
         for fn in vfs_ls:
             base = ""
             href = fn
-            if self.absolute_urls and vpath:
+            if not is_ls and self.absolute_urls and vpath:
                 base = "/" + vpath + "/"
                 href = base + fn
 
             if fn in vfs_virt:
                 fspath = vfs_virt[fn].realpath
+            elif hidden and any(fn.startswith(x) for x in hidden):
+                continue
             else:
                 fspath = fsroot + "/" + fn
 
@@ -1098,30 +1225,20 @@ class HttpCli(object):
             except:
                 ext = "%"
 
-            item = [margin, quotep(href), html_escape(fn), sz, ext, dt]
+            item = {
+                "lead": margin,
+                "href": quotep(href),
+                "name": fn,
+                "sz": sz,
+                "ext": ext,
+                "dt": dt,
+                "ts": inf.st_mtime,
+            }
             if is_dir:
                 dirs.append(item)
             else:
                 files.append(item)
 
-        logues = [None, None]
-        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
-            fn = os.path.join(abspath, fn)
-            if os.path.exists(fsenc(fn)):
-                with open(fsenc(fn), "rb") as f:
-                    logues[n] = f.read().decode("utf-8")
-
-        if False:
-            # this is a mistake
-            md = None
-            for fn in [x[2] for x in files]:
-                if fn.lower() == "readme.md":
-                    fn = os.path.join(abspath, fn)
-                    with open(fn, "rb") as f:
-                        md = f.read().decode("utf-8")
-
-                    break
-
         srv_info = []
 
         try:
@@ -1150,21 +1267,44 @@ class HttpCli(object):
         except:
             pass
 
+        srv_info = "</span> /// <span>".join(srv_info)
+
+        perms = []
+        if self.readable:
+            perms.append("read")
+        if self.writable:
+            perms.append("write")
+
+        if is_ls:
+            [x.pop(k) for k in ["name", "dt"] for y in [dirs, files] for x in y]
+            ret = {"dirs": dirs, "files": files, "srvinf": srv_info, "perms": perms}
+            ret = json.dumps(ret)
+            self.reply(ret.encode("utf-8", "replace"), mime="application/json")
+            return True
+
+        logues = [None, None]
+        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
+            fn = os.path.join(abspath, fn)
+            if os.path.exists(fsenc(fn)):
+                with open(fsenc(fn), "rb") as f:
+                    logues[n] = f.read().decode("utf-8")
+
         ts = ""
         # ts = "?{}".format(time.time())
 
         dirs.extend(files)
+
         html = self.conn.tpl_browser.render(
             vdir=quotep(self.vpath),
             vpnodes=vpnodes,
             files=dirs,
-            can_upload=self.writable,
-            can_read=self.readable,
             ts=ts,
+            perms=json.dumps(perms),
+            have_up2k_idx=self.args.e2d,
             prologue=logues[0],
             epilogue=logues[1],
             title=html_escape(self.vpath),
-            srv_info="</span> /// <span>".join(srv_info),
+            srv_info=srv_info,
         )
         self.reply(html.encode("utf-8", "replace"))
         return True

copyparty/httpconn.py

@@ -3,10 +3,15 @@ from __future__ import print_function, unicode_literals
 
 import os
 import sys
-import ssl
 import time
 import socket
 
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
+
 try:
     import jinja2
 except ImportError:
@@ -25,6 +30,7 @@ except ImportError:
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
+from .u2idx import U2idx
 
 
 class HttpConn(object):
@@ -45,6 +51,7 @@ class HttpConn(object):
         self.t0 = time.time()
         self.nbyte = 0
         self.workload = 0
+        self.u2idx = None
         self.log_func = hsrv.log
         self.set_rproxy()
 
@@ -75,9 +82,14 @@ class HttpConn(object):
     def log(self, msg):
         self.log_func(self.log_src, msg)
 
-    def run(self):
+    def get_u2idx(self):
+        if not self.u2idx:
+            self.u2idx = U2idx(self.args, self.log_func)
+
+        return self.u2idx
+
+    def _detect_https(self):
         method = None
-        self.sr = None
         if self.cert_path:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
@@ -102,16 +114,58 @@ class HttpConn(object):
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 
-        if method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]:
+        return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
+
+    def run(self):
+        self.sr = None
+        if self.args.https_only:
+            is_https = True
+        elif self.args.http_only or not HAVE_SSL:
+            is_https = False
+        else:
+            is_https = self._detect_https()
+
+        if is_https:
             if self.sr:
                 self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
                 return
 
             self.log_src = self.log_src.replace("[36m", "[35m")
             try:
-                self.s = ssl.wrap_socket(
+                ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-                    self.s, server_side=True, certfile=self.cert_path
+                ctx.load_cert_chain(self.cert_path)
-                )
+                if self.args.ssl_ver:
+                    ctx.options &= ~self.args.ssl_flags_en
+                    ctx.options |= self.args.ssl_flags_de
+                    # print(repr(ctx.options))
+
+                if self.args.ssl_log:
+                    try:
+                        ctx.keylog_filename = self.args.ssl_log
+                    except:
+                        self.log("keylog failed; openssl or python too old")
+
+                if self.args.ciphers:
+                    ctx.set_ciphers(self.args.ciphers)
+
+                self.s = ctx.wrap_socket(self.s, server_side=True)
+                msg = [
+                    "\033[1;3{:d}m{}".format(c, s)
+                    for c, s in zip([0, 5, 0], self.s.cipher())
+                ]
+                self.log(" ".join(msg) + "\033[0m")
+
+                if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
+                    overlap = [y[::-1] for y in self.s.shared_ciphers()]
+                    lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
+                    self.log("\n".join(lines))
+                    for k, v in [
+                        ["compression", self.s.compression()],
+                        ["ALPN proto", self.s.selected_alpn_protocol()],
+                        ["NPN proto", self.s.selected_npn_protocol()],
+                    ]:
+                        self.log("TLS {}: {}".format(k, v or "nah"))
+
             except Exception as ex:
                 em = str(ex)
 

copyparty/httpsrv.py

@@ -78,7 +78,7 @@ class HttpSrv(object):
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "shut_rdwr err:\n  {}\n  {}".format(repr(sck), ex),
+                        "\033[1;30mshut({}): {}\033[0m".format(sck.fileno(), ex),
                     )
                 if ex.errno not in [10038, 10054, 107, 57, 9]:
                     # 10038 No longer considered a socket

copyparty/svchub.py

@@ -39,9 +39,13 @@ class SvcHub(object):
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
-        if self.args.e2d and self.args.e2s:
+        if self.args.e2ds:
             auth = AuthSrv(self.args, self.log, False)
-            self.up2k.build_indexes(auth.all_writable)
+            vols = auth.vfs.all_vols.values()
+            if not self.args.e2dsa:
+                vols = [x for x in vols if x.uwrite]
+
+            self.up2k.build_indexes(vols)
 
         # decide which worker impl to use
         if self.check_mp_enable():
@@ -79,7 +83,7 @@ class SvcHub(object):
             now = time.time()
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
-                print("\033[36m{}\033[0m".format(dt.strftime("%Y-%m-%d")))
+                print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
 
                 # unix timestamp of next 00:00:00 (leap-seconds safe)
                 day_now = dt.day
@@ -89,7 +93,7 @@ class SvcHub(object):
                 dt = dt.replace(hour=0, minute=0, second=0)
                 self.next_day = calendar.timegm(dt.utctimetuple())
 
-            fmt = "\033[36m{} \033[33m{:21} \033[0m{}"
+            fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
                 fmt = "{} {:21} {}"
                 if "\033" in msg:
@@ -100,12 +104,12 @@ class SvcHub(object):
             ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             msg = fmt.format(ts, src, msg)
             try:
-                print(msg)
+                print(msg, end="")
             except UnicodeEncodeError:
                 try:
-                    print(msg.encode("utf-8", "replace").decode())
+                    print(msg.encode("utf-8", "replace").decode(), end="")
                 except:
-                    print(msg.encode("ascii", "replace").decode())
+                    print(msg.encode("ascii", "replace").decode(), end="")
 
     def check_mp_support(self):
         vmin = sys.version_info[1]

copyparty/tcpsrv.py

@@ -53,15 +53,13 @@ class TcpSrv(object):
             srv.bind((ip, port))
             return srv
         except (OSError, socket.error) as ex:
-            if ex.errno == 98:
+            if ex.errno in [98, 48]:
-                raise Exception(
+                e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
-                    "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
+            elif ex.errno in [99, 49]:
-                )
+                e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
-
+            else:
-            if ex.errno == 99:
+                raise
-                raise Exception(
+            raise Exception(e)
-                    "\033[1;31minterface {} does not exist\033[0m".format(ip)
-                )
 
     def run(self):
         for srv in self.srv:

copyparty/u2idx.py

@@ -0,0 +1,148 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+from datetime import datetime
+
+from .util import u8safe
+from .up2k import up2k_wark_from_hashlist
+
+
+try:
+    HAVE_SQLITE3 = True
+    import sqlite3
+except:
+    HAVE_SQLITE3 = False
+
+
+class U2idx(object):
+    def __init__(self, args, log_func):
+        self.args = args
+        self.log_func = log_func
+
+        if not HAVE_SQLITE3:
+            self.log("could not load sqlite3; searchign wqill be disabled")
+            return
+
+        self.dbs = {}
+
+    def log(self, msg):
+        self.log_func("u2idx", msg)
+
+    def fsearch(self, vols, body):
+        """search by up2k hashlist"""
+        if not HAVE_SQLITE3:
+            return []
+
+        fsize = body["size"]
+        fhash = body["hash"]
+        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
+        return self.run_query(vols, "select * from up where w = ?", [wark])
+
+    def search(self, vols, body):
+        """search by query params"""
+        if not HAVE_SQLITE3:
+            return []
+
+        qobj = {}
+        _conv_sz(qobj, body, "sz_min", "sz >= ?")
+        _conv_sz(qobj, body, "sz_max", "sz <= ?")
+        _conv_dt(qobj, body, "dt_min", "mt >= ?")
+        _conv_dt(qobj, body, "dt_max", "mt <= ?")
+        for seg, dk in [["path", "rd"], ["name", "fn"]]:
+            if seg in body:
+                _conv_txt(qobj, body, seg, dk)
+
+        qstr = "select * from up"
+        qv = []
+        if qobj:
+            qk = []
+            for k, v in sorted(qobj.items()):
+                qk.append(k.split("\n")[0])
+                qv.append(v)
+
+            qstr = " and ".join(qk)
+            qstr = "select * from up where " + qstr
+
+        return self.run_query(vols, qstr, qv)
+
+    def run_query(self, vols, qstr, qv):
+        qv = tuple(qv)
+        self.log("qs: {} {}".format(qstr, repr(qv)))
+
+        ret = []
+        lim = 100
+        for (vtop, ptop, flags) in vols:
+            db = self.dbs.get(ptop)
+            if not db:
+                db = _open(ptop)
+                if not db:
+                    continue
+
+                self.dbs[ptop] = db
+                # self.log("idx /{} @ {} {}".format(vtop, ptop, flags))
+
+            c = db.execute(qstr, qv)
+            for _, ts, sz, rd, fn in c:
+                lim -= 1
+                if lim <= 0:
+                    break
+
+                rp = os.path.join(vtop, rd, fn).replace("\\", "/")
+                ret.append({"ts": int(ts), "sz": sz, "rp": rp})
+
+        return ret
+
+
+def _open(ptop):
+    db_path = os.path.join(ptop, ".hist", "up2k.db")
+    if os.path.exists(db_path):
+        return sqlite3.connect(db_path)
+
+
+def _conv_sz(q, body, k, sql):
+    if k in body:
+        q[sql] = int(float(body[k]) * 1024 * 1024)
+
+
+def _conv_dt(q, body, k, sql):
+    if k not in body:
+        return
+
+    v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ")
+    while "  " in v:
+        v = v.replace("  ", " ")
+
+    for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]:
+        try:
+            ts = datetime.strptime(v, fmt).timestamp()
+            break
+        except:
+            ts = None
+
+    if ts:
+        q[sql] = ts
+
+
+def _conv_txt(q, body, k, sql):
+    for v in body[k].split(" "):
+        inv = ""
+        if v.startswith("-"):
+            inv = "not"
+            v = v[1:]
+
+        if not v:
+            continue
+
+        head = "'%'||"
+        if v.startswith("^"):
+            head = ""
+            v = v[1:]
+
+        tail = "||'%'"
+        if v.endswith("$"):
+            tail = ""
+            v = v[:-1]
+
+        qk = "{} {} like {}?{}".format(sql, inv, head, tail)
+        q[qk + "\n" + v] = u8safe(v)

copyparty/up2k.py

@@ -1,9 +1,8 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-
-import os
 import re
+import os
 import time
 import math
 import json
@@ -16,15 +15,24 @@ import threading
 from copy import deepcopy
 
 from .__init__ import WINDOWS
-from .util import Pebkac, Queue, fsdec, fsenc, sanitize_fn, ren_open, atomic_move
+from .util import (
+    Pebkac,
+    Queue,
+    ProgressPrinter,
+    fsdec,
+    fsenc,
+    sanitize_fn,
+    ren_open,
+    atomic_move,
+    w8b64enc,
+    w8b64dec,
+)
 
-HAVE_SQLITE3 = False
 try:
-    import sqlite3
-
     HAVE_SQLITE3 = True
+    import sqlite3
 except:
-    pass
+    HAVE_SQLITE3 = False
 
 
 class Up2k(object):
@@ -38,17 +46,24 @@ class Up2k(object):
     def __init__(self, broker):
         self.broker = broker
         self.args = broker.args
-        self.log = broker.log
+        self.log_func = broker.log
         self.persist = self.args.e2d
 
         # config
-        self.salt = "hunter2"  # TODO: config
+        self.salt = broker.args.salt
 
         # state
         self.mutex = threading.Lock()
         self.registry = {}
         self.db = {}
 
+        self.mem_db = None
+        if HAVE_SQLITE3:
+            # mojibake detector
+            self.mem_db = sqlite3.connect(":memory:", check_same_thread=False)
+            self.mem_db.execute(r"create table a (b text)")
+            self.mem_db.commit()
+
         if WINDOWS:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
@@ -65,8 +80,33 @@ class Up2k(object):
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$")
 
         if self.persist and not HAVE_SQLITE3:
-            m = "could not initialize sqlite3, will use in-memory registry only"
+            self.log("could not initialize sqlite3, will use in-memory registry only")
-            self.log("up2k", m)
+
+    def log(self, msg):
+        self.log_func("up2k", msg + "\033[K")
+
+    def w8enc(self, rd, fn):
+        ret = []
+        for k, v in [["d", rd], ["f", fn]]:
+            try:
+                self.mem_db.execute("select * from a where b = ?", (v,))
+                ret.append(v)
+            except:
+                ret.append("//" + w8b64enc(v))
+                # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+        return tuple(ret)
+
+    def w8dec(self, rd, fn):
+        ret = []
+        for k, v in [["d", rd], ["f", fn]]:
+            if v.startswith("//"):
+                ret.append(w8b64dec(v[2:]))
+                # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+            else:
+                ret.append(v)
+
+        return tuple(ret)
 
     def _vis_job_progress(self, job):
         perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
@@ -97,7 +137,7 @@ class Up2k(object):
 
                 m = "loaded snap {} |{}|".format(path, len(reg.keys()))
                 m = [m] + self._vis_reg_progress(reg)
-                self.log("up2k", "\n".join(m))
+                self.log("\n".join(m))
 
             self.registry[ptop] = reg
             if not self.persist or not HAVE_SQLITE3:
@@ -118,53 +158,87 @@ class Up2k(object):
                 self.db[ptop] = db
                 return db
             except Exception as ex:
-                m = "failed to open [{}]: {}".format(ptop, repr(ex))
+                self.log("cannot use database at [{}]: {}".format(ptop, repr(ex)))
-                self.log("up2k", m)
 
             return None
 
     def build_indexes(self, writeables):
         tops = [d.realpath for d in writeables]
+        self.pp = ProgressPrinter()
+        t0 = time.time()
         for top in tops:
             db = self.register_vpath(top)
-            if db:
+            if not db:
-                # can be symlink so don't `and d.startswith(top)``
+                continue
-                excl = set([d for d in tops if d != top])
+
-                self._build_dir([db, 0], top, excl, top)
+            self.pp.n = next(db.execute("select count(w) from up"))[0]
-                self._drop_lost(db, top)
+            db_path = os.path.join(top, ".hist", "up2k.db")
-                db.commit()
+            sz0 = os.path.getsize(db_path) // 1024
+
+            # can be symlink so don't `and d.startswith(top)``
+            excl = set([d for d in tops if d != top])
+            dbw = [db, 0, time.time()]
+
+            n_add = self._build_dir(dbw, top, excl, top)
+            n_rm = self._drop_lost(db, top)
+            if dbw[1]:
+                self.log("commit {} new files".format(dbw[1]))
+
+            db.commit()
+            if n_add or n_rm:
+                db_path = os.path.join(top, ".hist", "up2k.db")
+                sz1 = os.path.getsize(db_path) // 1024
+                db.execute("vacuum")
+                sz2 = os.path.getsize(db_path) // 1024
+                msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format(
+                    n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2
+                )
+                self.log(msg)
+
+        self.pp.end = True
+        self.log("{} volumes in {:.2f} sec".format(len(tops), time.time() - t0))
 
     def _build_dir(self, dbw, top, excl, cdir):
         try:
             inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
         except Exception as ex:
-            self.log("up2k", "listdir: " + repr(ex))
+            self.log("listdir: {} @ [{}]".format(repr(ex), cdir))
-            return
+            return 0
 
+        self.pp.msg = "a{} {}".format(self.pp.n, cdir)
         histdir = os.path.join(top, ".hist")
+        ret = 0
         for inode in inodes:
             abspath = os.path.join(cdir, inode)
             try:
                 inf = os.stat(fsenc(abspath))
             except Exception as ex:
-                self.log("up2k", "stat: " + repr(ex))
+                self.log("stat: {} @ [{}]".format(repr(ex), abspath))
                 continue
 
             if stat.S_ISDIR(inf.st_mode):
                 if abspath in excl or abspath == histdir:
                     continue
-                # self.log("up2k", " dir: {}".format(abspath))
+                # self.log(" dir: {}".format(abspath))
-                self._build_dir(dbw, top, excl, abspath)
+                ret += self._build_dir(dbw, top, excl, abspath)
             else:
-                # self.log("up2k", "file: {}".format(abspath))
+                # self.log("file: {}".format(abspath))
                 rp = abspath[len(top) :].replace("\\", "/").strip("/")
-                c = dbw[0].execute("select * from up where rp = ?", (rp,))
+                rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
+                sql = "select * from up where rd = ? and fn = ?"
+                try:
+                    c = dbw[0].execute(sql, (rd, fn))
+                except:
+                    c = dbw[0].execute(sql, self.w8enc(rd, fn))
+
                 in_db = list(c.fetchall())
                 if in_db:
-                    _, dts, dsz, _ = in_db[0]
+                    self.pp.n -= 1
+                    _, dts, dsz, _, _ = in_db[0]
                     if len(in_db) > 1:
-                        m = "WARN: multiple entries: [{}] => [{}] ({})"
+                        m = "WARN: multiple entries: [{}] => [{}] |{}|\n{}"
-                        self.log("up2k", m.format(top, rp, len(in_db)))
+                        rep_db = "\n".join([repr(x) for x in in_db])
+                        self.log(m.format(top, rp, len(in_db), rep_db))
                         dts = -1
 
                     if dts == inf.st_mtime and dsz == inf.st_size:
@@ -173,65 +247,84 @@ class Up2k(object):
                     m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format(
                         top, rp, dts, inf.st_mtime, dsz, inf.st_size
                     )
-                    self.log("up2k", m)
+                    self.log(m)
-                    self.db_rm(dbw[0], rp)
+                    self.db_rm(dbw[0], rd, fn)
+                    ret += 1
                     dbw[1] += 1
                     in_db = None
 
-                self.log("up2k", "file: {}".format(abspath))
+                self.pp.msg = "a{} {}".format(self.pp.n, abspath)
+                if inf.st_size > 1024 * 1024:
+                    self.log("file: {}".format(abspath))
+
                 try:
                     hashes = self._hashlist_from_file(abspath)
                 except Exception as ex:
-                    self.log("up2k", "hash: " + repr(ex))
+                    self.log("hash: {} @ [{}]".format(repr(ex), abspath))
                     continue
 
-                wark = self._wark_from_hashlist(inf.st_size, hashes)
+                wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
-                self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size)
+                self.db_add(dbw[0], wark, rd, fn, inf.st_mtime, inf.st_size)
                 dbw[1] += 1
-                if dbw[1] > 1024:
+                ret += 1
+                td = time.time() - dbw[2]
+                if dbw[1] >= 4096 or td >= 60:
+                    self.log("commit {} new files".format(dbw[1]))
                     dbw[0].commit()
                     dbw[1] = 0
+                    dbw[2] = time.time()
+        return ret
 
     def _drop_lost(self, db, top):
         rm = []
+        nchecked = 0
+        nfiles = next(db.execute("select count(w) from up"))[0]
         c = db.execute("select * from up")
-        for dwark, dts, dsz, drp in c:
+        for dwark, dts, dsz, drd, dfn in c:
-            abspath = os.path.join(top, drp)
+            nchecked += 1
+            if drd.startswith("//") or dfn.startswith("//"):
+                drd, dfn = self.w8dec(drd, dfn)
+
+            abspath = os.path.join(top, drd, dfn)
+            # almost zero overhead dw
+            self.pp.msg = "b{} {}".format(nfiles - nchecked, abspath)
             try:
                 if not os.path.exists(fsenc(abspath)):
-                    rm.append(drp)
+                    rm.append([drd, dfn])
             except Exception as ex:
-                self.log("up2k", "stat-rm: " + repr(ex))
+                self.log("stat-rm: {} @ [{}]".format(repr(ex), abspath))
 
-        if not rm:
+        if rm:
-            return
+            self.log("forgetting {} deleted files".format(len(rm)))
+            for rd, fn in rm:
+                # self.log("{} / {}".format(rd, fn))
+                self.db_rm(db, rd, fn)
 
-        self.log("up2k", "forgetting {} deleted files".format(len(rm)))
+        return len(rm)
-        for rp in rm:
-            self.db_rm(db, rp)
 
     def _open_db(self, db_path):
+        existed = os.path.exists(db_path)
         conn = sqlite3.connect(db_path, check_same_thread=False)
         try:
-            c = conn.execute(r"select * from kv where k = 'sver'")
+            ver = self._read_ver(conn)
-            rows = c.fetchall()
-            if rows:
-                ver = rows[0][1]
-            else:
-                self.log("up2k", "WARN: no sver in kv, DB corrupt?")
-                ver = "unknown"
 
-            if ver == "1":
+            if ver == 1:
+                conn = self._upgrade_v1(conn, db_path)
+                ver = self._read_ver(conn)
+
+            if ver == 2:
                 try:
                     nfiles = next(conn.execute("select count(w) from up"))[0]
-                    self.log("up2k", "found DB at {} |{}|".format(db_path, nfiles))
+                    self.log("found DB at {} |{}|".format(db_path, nfiles))
                     return conn
                 except Exception as ex:
-                    m = "WARN: could not list files, DB corrupt?\n  " + repr(ex)
+                    self.log("WARN: could not list files, DB corrupt?\n  " + repr(ex))
-                    self.log("up2k", m)
+
+            if ver is not None:
+                self.log("REPLACING unsupported DB (v.{}) at {}".format(ver, db_path))
+            elif not existed:
+                raise Exception("whatever")
 
-            m = "REPLACING unsupported DB (v.{}) at {}".format(ver, db_path)
-            self.log("up2k", m)
             conn.close()
             os.unlink(db_path)
             conn = sqlite3.connect(db_path, check_same_thread=False)
@@ -239,17 +332,58 @@ class Up2k(object):
             pass
 
         # sqlite is variable-width only, no point in using char/nchar/varchar
+        self._create_v2(conn)
+        conn.commit()
+        self.log("created DB at {}".format(db_path))
+        return conn
+
+    def _read_ver(self, conn):
+        for tab in ["ki", "kv"]:
+            try:
+                c = conn.execute(r"select v from {} where k = 'sver'".format(tab))
+            except:
+                continue
+
+            rows = c.fetchall()
+            if rows:
+                return int(rows[0][0])
+
+    def _create_v2(self, conn):
         for cmd in [
-            r"create table kv (k text, v text)",
+            r"create table ks (k text, v text)",
-            r"create table up (w text, mt int, sz int, rp text)",
+            r"create table ki (k text, v int)",
-            r"insert into kv values ('sver', '1')",
+            r"create table up (w text, mt int, sz int, rd text, fn text)",
+            r"insert into ki values ('sver', 2)",
             r"create index up_w on up(w)",
+            r"create index up_rd on up(rd)",
+            r"create index up_fn on up(fn)",
         ]:
             conn.execute(cmd)
 
-        conn.commit()
+    def _upgrade_v1(self, odb, db_path):
-        self.log("up2k", "created DB at {}".format(db_path))
+        self.log("\033[33mupgrading v1 to v2:\033[0m {}".format(db_path))
-        return conn
+
+        npath = db_path + ".next"
+        if os.path.exists(npath):
+            os.unlink(npath)
+
+        ndb = sqlite3.connect(npath, check_same_thread=False)
+        self._create_v2(ndb)
+
+        c = odb.execute("select * from up")
+        for wark, ts, sz, rp in c:
+            rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
+            v = (wark, ts, sz, rd, fn)
+            ndb.execute("insert into up values (?,?,?,?,?)", v)
+
+        ndb.commit()
+        ndb.close()
+        odb.close()
+        bpath = db_path + ".bak.v1"
+        self.log("success; backup at: " + bpath)
+        atomic_move(db_path, bpath)
+        atomic_move(npath, db_path)
+        return sqlite3.connect(db_path, check_same_thread=False)
 
     def handle_json(self, cj):
         self.register_vpath(cj["ptop"])
@@ -263,19 +397,16 @@ class Up2k(object):
             reg = self.registry[cj["ptop"]]
             if db:
                 cur = db.execute(r"select * from up where w = ?", (wark,))
-                for _, dtime, dsize, dp_rel in cur:
+                for _, dtime, dsize, dp_dir, dp_fn in cur:
-                    dp_abs = os.path.join(cj["ptop"], dp_rel).replace("\\", "/")
+                    if dp_dir.startswith("//") or dp_fn.startswith("//"):
+                        dp_dir, dp_fn = self.w8dec(dp_dir, dp_fn)
+
+                    dp_abs = os.path.join(cj["ptop"], dp_dir, dp_fn).replace("\\", "/")
                     # relying on path.exists to return false on broken symlinks
                     if os.path.exists(fsenc(dp_abs)):
-                        try:
-                            prel, name = dp_rel.rsplit("/", 1)
-                        except:
-                            prel = ""
-                            name = dp_rel
-
                         job = {
-                            "name": name,
+                            "name": dp_fn,
-                            "prel": prel,
+                            "prel": dp_dir,
                             "vtop": cj["vtop"],
                             "ptop": cj["ptop"],
                             "flag": cj["flag"],
@@ -311,13 +442,13 @@ class Up2k(object):
                     vsrc = os.path.join(job["vtop"], job["prel"], job["name"])
                     vsrc = vsrc.replace("\\", "/")  # just for prints anyways
                     if job["need"]:
-                        self.log("up2k", "unfinished:\n  {0}\n  {1}".format(src, dst))
+                        self.log("unfinished:\n  {0}\n  {1}".format(src, dst))
                         err = "partial upload exists at a different location; please resume uploading here instead:\n"
-                        err += vsrc + " "
+                        err += "/" + vsrc + " "
                         raise Pebkac(400, err)
                     elif "nodupe" in job["flag"]:
-                        self.log("up2k", "dupe-reject:\n  {0}\n  {1}".format(src, dst))
+                        self.log("dupe-reject:\n  {0}\n  {1}".format(src, dst))
-                        err = "upload rejected, file already exists:\n " + vsrc + " "
+                        err = "upload rejected, file already exists:\n/" + vsrc + " "
                         raise Pebkac(400, err)
                     else:
                         # symlink to the client-provided name,
@@ -381,7 +512,7 @@ class Up2k(object):
 
     def _symlink(self, src, dst):
         # TODO store this in linktab so we never delete src if there are links to it
-        self.log("up2k", "linking dupe:\n  {0}\n  {1}".format(src, dst))
+        self.log("linking dupe:\n  {0}\n  {1}".format(src, dst))
         try:
             lsrc = src
             ldst = dst
@@ -404,7 +535,7 @@ class Up2k(object):
                     lsrc = "../" * (len(lsrc) - 1) + "/".join(lsrc)
             os.symlink(fsenc(lsrc), fsenc(ldst))
         except (AttributeError, OSError) as ex:
-            self.log("up2k", "cannot symlink; creating copy: " + repr(ex))
+            self.log("cannot symlink; creating copy: " + repr(ex))
             shutil.copy2(fsenc(src), fsenc(dst))
 
     def handle_chunk(self, ptop, wark, chash):
@@ -414,7 +545,7 @@ class Up2k(object):
                 raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
-                raise Pebkac(200, "already got that but thanks??")
+                raise Pebkac(400, "already got that but thanks??")
 
             nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
             if not nchunk:
@@ -422,7 +553,7 @@ class Up2k(object):
 
         job["poke"] = time.time()
 
-        chunksize = self._get_chunksize(job["size"])
+        chunksize = up2k_chunksize(job["size"])
         ofs = [chunksize * x for x in nchunk]
 
         path = os.path.join(job["ptop"], job["prel"], job["tnam"])
@@ -431,12 +562,19 @@ class Up2k(object):
 
     def confirm_chunk(self, ptop, wark, chash):
         with self.mutex:
-            job = self.registry[ptop][wark]
+            try:
-            pdir = os.path.join(job["ptop"], job["prel"])
+                job = self.registry[ptop][wark]
-            src = os.path.join(pdir, job["tnam"])
+                pdir = os.path.join(job["ptop"], job["prel"])
-            dst = os.path.join(pdir, job["name"])
+                src = os.path.join(pdir, job["tnam"])
+                dst = os.path.join(pdir, job["name"])
+            except Exception as ex:
+                return "confirm_chunk, wark, " + repr(ex)
+
+            try:
+                job["need"].remove(chash)
+            except Exception as ex:
+                return "confirm_chunk, chash, " + repr(ex)
 
-            job["need"].remove(chash)
             ret = len(job["need"])
             if ret > 0:
                 return ret, src
@@ -448,33 +586,31 @@ class Up2k(object):
 
             db = self.db.get(job["ptop"], None)
             if db:
-                rp = os.path.join(job["prel"], job["name"]).replace("\\", "/")
+                j = job
-                self.db_rm(db, rp)
+                self.db_rm(db, j["prel"], j["name"])
-                self.db_add(db, job["wark"], rp, job["lmod"], job["size"])
+                self.db_add(db, j["wark"], j["prel"], j["name"], j["lmod"], j["size"])
                 db.commit()
                 del self.registry[ptop][wark]
                 # in-memory registry is reserved for unfinished uploads
 
             return ret, dst
 
-    def _get_chunksize(self, filesize):
+    def db_rm(self, db, rd, fn):
-        chunksize = 1024 * 1024
+        sql = "delete from up where rd = ? and fn = ?"
-        stepsize = 512 * 1024
+        try:
-        while True:
+            db.execute(sql, (rd, fn))
-            for mul in [1, 2]:
+        except:
-                nchunks = math.ceil(filesize * 1.0 / chunksize)
+            db.execute(sql, self.w8enc(rd, fn))
-                if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
-                    return chunksize
 
-                chunksize += stepsize
+    def db_add(self, db, wark, rd, fn, ts, sz):
-                stepsize *= mul
+        sql = "insert into up values (?,?,?,?,?)"
-
+        v = (wark, ts, sz, rd, fn)
-    def db_rm(self, db, rp):
+        try:
-        db.execute("delete from up where rp = ?", (rp,))
+            db.execute(sql, v)
-
+        except:
-    def db_add(self, db, wark, rp, ts, sz):
+            rd, fn = self.w8enc(rd, fn)
-        v = (wark, ts, sz, rp)
+            v = (wark, ts, sz, rd, fn)
-        db.execute("insert into up values (?,?,?,?)", v)
+            db.execute(sql, v)
 
     def _get_wark(self, cj):
         if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024:  # 16TiB
@@ -492,28 +628,17 @@ class Up2k(object):
         except:
             cj["lmod"] = int(time.time())
 
-        wark = self._wark_from_hashlist(cj["size"], cj["hash"])
+        wark = up2k_wark_from_hashlist(self.salt, cj["size"], cj["hash"])
         return wark
 
-    def _wark_from_hashlist(self, filesize, hashes):
-        """ server-reproducible file identifier, independent of name or location """
-        ident = [self.salt, str(filesize)]
-        ident.extend(hashes)
-        ident = "\n".join(ident)
-
-        hasher = hashlib.sha512()
-        hasher.update(ident.encode("utf-8"))
-        digest = hasher.digest()[:32]
-
-        wark = base64.urlsafe_b64encode(digest)
-        return wark.decode("utf-8").rstrip("=")
-
     def _hashlist_from_file(self, path):
         fsz = os.path.getsize(path)
-        csz = self._get_chunksize(fsz)
+        csz = up2k_chunksize(fsz)
         ret = []
+        last_print = time.time()
         with open(path, "rb", 512 * 1024) as f:
             while fsz > 0:
+                self.pp.msg = msg = "{} MB".format(int(fsz / 1024 / 1024))
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
                 fsz -= rem
@@ -551,13 +676,14 @@ class Up2k(object):
             while not self.lastmod_q.empty():
                 ready.append(self.lastmod_q.get())
 
-            # self.log("lmod", "got {}".format(len(ready)))
+            # self.log("lmod: got {}".format(len(ready)))
             time.sleep(5)
             for path, times in ready:
+                self.log("lmod: setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:
-                    self.log("lmod", "failed to utime ({}, {})".format(path, times))
+                    self.log("lmod: failed to utime ({}, {})".format(path, times))
 
     def _snapshot(self):
         persist_interval = 30  # persist unfinished uploads index every 30 sec
@@ -575,7 +701,7 @@ class Up2k(object):
         if rm:
             m = "dropping {} abandoned uploads in {}".format(len(rm), k)
             vis = [self._vis_job_progress(x) for x in rm]
-            self.log("up2k", "\n".join([m] + vis))
+            self.log("\n".join([m] + vis))
             for job in rm:
                 del reg[job["wark"]]
                 try:
@@ -604,6 +730,11 @@ class Up2k(object):
         if etag == prev.get(k, None):
             return
 
+        try:
+            os.mkdir(os.path.join(k, ".hist"))
+        except:
+            pass
+
         path2 = "{}.{}".format(path, os.getpid())
         j = json.dumps(reg, indent=2, sort_keys=True).encode("utf-8")
         with gzip.GzipFile(path2, "wb") as f:
@@ -611,5 +742,32 @@ class Up2k(object):
 
         atomic_move(path2, path)
 
-        self.log("up2k", "snap: {} |{}|".format(path, len(reg.keys())))
+        self.log("snap: {} |{}|".format(path, len(reg.keys())))
         prev[k] = etag
+
+
+def up2k_chunksize(filesize):
+    chunksize = 1024 * 1024
+    stepsize = 512 * 1024
+    while True:
+        for mul in [1, 2]:
+            nchunks = math.ceil(filesize * 1.0 / chunksize)
+            if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
+                return chunksize
+
+            chunksize += stepsize
+            stepsize *= mul
+
+
+def up2k_wark_from_hashlist(salt, filesize, hashes):
+    """ server-reproducible file identifier, independent of name or location """
+    ident = [salt, str(filesize)]
+    ident.extend(hashes)
+    ident = "\n".join(ident)
+
+    hasher = hashlib.sha512()
+    hasher.update(ident.encode("utf-8"))
+    digest = hasher.digest()[:32]
+
+    wark = base64.urlsafe_b64encode(digest)
+    return wark.decode("utf-8").rstrip("=")

copyparty/util.py

@@ -99,6 +99,39 @@ class Unrecv(object):
         self.buf = buf + self.buf
 
 
+class ProgressPrinter(threading.Thread):
+    """
+    periodically print progress info without linefeeds
+    """
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.msg = None
+        self.end = False
+        self.start()
+
+    def run(self):
+        msg = None
+        while not self.end:
+            time.sleep(0.1)
+            if msg == self.msg or self.end:
+                continue
+
+            msg = self.msg
+            m = " {}\033[K\r".format(msg)
+            try:
+                print(m, end="")
+            except UnicodeEncodeError:
+                try:
+                    print(m.encode("utf-8", "replace").decode(), end="")
+                except:
+                    print(m.encode("ascii", "replace").decode(), end="")
+
+        print("\033[K", end="")
+        sys.stdout.flush()  # necessary on win10 even w/ stderr btw
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -108,7 +141,7 @@ def ren_open(fname, *args, **kwargs):
         with open(fname, *args, **kwargs) as f:
             yield {"orz": [f, fname]}
             return
-    
+
     orig_name = fname
     bname = fname
     ext = ""
@@ -146,7 +179,7 @@ def ren_open(fname, *args, **kwargs):
 
         except OSError as ex_:
             ex = ex_
-            if ex.errno != 36:
+            if ex.errno not in [36, 63] and (not WINDOWS or ex.errno != 22):
                 raise
 
         if not b64:
@@ -480,6 +513,13 @@ def sanitize_fn(fn):
     return fn.strip()
 
 
+def u8safe(txt):
+    try:
+        return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
+    except:
+        return txt.encode("utf-8", "replace").decode("utf-8", "replace")
+
+
 def exclude_dotfiles(filepaths):
     for fpath in filepaths:
         if not fpath.split("/")[-1].startswith("."):
@@ -536,6 +576,16 @@ def w8enc(txt):
     return txt.encode(FS_ENCODING, "surrogateescape")
 
 
+def w8b64dec(txt):
+    """decodes base64(filesystem-bytes) to wtf8"""
+    return w8dec(base64.urlsafe_b64decode(txt.encode("ascii")))
+
+
+def w8b64enc(txt):
+    """encodes wtf8 to base64(filesystem-bytes)"""
+    return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii")
+
+
 if PY2 and WINDOWS:
     # moonrunes become \x3f with bytestrings,
     # losing mojibake support is worth
@@ -632,10 +682,10 @@ def sendfile_kern(lower, upper, f, s):
         except Exception as ex:
             # print("sendfile: " + repr(ex))
             n = 0
-        
+
         if n <= 0:
             return upper - ofs
-        
+
         ofs += n
         # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
 
@@ -718,6 +768,22 @@ def py_desc():
     )
 
 
+def align_tab(lines):
+    rows = []
+    ncols = 0
+    for ln in lines:
+        row = [x for x in ln.split(" ") if x]
+        ncols = max(ncols, len(row))
+        rows.append(row)
+
+    lens = [0] * ncols
+    for row in rows:
+        for n, col in enumerate(row):
+            lens[n] = max(lens[n], len(col))
+
+    return ["".join(x.ljust(y + 2) for x, y in zip(row, lens)) for row in rows]
+
+
 class Pebkac(Exception):
     def __init__(self, code, msg=None):
         super(Pebkac, self).__init__(msg or HTTPCODE[code])

copyparty/web/browser.css

@@ -39,15 +39,27 @@ body {
 	margin: 1.3em 0 0 0;
 	font-size: 1.4em;
 }
+#path #entree {
+	margin-left: -.7em;
+}
+#treetab {
+	display: none;
+}
 #files {
 	border-collapse: collapse;
 	margin-top: 2em;
+	z-index: 1;
+	position: relative;
 }
 #files tbody a {
 	display: block;
 	padding: .3em 0;
 }
-a {
+#files[ts] tbody div a {
+	color: #f5a;
+}
+a,
+#files[ts] tbody div a:last-child {
 	color: #fc5;
 	padding: .2em;
 	text-decoration: none;
@@ -142,10 +154,12 @@ a {
 #srv_info span {
 	color: #fff;
 }
-a.play {
+#files tbody a.play {
 	color: #e70;
+	padding: .2em;
+	margin: -.2em;
 }
-a.play.act {
+#files tbody a.play.act {
 	color: #af0;
 }
 #blocked {
@@ -156,7 +170,7 @@ a.play.act {
 	height: 100%;
 	background: #333;
 	font-size: 2.5em;
-	z-index:99;
+	z-index: 99;
 }
 #blk_play,
 #blk_abrt {
@@ -190,6 +204,7 @@ a.play.act {
 	bottom: -6em;
 	height: 6em;
 	width: 100%;
+	z-index: 3;
 	transition: bottom 0.15s;
 }
 #widget.open {
@@ -214,6 +229,9 @@ a.play.act {
 	75% {cursor: url(/.cpr/dd/5.png), pointer}
 	85% {cursor: url(/.cpr/dd/1.png), pointer}
 }
+@keyframes spin {
+	100% {transform: rotate(360deg)}
+}
 #wtoggle {
 	position: absolute;
 	top: -1.2em;
@@ -273,3 +291,208 @@ a.play.act {
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
+
+
+
+.opview {
+	display: none;
+}
+.opview.act {
+	display: block;
+}
+#ops a {
+	color: #fc5;
+	font-size: 1.5em;
+	padding: .25em .3em;
+	margin: 0;
+	outline: none;
+}
+#ops a.act {
+	background: #281838;
+	border-radius: 0 0 .2em .2em;
+	border-bottom: .3em solid #d90;
+	box-shadow: 0 -.15em .2em #000 inset;
+	padding-bottom: .3em;
+}
+#ops i {
+	font-size: 1.5em;
+}
+#ops i:before {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #01a7e1;
+	position: relative;
+}
+#ops i:after {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #ff3f1a;
+	margin-left: -.35em;
+	font-size: 1.05em;
+}
+#ops,
+.opbox {
+	border: 1px solid #3a3a3a;
+	box-shadow: 0 0 1em #222 inset;
+}
+#ops {
+	background: #333;
+	margin: 1.7em 1.5em 0 1.5em;
+	padding: .3em .6em;
+	border-radius: .3em;
+	border-width: .15em 0;
+}
+.opbox {
+	background: #2d2d2d;
+	margin: 1.5em 0 0 0;
+	padding: .5em;
+	border-radius: 0 1em 1em 0;
+	border-width: .15em .3em .3em 0;
+	max-width: 40em;
+}
+.opbox input {
+	margin: .5em;
+}
+.opview input[type=text] {
+	color: #fff;
+	background: #383838;
+	border: none;
+	box-shadow: 0 0 .3em #222;
+	border-bottom: 1px solid #fc5;
+	border-radius: .2em;
+	padding: .2em .3em;
+}
+input[type="checkbox"]+label {
+	color: #f5a;
+}
+input[type="checkbox"]:checked+label {
+	color: #fc5;
+}
+
+
+
+#op_search table {
+	border: 1px solid #3a3a3a;
+	box-shadow: 0 0 1em #222 inset;
+	background: #2d2d2d;
+	border-radius: .4em;
+	margin: 1.4em;
+	margin-bottom: 0;
+	padding: 0 .5em .5em 0;
+}
+#srch_form td {
+	padding: .6em .6em;
+}
+#op_search input {
+	margin: 0;
+}
+#srch_q {
+	white-space: pre;
+}
+#files td div span {
+	color: #fff;
+	padding: 0 .4em;
+	font-weight: bold;
+	font-style: italic;
+}
+#files td div a:hover {
+	background: #444;
+	color: #fff;
+}
+#files td div a {
+	display: table-cell;
+	white-space: nowrap;
+}
+#files td div a:last-child {
+	width: 100%;
+}
+#files td div {
+	display: table;
+	border-collapse: collapse;
+	width: 100%;
+}
+#files td div a:last-child {
+	width: 100%;
+}
+#tree,
+#treefiles {
+	vertical-align: top;
+}
+#tree {
+	padding-top: 2em;
+}
+#detree {
+	padding: .3em .5em;
+	font-size: 1.5em;
+	display: inline-block;
+	min-width: 12em;
+	width: 100%;
+}
+#treefiles #files tbody {
+	border-radius: 0 .7em 0 .7em;
+}
+#treefiles #files thead th:nth-child(1) {
+	border-radius: .7em 0 0 0;
+}
+#tree ul,
+#tree li {
+	padding: 0;
+	margin: 0;
+}
+#tree ul {
+	border-left: .2em solid #444;
+}
+#tree li {
+	margin-left: 1em;
+	list-style: none;
+	white-space: nowrap;
+}
+#tree a.hl {
+	color: #400;
+	background: #fc4;
+	border-radius: .3em;
+	text-shadow: none;
+}
+#tree a {
+	display: inline-block;
+}
+#tree a+a {
+	width: calc(100% - 2em);
+	background: #333;
+}
+#tree a+a:hover {
+	background: #222;
+	color: #fff;
+}
+#treeul {
+	position: relative;
+	overflow: hidden;
+	left: -1.7em;
+}
+#treeul:hover {
+	z-index: 2;
+	overflow: visible;
+}
+#treeul:hover a+a {
+	width: auto;
+	min-width: calc(100% - 2em);
+}
+#treeul a:first-child {
+	font-family: monospace, monospace;
+}
+#treefiles {
+	opacity: 1;
+	transition: opacity 0.2s ease-in-out;
+}
+#tree:hover+#treefiles {
+	opacity: .8;
+}
+.dumb_loader_thing {
+	display: inline-block;
+	margin: 1em .3em 1em 1em;
+	padding: 0 1.2em 0 0;
+	font-size: 4em;
+	animation: spin 1s linear infinite;
+	position: absolute;
+	z-index: 9;
+}

copyparty/web/browser.html

@@ -7,27 +7,51 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
-    {%- if can_upload %}
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
-    {%- endif %}
 </head>
 
 <body>
-    {%- if can_upload %}
+    <div id="ops">
+        <a href="#" data-dest="">---</a>
+        <a href="#" data-perm="read" data-dest="search">🔎</a>
+        {%- if have_up2k_idx %}
+        <a href="#" data-dest="up2k">🚀</a>
+        {%- else %}
+        <a href="#" data-perm="write" data-dest="up2k">🚀</a>
+        {%- endif %}
+        <a href="#" data-perm="write" data-dest="bup">🎈</a>
+        <a href="#" data-perm="write" data-dest="mkdir">📂</a>
+        <a href="#" data-perm="write" data-dest="new_md">📝</a>
+        <a href="#" data-perm="write" data-dest="msg">📟</a>
+    </div>
+
+    <div id="op_search" class="opview">
+        <table id="srch_form"></table>
+        <div id="srch_q"></div>
+    </div>
     {%- include 'upload.html' %}
-    {%- endif %}
     
     <h1 id="path">
+        <a href="#" id="entree">🌲</a>
         {%- for n in vpnodes %}
         <a href="/{{ n[0] }}">{{ n[1] }}</a>
         {%- endfor %}
     </h1>
     
-    {%- if can_read %}
     {%- if prologue %}
     <div id="pro" class="logue">{{ prologue }}</div>
     {%- endif %}
 
+    <table id="treetab">
+        <tr>
+            <td id="tree">
+                <a href="#" id="detree">🍞...</a>
+                <ul id="treeul"></ul>
+            </td>
+            <td id="treefiles"></td>
+        </tr>
+    </table>
+
     <table id="files">
         <thead>
             <tr>
@@ -41,7 +65,7 @@
         <tbody>
 
 {%- for f in files %}
-<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td><td>{{ f[5] }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
         </tbody>
@@ -50,7 +74,6 @@
     {%- if epilogue %}
     <div id="epi" class="logue">{{ epilogue }}</div>
     {%- endif %}
-    {%- endif %}
 
     <h2><a href="?h">control-panel</a></h2>
 
@@ -69,14 +92,11 @@
     </div>
     
     <script src="/.cpr/util.js{{ ts }}"></script>
-
-    {%- if can_read %}
     <script src="/.cpr/browser.js{{ ts }}"></script>
-    {%- endif %}
-    
-    {%- if can_upload %}
     <script src="/.cpr/up2k.js{{ ts }}"></script>
-    {%- endif %}
+    <script>
+        apply_perms({{ perms }});
+    </script>
 </body>
 
 </html>

copyparty/web/browser.js

@@ -8,6 +8,8 @@ function dbg(msg) {
 
 function ev(e) {
 	e = e || window.event;
+	if (!e)
+		return;
 
 	if (e.preventDefault)
 		e.preventDefault()
@@ -23,7 +25,7 @@ makeSortable(ebi('files'));
 
 
 // extract songs + add play column
-var mp = (function () {
+function init_mp() {
 	var tracks = [];
 	var ret = {
 		'au': null,
@@ -37,7 +39,8 @@ var mp = (function () {
 	var trs = ebi('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr');
 	for (var a = 0, aa = trs.length; a < aa; a++) {
 		var tds = trs[a].getElementsByTagName('td');
-		var link = tds[1].getElementsByTagName('a')[0];
+		var link = tds[1].getElementsByTagName('a');
+		link = link[link.length - 1];
 		var url = link.getAttribute('href');
 
 		var m = re_audio.exec(url);
@@ -71,7 +74,8 @@ var mp = (function () {
 	};
 
 	return ret;
-})();
+}
+var mp = init_mp();
 
 
 // toggle player widget
@@ -466,7 +470,13 @@ function play(tid, call_depth) {
 
 		var o = ebi(oid);
 		o.setAttribute('id', 'thx_js');
-		location.hash = oid;
+		if (window.history && history.replaceState) {
+			var nurl = (document.location + '').split('#')[0] + '#' + oid;
+			history.replaceState(ebi('files').tBodies[0].innerHTML, nurl, nurl);
+		}
+		else {
+			document.location.hash = oid;
+		}
 		o.setAttribute('id', oid);
 
 		pbar.drawbuf();
@@ -561,3 +571,470 @@ function autoplay_blocked() {
 
 
 //widget.open();
+
+
+// search
+(function () {
+	var sconf = [
+		["size",
+			["szl", "sz_min", "minimum MiB", ""],
+			["szu", "sz_max", "maximum MiB", ""]
+		],
+		["date",
+			["dtl", "dt_min", "min. iso8601", ""],
+			["dtu", "dt_max", "max. iso8601", ""]
+		],
+		["path",
+			["path", "path", "path contains &nbsp; (space-separated)", "46"]
+		],
+		["name",
+			["name", "name", "name contains &nbsp; (negate with -nope)", "46"]
+		]
+	];
+	var html = [];
+	var orig_html = null;
+	for (var a = 0; a < sconf.length; a++) {
+		html.push('<tr><td><br />' + sconf[a][0] + '</td>');
+		for (var b = 1; b < 3; b++) {
+			var hn = "srch_" + sconf[a][b][0];
+			var csp = (sconf[a].length == 2) ? 2 : 1;
+			html.push(
+				'<td colspan="' + csp + '"><input id="' + hn + 'c" type="checkbox">\n' +
+				'<label for="' + hn + 'c">' + sconf[a][b][2] + '</label>\n' +
+				'<br /><input id="' + hn + 'v" type="text" size="' + sconf[a][b][3] +
+				'" name="' + sconf[a][b][1] + '" /></td>');
+			if (csp == 2)
+				break;
+		}
+		html.push('</tr>');
+	}
+	ebi('srch_form').innerHTML = html.join('\n');
+
+	var o = document.querySelectorAll('#op_search input[type="text"]');
+	for (var a = 0; a < o.length; a++) {
+		o[a].oninput = ev_search_input;
+	}
+
+	var search_timeout;
+
+	function ev_search_input() {
+		var v = this.value;
+		var chk = ebi(this.getAttribute('id').slice(0, -1) + 'c');
+		chk.checked = ((v + '').length > 0);
+		clearTimeout(search_timeout);
+		search_timeout = setTimeout(do_search, 100);
+	}
+
+	function do_search() {
+		clearTimeout(search_timeout);
+		var params = {};
+		var o = document.querySelectorAll('#op_search input[type="text"]');
+		for (var a = 0; a < o.length; a++) {
+			var chk = ebi(o[a].getAttribute('id').slice(0, -1) + 'c');
+			if (!chk.checked)
+				continue;
+
+			params[o[a].getAttribute('name')] = o[a].value;
+		}
+		// ebi('srch_q').textContent = JSON.stringify(params, null, 4);
+		var xhr = new XMLHttpRequest();
+		xhr.open('POST', '/?srch', true);
+		xhr.onreadystatechange = xhr_search_results;
+		xhr.ts = new Date().getTime();
+		xhr.send(JSON.stringify(params));
+	}
+
+	function xhr_search_results() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			alert('ah fug\n' + this.status + ": " + this.responseText);
+			return;
+		}
+
+		var ofiles = ebi('files');
+		if (ofiles.getAttribute('ts') > this.ts)
+			return;
+
+		ebi('path').style.display = 'none';
+		ebi('tree').style.display = 'none';
+
+		var html = ['<tr><td>-</td><td colspan="4"><a href="#" id="unsearch">close search results</a></td></tr>'];
+		var res = JSON.parse(this.responseText);
+		for (var a = 0; a < res.length; a++) {
+			var r = res[a],
+				ts = parseInt(r.ts),
+				sz = esc(r.sz + ''),
+				rp = esc(r.rp + ''),
+				ext = rp.lastIndexOf('.') > 0 ? rp.split('.').slice(-1)[0] : '%',
+				links = linksplit(rp);
+
+			if (ext.length > 8)
+				ext = '%';
+
+			links = links.join('');
+			html.push('<tr><td>-</td><td><div>' + links + '</div></td><td>' + sz +
+				'</td><td>' + ext + '</td><td>' + unix2iso(ts) + '</td></tr>');
+		}
+
+		if (!orig_html)
+			orig_html = ebi('files').tBodies[0].innerHTML;
+
+		ofiles.tBodies[0].innerHTML = html.join('\n');
+		ofiles.setAttribute("ts", this.ts);
+		reload_browser();
+
+		ebi('unsearch').onclick = unsearch;
+	}
+
+	function unsearch(e) {
+		ev(e);
+		ebi('path').style.display = 'inline-block';
+		ebi('tree').style.display = 'block';
+		ebi('files').tBodies[0].innerHTML = orig_html;
+		orig_html = null;
+		reload_browser();
+	}
+})();
+
+
+// tree
+(function () {
+	var treedata = null;
+
+	function entree(e) {
+		ev(e);
+		ebi('path').style.display = 'none';
+
+		var treetab = ebi('treetab');
+		var treefiles = ebi('treefiles');
+
+		treetab.style.display = 'table';
+
+		var pro = ebi('pro');
+		if (pro)
+			treefiles.appendChild(pro);
+
+		treefiles.appendChild(ebi('files'));
+
+		var epi = ebi('epi');
+		if (epi)
+			treefiles.appendChild(epi);
+
+		localStorage.setItem('entreed', 'tree');
+		get_tree("", get_vpath());
+	}
+
+	function get_tree(top, dst) {
+		var xhr = new XMLHttpRequest();
+		xhr.top = top;
+		xhr.dst = dst;
+		xhr.open('GET', dst + '?tree=' + top, true);
+		xhr.onreadystatechange = recvtree;
+		xhr.send();
+		enspin('#tree');
+	}
+
+	function recvtree() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			alert('ah fug\n' + this.status + ": " + this.responseText);
+			return;
+		}
+
+		var top = this.top == '.' ? this.dst : this.top,
+			name = top.split('/').slice(-2)[0],
+			rtop = top.replace(/^\/+/, "");
+
+		try {
+			var res = JSON.parse(this.responseText);
+		}
+		catch (ex) {
+			return;
+		}
+		var html = parsetree(res, rtop);
+		if (!this.top) {
+			html = '<li><a href="#">-</a><a href="/">[root]</a>\n<ul>' + html;
+			if (!ebi('treeul').getElementsByTagName('li').length)
+				ebi('treeul').innerHTML = html + '</ul></li>';
+		}
+		else {
+			html = '<a href="#">-</a><a href="' +
+				esc(top) + '">' + esc(name) +
+				"</a>\n<ul>\n" + html + "</ul>";
+
+			var links = document.querySelectorAll('#tree a+a');
+			for (var a = 0, aa = links.length; a < aa; a++) {
+				if (links[a].getAttribute('href') == top) {
+					var o = links[a].parentNode;
+					if (!o.getElementsByTagName('li').length)
+						o.innerHTML = html;
+					//else
+					//	links[a].previousSibling.textContent = '-';
+				}
+			}
+		}
+		document.querySelector('#treeul>li>a+a').textContent = '[root]';
+		despin('#tree');
+		reload_tree();
+
+		var q = '#tree';
+		var nq = 0;
+		while (true) {
+			nq++;
+			q += '>ul>li';
+			if (!document.querySelector(q))
+				break;
+		}
+		ebi('treeul').style.width = (24 + nq) + 'em';
+	}
+
+	function reload_tree() {
+		var cdir = get_vpath();
+		var links = document.querySelectorAll('#tree a+a');
+		for (var a = 0, aa = links.length; a < aa; a++) {
+			var href = links[a].getAttribute('href');
+			links[a].setAttribute('class', href == cdir ? 'hl' : '');
+			links[a].onclick = treego;
+		}
+		links = document.querySelectorAll('#tree li>a:first-child');
+		for (var a = 0, aa = links.length; a < aa; a++) {
+			links[a].setAttribute('dst', links[a].nextSibling.getAttribute('href'));
+			links[a].onclick = treegrow;
+		}
+	}
+
+	function treego(e) {
+		ev(e);
+		if (this.getAttribute('class') == 'hl' &&
+			this.previousSibling.textContent == '-') {
+			treegrow.call(this.previousSibling, e);
+			return;
+		}
+		var xhr = new XMLHttpRequest();
+		xhr.top = this.getAttribute('href');
+		xhr.open('GET', xhr.top + '?ls', true);
+		xhr.onreadystatechange = recvls;
+		xhr.send();
+		get_tree('.', xhr.top);
+		enspin('#files');
+	}
+
+	function treegrow(e) {
+		ev(e);
+		if (this.textContent == '-') {
+			while (this.nextSibling.nextSibling) {
+				var rm = this.nextSibling.nextSibling;
+				rm.parentNode.removeChild(rm);
+			}
+			this.textContent = '+';
+			return;
+		}
+		var dst = this.getAttribute('dst');
+		get_tree('.', dst);
+	}
+
+	function recvls() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			alert('ah fug\n' + this.status + ": " + this.responseText);
+			return;
+		}
+
+		try {
+			var res = JSON.parse(this.responseText);
+		}
+		catch (ex) {
+			window.location = this.top;
+			return;
+		}
+
+		ebi('srv_info').innerHTML = '<span>' + res.srvinf + '</span>';
+		var nodes = res.dirs.concat(res.files);
+		var top = this.top;
+		var html = [];
+		for (var a = 0; a < nodes.length; a++) {
+			var r = nodes[a],
+				ln = '<tr><td>' + r.lead + '</td><td><a href="' +
+					top + r.href + '">' + esc(decodeURIComponent(r.href)) + '</a>';
+
+			ln = [ln, r.sz, r.ext, unix2iso(r.ts)].join('</td><td>');
+			html.push(ln + '</td></tr>');
+		}
+		html = html.join('\n');
+		ebi('files').tBodies[0].innerHTML = html;
+		history.pushState(html, this.top, this.top);
+		apply_perms(res.perms);
+		despin('#files');
+
+		var o = ebi('pro');
+		if (o) o.parentNode.removeChild(o);
+
+		o = ebi('epi');
+		if (o) o.parentNode.removeChild(o);
+
+		reload_tree();
+		reload_browser();
+	}
+
+	function parsetree(res, top) {
+		var ret = '';
+		for (var a = 0; a < res.a.length; a++) {
+			if (res.a[a] !== '')
+				res['k' + res.a[a]] = 0;
+		}
+		delete res['a'];
+		var keys = Object.keys(res);
+		keys.sort();
+		for (var a = 0; a < keys.length; a++) {
+			var kk = keys[a],
+				k = kk.slice(1),
+				url = '/' + (top ? top + k : k) + '/',
+				ek = esc(k),
+				sym = res[kk] ? '-' : '+',
+				link = '<a href="#">' + sym + '</a><a href="' +
+					esc(url) + '">' + ek + '</a>';
+
+			if (res[kk]) {
+				var subtree = parsetree(res[kk], url.slice(1));
+				ret += '<li>' + link + '\n<ul>\n' + subtree + '</ul></li>\n';
+			}
+			else {
+				ret += '<li>' + link + '</li>\n';
+			}
+		}
+		return ret;
+	}
+
+	function detree(e) {
+		ev(e);
+		var treetab = ebi('treetab');
+
+		var pro = ebi('pro');
+		if (pro)
+			treetab.parentNode.insertBefore(pro, treetab);
+
+		treetab.parentNode.insertBefore(ebi('files'), treetab.nextSibling);
+
+		var epi = ebi('epi');
+		if (epi)
+			treetab.parentNode.insertBefore(epi, ebi('files').nextSibling);
+
+		ebi('path').style.display = 'inline-block';
+		treetab.style.display = 'none';
+
+		localStorage.setItem('entreed', 'na');
+	}
+
+	ebi('entree').onclick = entree;
+	ebi('detree').onclick = detree;
+	if (window.localStorage && localStorage.getItem('entreed') == 'tree')
+		entree();
+
+	window.onpopstate = function (e) {
+		console.log(e.url + ' ,, ' + ((e.state + '').slice(0, 64)));
+		if (e.state) {
+			ebi('files').tBodies[0].innerHTML = e.state;
+			reload_tree();
+			reload_browser();
+		}
+	};
+
+	if (window.history && history.pushState) {
+		var u = get_vpath();
+		history.replaceState(ebi('files').tBodies[0].innerHTML, u, u);
+	}
+})();
+
+
+function enspin(sel) {
+	despin(sel);
+	var d = document.createElement('div');
+	d.setAttribute('class', 'dumb_loader_thing');
+	d.innerHTML = '🌲';
+	var tgt = document.querySelector(sel);
+	tgt.insertBefore(d, tgt.childNodes[0]);
+}
+
+
+function despin(sel) {
+	var o = document.querySelectorAll(sel + '>.dumb_loader_thing');
+	for (var a = o.length - 1; a >= 0; a--)
+		o[a].parentNode.removeChild(o[a]);
+}
+
+
+function apply_perms(perms) {
+	perms = perms || [];
+
+	var o = document.querySelectorAll('#ops>a[data-perm]');
+	for (var a = 0; a < o.length; a++)
+		o[a].style.display = 'none';
+
+	for (var a = 0; a < perms.length; a++) {
+		o = document.querySelectorAll('#ops>a[data-perm="' + perms[a] + '"]');
+		for (var b = 0; b < o.length; b++)
+			o[b].style.display = 'inline';
+	}
+
+	var act = document.querySelector('#ops>a.act');
+	if (act) {
+		var areq = act.getAttribute('data-perm');
+		if (areq && !has(perms, areq))
+			goto();
+	}
+
+	document.body.setAttribute('perms', perms.join(' '));
+
+	var have_write = has(perms, "write");
+	var tds = document.querySelectorAll('#u2conf td');
+	for (var a = 0; a < tds.length; a++) {
+		tds[a].style.display =
+			(have_write || tds[a].getAttribute('data-perm') == 'read') ?
+				'table-cell' : 'none';
+	}
+	if (!have_write && up2k)
+		up2k.set_fsearch();
+}
+
+
+function reload_browser(not_mp) {
+	makeSortable(ebi('files'));
+
+	var parts = get_vpath().split('/');
+	var rm = document.querySelectorAll('#path>a+a+a');
+	for (a = rm.length - 1; a >= 0; a--)
+		rm[a].parentNode.removeChild(rm[a]);
+
+	var link = '/';
+	for (var a = 1; a < parts.length - 1; a++) {
+		link += parts[a] + '/';
+		var o = document.createElement('a');
+		o.setAttribute('href', link);
+		o.innerHTML = parts[a];
+		ebi('path').appendChild(o);
+	}
+
+	var oo = document.querySelectorAll('#files>tbody>tr>td:nth-child(3)');
+	for (var a = 0, aa = oo.length; a < aa; a++) {
+		var sz = oo[a].textContent.replace(/ /g, ""),
+			hsz = sz.replace(/\B(?=(\d{3})+(?!\d))/g, " ");
+
+		oo[a].textContent = hsz;
+	}
+
+	if (!not_mp) {
+		if (mp && mp.au) {
+			mp.au.pause();
+			mp.au = null;
+		}
+		widget.close();
+		mp = init_mp();
+	}
+}
+reload_browser(true);

copyparty/web/md2.css

@@ -124,5 +124,3 @@ html.dark #toast {
     transition: opacity 0.2s ease-in-out;
     opacity: 1;
 }
-
-# mt {opacity: .5;top:1px}

copyparty/web/up2k.js

@@ -3,51 +3,6 @@
 window.onerror = vis_exh;
 
 
-(function () {
-    var ops = document.querySelectorAll('#ops>a');
-    for (var a = 0; a < ops.length; a++) {
-        ops[a].onclick = opclick;
-    }
-})();
-
-
-function opclick(ev) {
-    if (ev) //ie
-        ev.preventDefault();
-
-    var dest = this.getAttribute('data-dest');
-    goto(dest);
-
-    // writing a blank value makes ie8 segfault w
-    if (window.localStorage)
-        localStorage.setItem('opmode', dest || '.');
-
-    var input = document.querySelector('.opview.act input:not([type="hidden"])')
-    if (input)
-        input.focus();
-}
-
-
-function goto(dest) {
-    var obj = document.querySelectorAll('.opview.act');
-    for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
-
-    obj = document.querySelectorAll('#ops>a');
-    for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
-
-    if (dest) {
-        ebi('op_' + dest).classList.add('act');
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
-
-        var fn = window['goto_' + dest];
-        if (fn)
-            fn();
-    }
-}
-
-
 function goto_up2k() {
     if (up2k === false)
         return goto('bup');
@@ -59,17 +14,6 @@ function goto_up2k() {
 }
 
 
-(function () {
-    goto();
-    if (window.localStorage) {
-        var op = localStorage.getItem('opmode');
-        if (op !== null && op !== '.')
-            goto(op);
-    }
-    ebi('ops').style.display = 'block';
-})();
-
-
 // chrome requires https to use crypto.subtle,
 // usually it's undefined but some chromes throw on invoke
 var up2k = null;
@@ -89,6 +33,104 @@ catch (ex) {
 }
 
 
+function up2k_flagbus() {
+    var flag = {
+        "id": Math.floor(Math.random() * 1024 * 1024 * 1023 * 2),
+        "ch": new BroadcastChannel("up2k_flagbus"),
+        "ours": false,
+        "owner": null,
+        "wants": null,
+        "act": false,
+        "last_tx": ["x", null]
+    };
+    var dbg = function (who, msg) {
+        console.log('flagbus(' + flag.id + '): [' + who + '] ' + msg);
+    };
+    flag.ch.onmessage = function (ev) {
+        var who = ev.data[0],
+            what = ev.data[1];
+
+        if (who == flag.id) {
+            dbg(who, 'hi me (??)');
+            return;
+        }
+        flag.act = new Date().getTime();
+        if (what == "want") {
+            // lowest id wins, don't care if that's us
+            if (who < flag.id) {
+                dbg(who, 'wants (ack)');
+                flag.wants = [who, flag.act];
+            }
+            else {
+                dbg(who, 'wants (ign)');
+            }
+        }
+        else if (what == "have") {
+            dbg(who, 'have');
+            flag.owner = [who, flag.act];
+        }
+        else if (what == "give") {
+            if (flag.owner && flag.owner[0] == who) {
+                flag.owner = null;
+                dbg(who, 'give (ok)');
+            }
+            else {
+                dbg(who, 'give, INVALID, ' + flag.owner);
+            }
+        }
+        else if (what == "hi") {
+            dbg(who, 'hi');
+            flag.ch.postMessage([flag.id, "hey"]);
+        }
+        else {
+            dbg('?', ev.data);
+        }
+    };
+    var tx = function (now, msg) {
+        var td = now - flag.last_tx[1];
+        if (td > 500 || flag.last_tx[0] != msg) {
+            dbg('*', 'tx ' + msg);
+            flag.ch.postMessage([flag.id, msg]);
+            flag.last_tx = [msg, now];
+        }
+    };
+    var do_take = function (now) {
+        //dbg('*', 'do_take');
+        tx(now, "have");
+        flag.owner = [flag.id, now];
+        flag.ours = true;
+    };
+    var do_want = function (now) {
+        //dbg('*', 'do_want');
+        tx(now, "want");
+    };
+    flag.take = function (now) {
+        if (flag.ours) {
+            do_take(now);
+            return;
+        }
+        if (flag.owner && now - flag.owner[1] > 5000) {
+            flag.owner = null;
+        }
+        if (flag.wants && now - flag.wants[1] > 5000) {
+            flag.wants = null;
+        }
+        if (!flag.owner && !flag.wants) {
+            do_take(now);
+            return;
+        }
+        do_want(now);
+    };
+    flag.give = function () {
+        dbg('#', 'put give');
+        flag.ch.postMessage([flag.id, "give"]);
+        flag.owner = null;
+        flag.ours = false;
+    };
+    flag.ch.postMessage([flag.id, 'hi']);
+    return flag;
+}
+
 function up2k_init(have_crypto) {
     //have_crypto = false;
     var need_filereader_cache = undefined;
@@ -109,10 +151,6 @@ function up2k_init(have_crypto) {
         ebi('u2notbtn').innerHTML = '';
     }
 
-    var post_url = ebi('op_bup').getElementsByTagName('form')[0].getAttribute('action');
-    if (post_url && post_url.charAt(post_url.length - 1) !== '/')
-        post_url += '/';
-
     var shame = 'your browser <a href="https://www.chromium.org/blink/webcrypto">disables sha512</a> unless you <a href="' + (window.location + '').replace(':', 's:') + '">use https</a>'
     var is_https = (window.location + '').indexOf('https:') === 0;
     if (is_https)
@@ -157,7 +195,7 @@ function up2k_init(have_crypto) {
     // handle user intent to use the basic uploader instead
     ebi('u2nope').onclick = function (e) {
         e.preventDefault();
-        setmsg('');
+        setmsg();
         goto('bup');
     };
 
@@ -181,13 +219,17 @@ function up2k_init(have_crypto) {
     }
 
     function bcfg_get(name, defval) {
+        var o = ebi(name);
+        if (!o)
+            return defval;
+
         var val = localStorage.getItem(name);
         if (val === null)
             val = defval;
         else
             val = (val == '1');
 
-        ebi(name).checked = val;
+        o.checked = val;
         return val;
     }
 
@@ -195,13 +237,18 @@ function up2k_init(have_crypto) {
         localStorage.setItem(
             name, val ? '1' : '0');
 
-        ebi(name).checked = val;
+        var o = ebi(name);
+        if (o)
+            o.checked = val;
+
         return val;
     }
 
     var parallel_uploads = cfg_get('nthread');
     var multitask = bcfg_get('multitask', true);
     var ask_up = bcfg_get('ask_up', true);
+    var flag_en = bcfg_get('flag_en', false);
+    var fsearch = bcfg_get('fsearch', false);
 
     var col_hashing = '#00bbff';
     var col_hashed = '#004466';
@@ -219,6 +266,10 @@ function up2k_init(have_crypto) {
             "hash": [],
             "handshake": [],
             "upload": []
+        },
+        "bytes": {
+            "hashed": 0,
+            "uploaded": 0
         }
     };
 
@@ -229,6 +280,10 @@ function up2k_init(have_crypto) {
     if (!bobslice || !window.FileReader || !window.FileList)
         return un2k("this is the basic uploader; up2k needs at least<br />chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1");
 
+    var flag = false;
+    apply_flag_cfg();
+    apply_fsearch_cfg();
+
     function nav() {
         ebi('file' + fdom_ctr).click();
     }
@@ -298,7 +353,7 @@ function up2k_init(have_crypto) {
         for (var a = 0; a < good_files.length; a++)
             msg.push(good_files[a].name);
 
-        if (ask_up && !confirm(msg.join('\n')))
+        if (ask_up && !fsearch && !confirm(msg.join('\n')))
             return;
 
         for (var a = 0; a < good_files.length; a++) {
@@ -312,6 +367,8 @@ function up2k_init(have_crypto) {
                 "name": fobj.name,
                 "size": fobj.size,
                 "lmod": lmod / 1000,
+                "purl": get_vpath(),
+                "done": false,
                 "hash": []
             };
 
@@ -326,7 +383,7 @@ function up2k_init(have_crypto) {
 
             var tr = document.createElement('tr');
             tr.innerHTML = '<td id="f{0}n"></td><td id="f{0}t">hashing</td><td id="f{0}p" class="prog"></td>'.format(st.files.length);
-            tr.getElementsByTagName('td')[0].textContent = entry.name;
+            tr.getElementsByTagName('td')[0].innerHTML = fsearch ? entry.name : linksplit(esc(entry.purl + entry.name)).join(' ');
             ebi('u2tab').appendChild(tr);
 
             st.files.push(entry);
@@ -344,6 +401,19 @@ function up2k_init(have_crypto) {
     }
     more_one_file();
 
+    function u2cleanup(e) {
+        ev(e);
+        for (var a = 0; a < st.files.length; a++) {
+            var t = st.files[a];
+            if (t.done && t.name) {
+                var tr = ebi('f{0}p'.format(t.n)).parentNode;
+                tr.parentNode.removeChild(tr);
+                t.name = undefined;
+            }
+        }
+    }
+    ebi('u2cleanup').onclick = u2cleanup;
+
     /////
     ////
     ///   actuator
@@ -357,14 +427,18 @@ function up2k_init(have_crypto) {
     }
 
     function hashing_permitted() {
-        var lim = multitask ? 1 : 0;
+        if (multitask) {
-        return handshakes_permitted() && lim >=
+            var ahead = st.bytes.hashed - st.bytes.uploaded;
+            return ahead < 1024 * 1024 * 128;
+        }
+        return handshakes_permitted() && 0 ==
             st.todo.handshake.length +
             st.busy.handshake.length;
     }
 
     var tasker = (function () {
         var mutex = false;
+        var was_busy = false;
 
         function taskerd() {
             if (mutex)
@@ -372,8 +446,63 @@ function up2k_init(have_crypto) {
 
             mutex = true;
             while (true) {
+                if (false) {
+                    ebi('srv_info').innerHTML =
+                        new Date().getTime() + ", " +
+                        st.todo.hash.length + ", " +
+                        st.todo.handshake.length + ", " +
+                        st.todo.upload.length + ", " +
+                        st.busy.hash.length + ", " +
+                        st.busy.handshake.length + ", " +
+                        st.busy.upload.length;
+                }
+
+                var is_busy = 0 !=
+                    st.todo.hash.length +
+                    st.todo.handshake.length +
+                    st.todo.upload.length +
+                    st.busy.hash.length +
+                    st.busy.handshake.length +
+                    st.busy.upload.length;
+
+                if (was_busy != is_busy) {
+                    was_busy = is_busy;
+
+                    if (is_busy)
+                        window.addEventListener("beforeunload", warn_uploader_busy);
+                    else
+                        window.removeEventListener("beforeunload", warn_uploader_busy);
+                }
+
+                if (flag) {
+                    if (is_busy) {
+                        var now = new Date().getTime();
+                        flag.take(now);
+                        if (!flag.ours) {
+                            setTimeout(taskerd, 100);
+                            mutex = false;
+                            return;
+                        }
+                    }
+                    else if (flag.ours) {
+                        flag.give();
+                    }
+                }
+
                 var mou_ikkai = false;
 
+                if (st.todo.handshake.length > 0 &&
+                    st.busy.handshake.length == 0 && (
+                        st.todo.handshake[0].t3 || (
+                            handshakes_permitted() &&
+                            st.busy.upload.length < parallel_uploads
+                        )
+                    )
+                ) {
+                    exec_handshake();
+                    mou_ikkai = true;
+                }
+
                 if (handshakes_permitted() &&
                     st.todo.handshake.length > 0 &&
                     st.busy.handshake.length == 0 &&
@@ -512,6 +641,8 @@ function up2k_init(have_crypto) {
 
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
+        st.bytes.hashed += t.size;
+        t.bytes_uploaded = 0;
         t.t1 = new Date().getTime();
 
         var nchunk = 0;
@@ -638,10 +769,38 @@ function up2k_init(have_crypto) {
             if (xhr.status == 200) {
                 var response = JSON.parse(xhr.responseText);
 
+                if (!response.name) {
+                    var msg = '';
+                    var smsg = '';
+                    if (!response || !response.length) {
+                        msg = 'not found on server';
+                        smsg = '404';
+                    }
+                    else {
+                        smsg = 'found';
+                        var hit = response[0],
+                            msg = linksplit(hit.rp).join(''),
+                            tr = unix2iso(hit.ts),
+                            tu = unix2iso(t.lmod),
+                            diff = parseInt(t.lmod) - parseInt(hit.ts),
+                            cdiff = (Math.abs(diff) <= 2) ? '3c0' : 'f0b',
+                            sdiff = '<span style="color:#' + cdiff + '">diff ' + diff;
+
+                        msg += '<br /><small>' + tr + ' (srv), ' + tu + ' (You), ' + sdiff + '</span></span>';
+                    }
+                    ebi('f{0}p'.format(t.n)).innerHTML = msg;
+                    ebi('f{0}t'.format(t.n)).innerHTML = smsg;
+                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    st.bytes.uploaded += t.size;
+                    t.done = true;
+                    tasker();
+                    return;
+                }
+
                 if (response.name !== t.name) {
                     // file exists; server renamed us
                     t.name = response.name;
-                    ebi('f{0}n'.format(t.n)).textContent = t.name;
+                    ebi('f{0}n'.format(t.n)).innerHTML = linksplit(esc(t.purl + t.name)).join(' ');
                 }
 
                 t.postlist = [];
@@ -675,11 +834,15 @@ function up2k_init(have_crypto) {
                 st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
 
                 if (done) {
+                    t.done = true;
+                    st.bytes.uploaded += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
                     var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
                     ebi('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2));
                 }
+                else t.t3 = undefined;
+
                 tasker();
             }
             else {
@@ -691,6 +854,11 @@ function up2k_init(have_crypto) {
                     var ofs = err.lastIndexOf(' : ');
                     if (ofs > 0)
                         err = err.slice(0, ofs);
+
+                    ofs = err.indexOf('\n/');
+                    if (ofs !== -1) {
+                        err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2, -1)).join(' ');
+                    }
                 }
                 if (err != "") {
                     ebi('f{0}t'.format(t.n)).innerHTML = "ERROR";
@@ -707,14 +875,19 @@ function up2k_init(have_crypto) {
                     "no further information"));
             }
         };
-        xhr.open('POST', post_url + 'handshake.php', true);
+
-        xhr.responseType = 'text';
+        var req = {
-        xhr.send(JSON.stringify({
             "name": t.name,
             "size": t.size,
             "lmod": t.lmod,
             "hash": t.hash
-        }));
+        };
+        if (fsearch)
+            req.srch = 1;
+
+        xhr.open('POST', t.purl + 'handshake.php', true);
+        xhr.responseType = 'text';
+        xhr.send(JSON.stringify(req));
     }
 
     /////
@@ -752,12 +925,14 @@ function up2k_init(have_crypto) {
             xhr.onload = function (xev) {
                 if (xhr.status == 200) {
                     prog(t.n, npart, col_uploaded);
+                    st.bytes.uploaded += cdr - car;
+                    t.bytes_uploaded += cdr - car;
                     st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
                     t.postlist.splice(t.postlist.indexOf(npart), 1);
                     if (t.postlist.length == 0) {
                         t.t3 = new Date().getTime();
                         ebi('f{0}t'.format(t.n)).innerHTML = 'verifying';
-                        st.todo.handshake.push(t);
+                        st.todo.handshake.unshift(t);
                     }
                     tasker();
                 }
@@ -768,7 +943,7 @@ function up2k_init(have_crypto) {
                         (xhr.responseText && xhr.responseText) ||
                         "no further information"));
             };
-            xhr.open('POST', post_url + 'chunkpit.php', true);
+            xhr.open('POST', t.purl + 'chunkpit.php', true);
             //xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart].substr(1) + "x");
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
@@ -804,6 +979,46 @@ function up2k_init(have_crypto) {
     ///   config ui
     //
 
+    function onresize(ev) {
+        var bar = ebi('ops'),
+            wpx = innerWidth,
+            fpx = parseInt(getComputedStyle(bar)['font-size']),
+            wem = wpx * 1.0 / fpx,
+            wide = wem > 54,
+            parent = ebi(wide ? 'u2btn_cw' : 'u2btn_ct'),
+            btn = ebi('u2btn');
+
+        //console.log([wpx, fpx, wem]);
+        if (btn.parentNode !== parent) {
+            parent.appendChild(btn);
+            ebi('u2conf').setAttribute('class', wide ? 'has_btn' : '');
+        }
+    }
+    window.onresize = onresize;
+    onresize();
+
+    function desc_show(ev) {
+        var msg = this.getAttribute('alt');
+        msg = msg.replace(/\$N/g, "<br />");
+        var cdesc = ebi('u2cdesc');
+        cdesc.innerHTML = msg;
+        cdesc.setAttribute('class', 'show');
+    }
+    function desc_hide(ev) {
+        ebi('u2cdesc').setAttribute('class', '');
+    }
+    var o = document.querySelectorAll('#u2conf *[alt]');
+    for (var a = o.length - 1; a >= 0; a--) {
+        o[a].parentNode.getElementsByTagName('input')[0].setAttribute('alt', o[a].getAttribute('alt'));
+    }
+    var o = document.querySelectorAll('#u2conf *[alt]');
+    for (var a = 0; a < o.length; a++) {
+        o[a].onfocus = desc_show;
+        o[a].onblur = desc_hide;
+        o[a].onmouseenter = desc_show;
+        o[a].onmouseleave = desc_hide;
+    }
+
     function bumpthread(dir) {
         try {
             dir.stopPropagation();
@@ -845,6 +1060,51 @@ function up2k_init(have_crypto) {
         bcfg_set('ask_up', ask_up);
     }
 
+    function tgl_fsearch() {
+        fsearch = !fsearch;
+        bcfg_set('fsearch', fsearch);
+        apply_fsearch_cfg();
+    }
+
+    function apply_fsearch_cfg() {
+        try {
+            var fun = fsearch ? 'add' : 'remove';
+            ebi('op_up2k').classList[fun]('srch');
+
+            var ico = fsearch ? '🔎' : '🚀';
+            var desc = fsearch ? 'Search' : 'Upload';
+            ebi('u2bm').innerHTML = ico + ' <sup>' + desc + '</sup>';
+        }
+        catch (ex) { }
+    }
+
+    function set_fsearch() {
+        if (!fsearch)
+            tgl_fsearch();
+    }
+
+    function tgl_flag_en() {
+        flag_en = !flag_en;
+        bcfg_set('flag_en', flag_en);
+        apply_flag_cfg();
+    }
+
+    function apply_flag_cfg() {
+        if (flag_en && !flag) {
+            try {
+                flag = up2k_flagbus();
+            }
+            catch (ex) {
+                console.log("flag error: " + ex.toString());
+                tgl_flag_en();
+            }
+        }
+        else if (!flag_en && flag) {
+            flag.ch.close();
+            flag = false;
+        }
+    }
+
     function nop(ev) {
         ev.preventDefault();
         this.click();
@@ -862,12 +1122,31 @@ function up2k_init(have_crypto) {
     ebi('nthread').addEventListener('input', bumpthread, false);
     ebi('multitask').addEventListener('click', tgl_multitask, false);
     ebi('ask_up').addEventListener('click', tgl_ask_up, false);
+    ebi('flag_en').addEventListener('click', tgl_flag_en, false);
+    var o = ebi('fsearch');
+    if (o)
+        o.addEventListener('click', tgl_fsearch, false);
 
     var nodes = ebi('u2conf').getElementsByTagName('a');
     for (var a = nodes.length - 1; a >= 0; a--)
         nodes[a].addEventListener('touchend', nop, false);
 
+    var perms = document.body.getAttribute('perms');
+    if (perms && perms.indexOf('write') === -1)
+        set_fsearch();
+
     bumpthread({ "target": 1 })
 
-    return { "init_deps": init_deps }
+    return { "init_deps": init_deps, "set_fsearch": set_fsearch }
 }
+
+
+function warn_uploader_busy(e) {
+    e.preventDefault();
+    e.returnValue = '';
+    return "upload in progress, click abort and use the file-tree to navigate instead";
+}
+
+
+if (document.querySelector('#op_up2k.act'))
+    goto_up2k();

copyparty/web/upload.css

@@ -1,92 +1,4 @@
-.opview {
+
-	display: none;
-}
-.opview.act {
-	display: block;
-}
-#ops a {
-	color: #fc5;
-	font-size: 1.5em;
-	padding: 0 .3em;
-	margin: 0;
-	outline: none;
-}
-#ops a.act {
-	text-decoration: underline;
-}
-/*
-#ops a+a:after,
-#ops a:first-child:after {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #01a7e1;
-	margin-left: .3em;
-	position: relative;
-}
-#ops a+a:before {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #ff3f1a;
-	margin-right: .3em;
-	margin-left: -.3em;
-}
-#ops a:last-child:after {
-	content: '';
-}
-#ops a.act:before,
-#ops a.act:after {
-	text-decoration: none !important;
-}
-*/
-#ops i {
-	font-size: 1.5em;
-}
-#ops i:before {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #01a7e1;
-	position: relative;
-}
-#ops i:after {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #ff3f1a;
-	margin-left: -.35em;
-	font-size: 1.05em;
-}
-#ops,
-.opbox {
-	border: 1px solid #3a3a3a;
-	box-shadow: 0 0 1em #222 inset;
-}
-#ops {
-	display: none;
-	background: #333;
-	margin: 1.7em 1.5em 0 1.5em;
-	padding: .3em .6em;
-	border-radius: .3em;
-	border-width: .15em 0;
-}
-.opbox {
-	background: #2d2d2d;
-	margin: 1.5em 0 0 0;
-	padding: .5em;
-	border-radius: 0 1em 1em 0;
-	border-width: .15em .3em .3em 0;
-	max-width: 40em;
-}
-.opbox input {
-	margin: .5em;
-}
-.opbox input[type=text] {
-	color: #fff;
-	background: #383838;
-	border: none;
-	box-shadow: 0 0 .3em #222;
-	border-bottom: 1px solid #fc5;
-	border-radius: .2em;
-	padding: .2em .3em;
-}
 #op_up2k {
 	padding: 0 1em 1em 1em;
 }
@@ -94,6 +6,9 @@
 	position: absolute;
 	top: 0;
 	left: 0;
+	width: 2px;
+	height: 2px;
+	overflow: hidden;
 }
 #u2form input {
 	background: #444;
@@ -104,11 +19,6 @@
 	color: #f87;
 	padding: .5em;
 }
-#u2form {
-	width: 2px;
-	height: 2px;
-	overflow: hidden;
-}
 #u2btn {
 	color: #eee;
 	background: #555;
@@ -117,17 +27,27 @@
 	background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%);
 	filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0);
 	text-decoration: none;
-	line-height: 1.5em;
+	line-height: 1.3em;
 	border: 1px solid #222;
 	border-radius: .4em;
 	text-align: center;
-	font-size: 2em;
+	font-size: 1.5em;
-	margin: 1em auto;
+	margin: .5em auto;
-	padding: 1em 0;
+	padding: .8em 0;
-	width: 12em;
+	width: 16em;
 	cursor: pointer;
 	box-shadow: .4em .4em 0 #111;
 }
+#op_up2k.srch #u2btn {
+	background: linear-gradient(to bottom, #ca3 0%, #fd8 50%, #fc6 51%, #b92 100%);
+	text-shadow: 1px 1px 1px #fc6;
+	color: #333;
+}
+#u2conf #u2btn {
+	margin: -1.5em 0;
+	padding: .8em 0;
+	width: 100%;
+}
 #u2notbtn {
 	display: none;
 	text-align: center;
@@ -142,6 +62,9 @@
 	width: calc(100% - 2em);
 	max-width: 100em;
 }
+#u2form.srch #u2tab {
+	max-width: none;
+}
 #u2tab td {
 	border: 1px solid #ccc;
 	border-width: 0 0px 1px 0;
@@ -153,12 +76,19 @@
 #u2tab td:nth-child(3) {
 	width: 40%;
 }
+#u2form.srch #u2tab td:nth-child(3) {
+	font-family: sans-serif;
+	width: auto;
+}
 #u2tab tr+tr:hover td {
 	background: #222;
 }
 #u2conf {
 	margin: 1em auto;
-	width: 26em;
+	width: 30em;
+}
+#u2conf.has_btn {
+	width: 46em;
 }
 #u2conf * {
 	text-align: center;
@@ -194,16 +124,72 @@
 #u2conf input+a {
 	background: #d80;
 }
+#u2conf label {
+	font-size: 1.6em;
+	width: 2em;
+	height: 1em;
+	padding: .4em 0;
+	display: block;
+	user-select: none;
+	border-radius: .25em;
+}
+#u2conf input[type="checkbox"] {
+	position: relative;
+	opacity: .02;
+	top: 2em;
+}
 #u2conf input[type="checkbox"]+label {
-	color: #f5a;
+	position: relative;
+	background: #603;
+	border-bottom: .2em solid #a16;
+	box-shadow: 0 .1em .3em #a00 inset;
 }
 #u2conf input[type="checkbox"]:checked+label {
-	color: #fc5;
+	background: #6a1;
+	border-bottom: .2em solid #efa;
+	box-shadow: 0 .1em .5em #0c0;
+}
+#u2conf input[type="checkbox"]+label:hover {
+	box-shadow: 0 .1em .3em #fb0;
+	border-color: #fb0;
+}
+#op_up2k.srch #u2conf td:nth-child(1)>*,
+#op_up2k.srch #u2conf td:nth-child(2)>*,
+#op_up2k.srch #u2conf td:nth-child(3)>* {
+	background: #777;
+	border-color: #ccc;
+	box-shadow: none;
+	opacity: .2;
+}
+#u2cdesc {
+	position: absolute;
+	width: 34em;
+	left: calc(50% - 15em);
+	background: #222;
+	border: 0 solid #555;
+	text-align: center;
+	overflow: hidden;
+	margin: 0 -2em;
+	height: 0;
+	padding: 0 1em;
+	opacity: .1;
+    transition: all 0.14s ease-in-out;
+	border-radius: .4em;
+	box-shadow: 0 .2em .5em #222;
+}
+#u2cdesc.show {
+	padding: 1em;
+	height: auto;
+	border-width: .2em 0;
+	opacity: 1;
 }
 #u2foot {
 	color: #fff;
 	font-style: italic;
 }
+#u2footfoot {
+	margin-bottom: -1em;
+}
 .prog {
 	font-family: monospace;
 }
@@ -225,3 +211,13 @@
 	bottom: 0;
 	background: #0a0;
 }
+#u2tab a>span {
+	font-weight: bold;
+	font-style: italic;
+	color: #fff;
+	padding-left: .2em;
+}
+#u2cleanup {
+	float: right;
+	margin-bottom: -.3em;
+}

copyparty/web/upload.html

@@ -1,13 +1,7 @@
-    <div id="ops"><a
-        href="#" data-dest="">---</a><i></i><a
-        href="#" data-dest="up2k">up2k</a><i></i><a
-        href="#" data-dest="bup">bup</a><i></i><a
-        href="#" data-dest="mkdir">mkdir</a><i></i><a
-        href="#" data-dest="new_md">new.md</a></div>
 
     <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
@@ -15,7 +9,7 @@
     </div>
 
     <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="mkdir" />
             <input type="text" name="name" size="30">
             <input type="submit" value="mkdir">
@@ -23,19 +17,45 @@
     </div>
 
     <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="new_md" />
             <input type="text" name="name" size="30">
             <input type="submit" value="create doc">
         </form>
     </div>
 
+    <div id="op_msg" class="opview opbox">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
+            <input type="text" name="msg" size="30">
+            <input type="submit" value="send msg">
+        </form>
+    </div>
+
     <div id="op_up2k" class="opview">
         <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
 
             <table id="u2conf">
                 <tr>
                     <td>parallel uploads</td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="multitask" />
+                        <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="ask_up" />
+                        <label for="ask_up" alt="ask for confirmation befofre upload starts">💭</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="flag_en" />
+                        <label for="flag_en" alt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
+                    </td>
+                {%- if have_up2k_idx %}
+                    <td data-perm="read" rowspan="2">
+                        <input type="checkbox" id="fsearch" />
+                        <label for="fsearch" alt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
+                    </td>
+                {%- endif %}
+                    <td data-perm="read" rowspan="2" id="u2btn_cw"></td>
                 </tr>
                 <tr>
                     <td>
@@ -43,32 +63,29 @@
                         <input class="txtbox" id="nthread" value="2" />
                         <a href="#" id="nthread_add">+</a>
                     </td>
-                    <td rowspan="2" style="padding-left:1.5em">
-                        <input type="checkbox" id="multitask" />
-                        <label for="multitask">hash while<br />uploading</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="ask_up" />
-                        <label for="ask_up">ask for<br />confirmation</label>
-                    </td>
                 </tr>
             </table>
 
+            <div id="u2cdesc"></div>
+
             <div id="u2notbtn"></div>
 
-            <div id="u2btn">
+            <div id="u2btn_ct">
-                drop files here<br />
+                <div id="u2btn">
-                (or click me)
+                    <span id="u2bm"></span><br />
+                    drop files here<br />
+                    (or click me)
+                </div>
             </div>
 
             <table id="u2tab">
                 <tr>
                     <td>filename</td>
                     <td>status</td>
-                    <td>progress</td>
+                    <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
                 </tr>
             </table>
 
             <p id="u2foot"></p>
-            <p>( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
+            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
     </div>

copyparty/web/util.js

@@ -85,6 +85,11 @@ function sortTable(table, col) {
     th[col].className = 'sort' + reverse;
     var stype = th[col].getAttribute('sort');
     tr = tr.sort(function (a, b) {
+        if (!a.cells[col])
+            return -1;
+        if (!b.cells[col])
+            return 1;
+
         var v1 = a.cells[col].textContent.trim();
         var v2 = b.cells[col].textContent.trim();
         if (stype == 'int') {
@@ -106,4 +111,124 @@ function makeSortable(table) {
             sortTable(table, i);
         };
     }(i));
-}
+}
+
+
+
+(function () {
+    var ops = document.querySelectorAll('#ops>a');
+    for (var a = 0; a < ops.length; a++) {
+        ops[a].onclick = opclick;
+    }
+})();
+
+
+function opclick(ev) {
+    if (ev) //ie
+        ev.preventDefault();
+
+    var dest = this.getAttribute('data-dest');
+    goto(dest);
+
+    // writing a blank value makes ie8 segfault w
+    if (window.localStorage)
+        localStorage.setItem('opmode', dest || '.');
+
+    var input = document.querySelector('.opview.act input:not([type="hidden"])')
+    if (input)
+        input.focus();
+}
+
+
+function goto(dest) {
+    var obj = document.querySelectorAll('.opview.act');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].classList.remove('act');
+
+    obj = document.querySelectorAll('#ops>a');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].classList.remove('act');
+
+    var others = ['path', 'files', 'widget'];
+    for (var a = 0; a < others.length; a++)
+        ebi(others[a]).classList.remove('hidden');
+
+    if (dest) {
+        var ui = ebi('op_' + dest);
+        ui.classList.add('act');
+        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+
+        var fn = window['goto_' + dest];
+        if (fn)
+            fn();
+    }
+}
+
+
+(function () {
+    goto();
+    if (window.localStorage) {
+        var op = localStorage.getItem('opmode');
+        if (op !== null && op !== '.')
+            goto(op);
+    }
+})();
+
+
+function linksplit(rp) {
+    var ret = [];
+    var apath = '/';
+    if (rp && rp.charAt(0) == '/')
+        rp = rp.slice(1);
+
+    while (rp) {
+        var link = rp;
+        var ofs = rp.indexOf('/');
+        if (ofs === -1) {
+            rp = null;
+        }
+        else {
+            link = rp.slice(0, ofs + 1);
+            rp = rp.slice(ofs + 1);
+        }
+        var vlink = link;
+        if (link.indexOf('/') !== -1)
+            vlink = link.slice(0, -1) + '<span>/</span>';
+
+        ret.push('<a href="' + apath + link + '">' + vlink + '</a>');
+        apath += link;
+    }
+    return ret;
+}
+
+
+function get_evpath() {
+    var ret = document.location.pathname;
+
+    if (ret.indexOf('/') !== 0)
+        ret = '/' + ret;
+
+    if (ret.lastIndexOf('/') !== ret.length - 1)
+        ret += '/';
+
+    return ret;
+}
+
+
+function get_vpath() {
+    return decodeURIComponent(get_evpath());
+}
+
+
+function unix2iso(ts) {
+    return new Date(ts * 1000).toISOString().replace("T", " ").slice(0, -5);
+}
+
+
+function has(haystack, needle) {
+    for (var a = 0; a < haystack.length; a++)
+        if (haystack[a] == needle)
+            return true;
+
+    return false;
+}

scripts/deps-docker/Dockerfile

@@ -1,12 +1,10 @@
-FROM    alpine:3.11
+FROM    alpine:3.13
 WORKDIR /z
-ENV     ver_asmcrypto=2821dd1dedd1196c378f5854037dda5c869313f3 \
+ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
-        ver_markdownit=10.0.0 \
-        ver_showdown=1.9.1 \
         ver_marked=1.1.0 \
-        ver_ogvjs=1.6.1 \
+        ver_ogvjs=1.8.0 \
-        ver_mde=2.10.1 \
+        ver_mde=2.14.0 \
-        ver_codemirror=5.53.2 \
+        ver_codemirror=5.59.3 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 
@@ -17,7 +15,7 @@ RUN     mkdir -p /z/dist/no-pk \
         && wget https://fonts.gstatic.com/s/sourcecodepro/v11/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevW.woff2 -O scp.woff2 \
         && apk add cmake make g++ git bash npm patch wget tar pigz brotli gzip unzip python3 python3-dev brotli py3-brotli \
         && wget https://github.com/brion/ogv.js/releases/download/$ver_ogvjs/ogvjs-$ver_ogvjs.zip -O ogvjs.zip \
-        && wget https://github.com/asmcrypto/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
+        && wget https://github.com/openpgpjs/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
         && wget https://github.com/markedjs/marked/archive/v$ver_marked.tar.gz -O marked.tgz \
         && wget https://github.com/Ionaru/easy-markdown-editor/archive/$ver_mde.tar.gz -O mde.tgz \
         && wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
@@ -52,6 +50,7 @@ RUN     tar -xf zopfli.tgz \
             -S . \
         && make -C build \
         && make -C build install \
+        && python3 -m ensurepip \
         && python3 -m pip install fonttools zopfli
 
 

scripts/deps-docker/codemirror.patch

@@ -1,6 +1,6 @@
-diff -NarU2 CodeMirror-orig/mode/gfm/gfm.js CodeMirror-edit/mode/gfm/gfm.js
+diff -NarU2 codemirror-5.59.3-orig/mode/gfm/gfm.js codemirror-5.59.3/mode/gfm/gfm.js
---- CodeMirror-orig/mode/gfm/gfm.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/mode/gfm/gfm.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/mode/gfm/gfm.js	2020-05-02 02:13:32.142131800 +0200
++++ codemirror-5.59.3/mode/gfm/gfm.js	2021-02-21 20:42:02.166174775 +0000
 @@ -97,5 +97,5 @@
          }
        }
@@ -15,9 +15,9 @@ diff -NarU2 CodeMirror-orig/mode/gfm/gfm.js CodeMirror-edit/mode/gfm/gfm.js
 +      }*/
        stream.next();
        return null;
-diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
+diff -NarU2 codemirror-5.59.3-orig/mode/meta.js codemirror-5.59.3/mode/meta.js
---- CodeMirror-orig/mode/meta.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/mode/meta.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/mode/meta.js	2020-05-02 03:56:58.852408400 +0200
++++ codemirror-5.59.3/mode/meta.js	2021-02-21 20:42:54.798742821 +0000
 @@ -13,4 +13,5 @@
  
    CodeMirror.modeInfo = [
@@ -28,7 +28,7 @@ diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
      {name: "Gas", mime: "text/x-gas", mode: "gas", ext: ["s"]},
      {name: "Gherkin", mime: "text/x-feature", mode: "gherkin", ext: ["feature"]},
 +    */
-     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history).md$/i},
+     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history)\.md$/i},
 +    /*
      {name: "Go", mime: "text/x-go", mode: "go", ext: ["go"]},
      {name: "Groovy", mime: "text/x-groovy", mode: "groovy", ext: ["groovy", "gradle"], file: /^Jenkinsfile$/},
@@ -56,16 +56,16 @@ diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
 +    /*
      {name: "XQuery", mime: "application/xquery", mode: "xquery", ext: ["xy", "xquery"]},
      {name: "Yacas", mime: "text/x-yacas", mode: "yacas", ext: ["ys"]},
-@@ -171,4 +180,5 @@
+@@ -172,4 +181,5 @@
-     {name: "xu", mime: "text/x-xu", mode: "mscgen", ext: ["xu"]},
+     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]},
-     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]}
+     {name: "WebAssembly", mime: "text/webassembly", mode: "wast", ext: ["wat", "wast"]},
 +    */
    ];
    // Ensure all modes have a mime property for backwards compatibility
-diff -NarU2 CodeMirror-orig/src/display/selection.js CodeMirror-edit/src/display/selection.js
+diff -NarU2 codemirror-5.59.3-orig/src/display/selection.js codemirror-5.59.3/src/display/selection.js
---- CodeMirror-orig/src/display/selection.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/display/selection.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/display/selection.js	2020-05-02 03:27:30.144662800 +0200
++++ codemirror-5.59.3/src/display/selection.js	2021-02-21 20:44:14.860894328 +0000
-@@ -83,29 +83,21 @@
+@@ -84,29 +84,21 @@
      let order = getOrder(lineObj, doc.direction)
      iterateBidiSections(order, fromArg || 0, toArg == null ? lineLen : toArg, (from, to, dir, i) => {
 -      let ltr = dir == "ltr"
@@ -105,24 +105,24 @@ diff -NarU2 CodeMirror-orig/src/display/selection.js CodeMirror-edit/src/display
 +          botRight = openEnd && last ? rightSide : toPos.right
          add(topLeft, fromPos.top, topRight - topLeft, fromPos.bottom)
          if (fromPos.bottom < toPos.top) add(leftSide, fromPos.bottom, null, toPos.top)
-diff -NarU2 CodeMirror-orig/src/input/ContentEditableInput.js CodeMirror-edit/src/input/ContentEditableInput.js
+diff -NarU2 codemirror-5.59.3-orig/src/input/ContentEditableInput.js codemirror-5.59.3/src/input/ContentEditableInput.js
---- CodeMirror-orig/src/input/ContentEditableInput.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/input/ContentEditableInput.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/input/ContentEditableInput.js	2020-05-02 03:33:05.707995500 +0200
++++ codemirror-5.59.3/src/input/ContentEditableInput.js	2021-02-21 20:44:33.273953867 +0000
-@@ -391,4 +391,5 @@
+@@ -399,4 +399,5 @@
    let info = mapFromLineView(view, line, pos.line)
  
 +  /*
    let order = getOrder(line, cm.doc.direction), side = "left"
    if (order) {
-@@ -396,4 +397,5 @@
+@@ -404,4 +405,5 @@
      side = partPos % 2 ? "right" : "left"
    }
 +  */
    let result = nodeAndOffsetInLineMap(info.map, pos.ch, side)
    result.offset = result.collapse == "right" ? result.end : result.start
-diff -NarU2 CodeMirror-orig/src/input/movement.js CodeMirror-edit/src/input/movement.js
+diff -NarU2 codemirror-5.59.3-orig/src/input/movement.js codemirror-5.59.3/src/input/movement.js
---- CodeMirror-orig/src/input/movement.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/input/movement.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/input/movement.js	2020-05-02 03:31:19.710773500 +0200
++++ codemirror-5.59.3/src/input/movement.js	2021-02-21 20:45:12.763093671 +0000
 @@ -15,4 +15,5 @@
  
  export function endOfLine(visually, cm, lineObj, lineNo, dir) {
@@ -146,9 +146,9 @@ diff -NarU2 CodeMirror-orig/src/input/movement.js CodeMirror-edit/src/input/move
    return null
 +  */
  }
-diff -NarU2 CodeMirror-orig/src/line/line_data.js CodeMirror-edit/src/line/line_data.js
+diff -NarU2 codemirror-5.59.3-orig/src/line/line_data.js codemirror-5.59.3/src/line/line_data.js
---- CodeMirror-orig/src/line/line_data.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/line/line_data.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/line/line_data.js	2020-05-02 03:17:02.785065000 +0200
++++ codemirror-5.59.3/src/line/line_data.js	2021-02-21 20:45:36.472549599 +0000
 @@ -79,6 +79,6 @@
      // Optionally wire in some hacks into the token-rendering
      // algorithm, to deal with browser quirks.
@@ -158,9 +158,9 @@ diff -NarU2 CodeMirror-orig/src/line/line_data.js CodeMirror-edit/src/line/line_
 +    //  builder.addToken = buildTokenBadBidi(builder.addToken, order)
      builder.map = []
      let allowFrontierUpdate = lineView != cm.display.externalMeasured && lineNo(line)
-diff -NarU2 CodeMirror-orig/src/measurement/position_measurement.js CodeMirror-edit/src/measurement/position_measurement.js
+diff -NarU2 codemirror-5.59.3-orig/src/measurement/position_measurement.js codemirror-5.59.3/src/measurement/position_measurement.js
---- CodeMirror-orig/src/measurement/position_measurement.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/measurement/position_measurement.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/measurement/position_measurement.js	2020-05-02 03:35:20.674159600 +0200
++++ codemirror-5.59.3/src/measurement/position_measurement.js	2021-02-21 20:50:52.372945293 +0000
 @@ -380,5 +380,6 @@
      sticky = "after"
    }
@@ -199,9 +199,9 @@ diff -NarU2 CodeMirror-orig/src/measurement/position_measurement.js CodeMirror-e
 +*/
  
  let measureText
-diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
+diff -NarU2 codemirror-5.59.3-orig/src/util/bidi.js codemirror-5.59.3/src/util/bidi.js
---- CodeMirror-orig/src/util/bidi.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/util/bidi.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/util/bidi.js	2020-05-02 03:12:44.418649800 +0200
++++ codemirror-5.59.3/src/util/bidi.js	2021-02-21 20:52:18.168092225 +0000
 @@ -4,5 +4,5 @@
  
  export function iterateBidiSections(order, from, to, f) {
@@ -239,20 +239,19 @@ diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
 +  var fun = function(str, direction) {
      let outerType = direction == "ltr" ? "L" : "R"
  
-@@ -204,12 +210,16 @@
+@@ -204,5 +210,11 @@
      return direction == "rtl" ? order.reverse() : order
    }
--})()
- 
 +  return function(str, direction) {
 +    var ret = fun(str, direction);
 +    console.log("bidiOrdering inner ([%s], %s) => [%s]", str, direction, ret);
 +    return ret;
 +  }
-+})()
+ })()
 +*/
+ 
  // Get the bidi ordering for the given line (and cache it). Returns
- // false for lines that are fully left-to-right, and an array of
+@@ -210,6 +222,4 @@
  // BidiSpan objects otherwise.
  export function getOrder(line, direction) {
 -  let order = line.order
@@ -260,9 +259,9 @@ diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
 -  return order
 +  return false;
  }
-diff -NarU2 CodeMirror-orig/src/util/feature_detection.js CodeMirror-edit/src/util/feature_detection.js
+diff -NarU2 codemirror-5.59.3-orig/src/util/feature_detection.js codemirror-5.59.3/src/util/feature_detection.js
---- CodeMirror-orig/src/util/feature_detection.js	2020-04-21 12:47:20.000000000 +0200
+--- codemirror-5.59.3-orig/src/util/feature_detection.js	2021-02-20 21:24:57.000000000 +0000
-+++ CodeMirror-edit/src/util/feature_detection.js	2020-05-02 03:16:21.085621400 +0200
++++ codemirror-5.59.3/src/util/feature_detection.js	2021-02-21 20:49:22.191269270 +0000
 @@ -25,4 +25,5 @@
  }
  

scripts/deps-docker/easymde.patch

@@ -1,33 +1,57 @@
-diff -NarU2 easymde-orig/gulpfile.js easymde-mod1/gulpfile.js
+diff -NarU2 easy-markdown-editor-2.14.0-orig/gulpfile.js easy-markdown-editor-2.14.0/gulpfile.js
---- easymde-orig/gulpfile.js	2020-04-06 14:09:36.000000000 +0200
+--- easy-markdown-editor-2.14.0-orig/gulpfile.js	2021-02-14 12:11:48.000000000 +0000
-+++ easymde-mod1/gulpfile.js	2020-05-01 14:33:52.260175200 +0200
++++ easy-markdown-editor-2.14.0/gulpfile.js	2021-02-21 20:55:37.134701007 +0000
 @@ -25,5 +25,4 @@
      './node_modules/codemirror/lib/codemirror.css',
      './src/css/*.css',
 -    './node_modules/codemirror-spell-checker/src/css/spell-checker.css',
  ];
  
-diff -NarU2 easymde-orig/package.json easymde-mod1/package.json
+diff -NarU2 easy-markdown-editor-2.14.0-orig/package.json easy-markdown-editor-2.14.0/package.json
---- easymde-orig/package.json	2020-04-06 14:09:36.000000000 +0200
+--- easy-markdown-editor-2.14.0-orig/package.json	2021-02-14 12:11:48.000000000 +0000
-+++ easymde-mod1/package.json	2020-05-01 14:33:57.189975800 +0200
++++ easy-markdown-editor-2.14.0/package.json	2021-02-21 20:55:47.761190082 +0000
 @@ -21,5 +21,4 @@
      "dependencies": {
-         "codemirror": "^5.52.2",
+         "codemirror": "^5.59.2",
 -        "codemirror-spell-checker": "1.1.2",
-         "marked": "^0.8.2"
+         "marked": "^2.0.0"
      },
-diff -NarU2 easymde-orig/src/js/easymde.js easymde-mod1/src/js/easymde.js
+diff -NarU2 easy-markdown-editor-2.14.0-orig/src/js/easymde.js easy-markdown-editor-2.14.0/src/js/easymde.js
---- easymde-orig/src/js/easymde.js	2020-04-06 14:09:36.000000000 +0200
+--- easy-markdown-editor-2.14.0-orig/src/js/easymde.js	2021-02-14 12:11:48.000000000 +0000
-+++ easymde-mod1/src/js/easymde.js	2020-05-01 14:34:19.878774400 +0200
++++ easy-markdown-editor-2.14.0/src/js/easymde.js	2021-02-21 20:57:09.143171536 +0000
-@@ -11,5 +11,4 @@
+@@ -12,5 +12,4 @@
  require('codemirror/mode/gfm/gfm.js');
  require('codemirror/mode/xml/xml.js');
 -var CodeMirrorSpellChecker = require('codemirror-spell-checker');
  var marked = require('marked/lib/marked');
  
-@@ -1889,18 +1888,7 @@
+@@ -1762,9 +1761,4 @@
+         options.autosave.uniqueId = options.autosave.unique_id;
  
+-    // If overlay mode is specified and combine is not provided, default it to true
+-    if (options.overlayMode && options.overlayMode.combine === undefined) {
+-      options.overlayMode.combine = true;
+-    }
+-
+     // Update this options
+     this.options = options;
+@@ -2003,28 +1997,7 @@
      var mode, backdrop;
+ 
+-    // CodeMirror overlay mode
+-    if (options.overlayMode) {
+-      CodeMirror.defineMode('overlay-mode', function(config) {
+-        return CodeMirror.overlayMode(CodeMirror.getMode(config, options.spellChecker !== false ? 'spell-checker' : 'gfm'), options.overlayMode.mode, options.overlayMode.combine);
+-      });
+-
+-      mode = 'overlay-mode';
+-      backdrop = options.parsingConfig;
+-      backdrop.gitHubSpice = false;
+-    } else {
+         mode = options.parsingConfig;
+         mode.name = 'gfm';
+         mode.gitHubSpice = false;
+-    }
 -    if (options.spellChecker !== false) {
 -        mode = 'spell-checker';
 -        backdrop = options.parsingConfig;
@@ -37,16 +61,28 @@ diff -NarU2 easymde-orig/src/js/easymde.js easymde-mod1/src/js/easymde.js
 -        CodeMirrorSpellChecker({
 -            codeMirrorInstance: CodeMirror,
 -        });
--    } else {
-         mode = options.parsingConfig;
-         mode.name = 'gfm';
-         mode.gitHubSpice = false;
 -    }
  
      // eslint-disable-next-line no-unused-vars
-@@ -1927,5 +1915,4 @@
+diff -NarU2 easy-markdown-editor-2.14.0-orig/types/easymde.d.ts easy-markdown-editor-2.14.0/types/easymde.d.ts
-         configureMouse: configureMouse,
+--- easy-markdown-editor-2.14.0-orig/types/easymde.d.ts	2021-02-14 12:11:48.000000000 +0000
-         inputStyle: (options.inputStyle != undefined) ? options.inputStyle : isMobile() ? 'contenteditable' : 'textarea',
++++ easy-markdown-editor-2.14.0/types/easymde.d.ts	2021-02-21 20:57:42.492620979 +0000
--        spellcheck: (options.nativeSpellcheck != undefined) ? options.nativeSpellcheck : true,
+@@ -160,9 +160,4 @@
-     });
+     }
  
+-    interface OverlayModeOptions {
+-      mode: CodeMirror.Mode<any>
+-      combine?: boolean
+-    }
+-
+     interface Options {
+         autoDownloadFontAwesome?: boolean;
+@@ -214,7 +209,5 @@
+ 
+         promptTexts?: PromptTexts;
+-        syncSideBySidePreviewScroll?: boolean;
+-
+-        overlayMode?: OverlayModeOptions
++        syncSideBySidePreviewScroll?: boolean
+     }
+ }

scripts/make-pypi-release.sh

@@ -86,6 +86,8 @@ function have() {
 	python -c "import $1; $1; $1.__version__"
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 . buildenv/bin/activate
 have setuptools
 have wheel

scripts/make-sfx.sh

@@ -62,28 +62,32 @@ cd sfx
 	)/pe-copyparty"
 
 	echo "repack of files in $old"
-	cp -pR "$old/"*{jinja2,copyparty} .
+	cp -pR "$old/"*{dep-j2,copyparty} .
-	mv {x.,}jinja2 2>/dev/null || true
 }
 
 [ $repack ] || {
 	echo collecting jinja2
-	f="../build/Jinja2-2.6.tar.gz"
+	f="../build/Jinja2-2.11.3.tar.gz"
 	[ -e "$f" ] ||
-		(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz;
+		(url=https://files.pythonhosted.org/packages/4f/e7/65300e6b32e69768ded990494809106f87da1d436418d5f1367ed3966fd7/Jinja2-2.11.3.tar.gz;
 		wget -O$f "$url" || curl -L "$url" >$f)
 
 	tar -zxf $f
-	mv Jinja2-*/jinja2 .
+	mv Jinja2-*/src/jinja2 .
-	rm -rf Jinja2-* jinja2/testsuite jinja2/_markupsafe/tests.py jinja2/_stringdefs.py
+	rm -rf Jinja2-*
 	
-	f=jinja2/lexer.py
+	echo collecting markupsafe
-	sed -r '/.*föö.*/    raise SyntaxError/' <$f >t
+	f="../build/MarkupSafe-1.1.1.tar.gz"
-	tmv $f
+	[ -e "$f" ] ||
-	
+		(url=https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz;
-	f=jinja2/_markupsafe/_constants.py
+		wget -O$f "$url" || curl -L "$url" >$f)
-	awk '!/: [0-9]+,?$/ || /(amp|gt|lt|quot|apos|nbsp).:/' <$f >t
+
-	tmv $f
+	tar -zxf $f
+	mv MarkupSafe-*/src/markupsafe .
+	rm -rf MarkupSafe-* markupsafe/_speedups.c
+
+	mkdir dep-j2/
+	mv {markupsafe,jinja2} dep-j2/
 
 	# msys2 tar is bad, make the best of it
 	echo collecting source
@@ -165,6 +169,15 @@ done
 	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
 }
 
+find | grep -E '\.py$' |
+  grep -vE '__version__' |
+  tr '\n' '\0' |
+  xargs -0 python ../scripts/uncomment.py
+
+f=dep-j2/jinja2/constants.py
+awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
+tmv "$f"
+
 # up2k goes from 28k to 22k laff
 echo entabbening
 find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
@@ -177,7 +190,7 @@ args=(--owner=1000 --group=1000)
 [ "$OSTYPE" = msys ] &&
 	args=()
 
-tar -cf tar "${args[@]}" --numeric-owner copyparty jinja2
+tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9

scripts/make-tgz-release.sh

@@ -35,6 +35,8 @@ ver="$1"
 	exit 1
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 mkdir -p dist
 zip_path="$(pwd)/dist/copyparty-$ver.zip"
 tgz_path="$(pwd)/dist/copyparty-$ver.tar.gz"

scripts/sfx.py

@@ -2,7 +2,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re, os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
+import os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
 import subprocess as sp
 
 """
@@ -202,93 +202,6 @@ def u8(gen):
             yield s
 
 
-def get_py_win(ret):
-    tops = []
-    p = str(os.getenv("LocalAppdata"))
-    if p:
-        tops.append(os.path.join(p, "Programs", "Python"))
-
-    progfiles = {}
-    for p in ["ProgramFiles", "ProgramFiles(x86)"]:
-        p = str(os.getenv(p))
-        if p:
-            progfiles[p] = 1
-            # 32bit apps get x86 for both
-            if p.endswith(" (x86)"):
-                progfiles[p[:-6]] = 1
-
-    tops += list(progfiles.keys())
-
-    for sysroot in [me, sys.executable]:
-        sysroot = sysroot[:3].upper()
-        if sysroot[1] == ":" and sysroot not in tops:
-            tops.append(sysroot)
-
-    # $WIRESHARK_SLOGAN
-    for top in tops:
-        try:
-            for name1 in u8(sorted(os.listdir(top), reverse=True)):
-                if name1.lower().startswith("python"):
-                    path1 = os.path.join(top, name1)
-                    try:
-                        for name2 in u8(os.listdir(path1)):
-                            if name2.lower() == "python.exe":
-                                path2 = os.path.join(path1, name2)
-                                ret[path2.lower()] = path2
-                    except:
-                        pass
-        except:
-            pass
-
-
-def get_py_nix(ret):
-    ptn = re.compile(r"^(python|pypy)[0-9\.-]*$")
-    for bindir in os.getenv("PATH").split(":"):
-        if not bindir:
-            next
-
-        try:
-            for fn in u8(os.listdir(bindir)):
-                if ptn.match(fn):
-                    fn = os.path.join(bindir, fn)
-                    ret[fn.lower()] = fn
-        except:
-            pass
-
-
-def read_py(binp):
-    cmd = [
-        binp,
-        "-c",
-        "import sys; sys.stdout.write(' '.join(str(x) for x in sys.version_info)); import jinja2",
-    ]
-    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
-    ver, _ = p.communicate()
-    ver = ver.decode("utf-8").split(" ")[:3]
-    ver = [int(x) if x.isdigit() else 0 for x in ver]
-    return ver, p.returncode == 0
-
-
-def get_pys():
-    ver, chk = read_py(sys.executable)
-    if chk or PY2:
-        return [[chk, ver, sys.executable]]
-
-    hits = {sys.executable.lower(): sys.executable}
-    if platform.system() == "Windows":
-        get_py_win(hits)
-    else:
-        get_py_nix(hits)
-
-    ret = []
-    for binp in hits.values():
-        ver, chk = read_py(binp)
-        ret.append([chk, ver, binp])
-        msg("\t".join(str(x) for x in ret[-1]))
-
-    return ret
-
-
 def yieldfile(fn):
     with open(fn, "rb") as f:
         for block in iter(lambda: f.read(64 * 1024), b""):
@@ -440,12 +353,11 @@ def confirm():
         pass
 
 
-def run(tmp, py):
+def run(tmp, j2ver):
     global cpp
 
-    msg("OK")
+    msg("jinja2:", j2ver or "bundled")
-    msg("will use:", py)
+    msg("sfxdir:", tmp)
-    msg("bound to:", tmp)
 
     # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
     try:
@@ -457,24 +369,20 @@ def run(tmp, py):
     except:
         pass
 
-    fp_py = os.path.join(tmp, "py")
+    ld = [tmp, os.path.join(tmp, "dep-j2")]
-    try:
+    if j2ver:
-        with open(fp_py, "wb") as f:
+        del ld[-1]
-            f.write(py.encode("utf-8") + b"\n")
-    except:
-        pass
 
-    # avoid loading ./copyparty.py
+    cmd = (
-    cmd = [
+        "import sys, runpy; "
-        py,
+        + "".join(['sys.path.insert(0, r"' + x + '"); ' for x in ld])
-        "-c",
+        + 'runpy.run_module("copyparty", run_name="__main__")'
-        'import sys, runpy; sys.path.insert(0, r"'
+    )
-        + tmp
+    cmd = [sys.executable, "-c", cmd] + list(sys.argv[1:])
-        + '"); runpy.run_module("copyparty", run_name="__main__")',
-    ] + list(sys.argv[1:])
 
+    cmd = [str(x) for x in cmd]
     msg("\n", cmd, "\n")
-    cpp = sp.Popen(str(x) for x in cmd)
+    cpp = sp.Popen(cmd)
     try:
         cpp.wait()
     except:
@@ -494,7 +402,6 @@ def bye(sig, frame):
 def main():
     sysver = str(sys.version).replace("\n", "\n" + " " * 18)
     pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
-    os.system("")
     msg()
     msg("   this is: copyparty", VER)
     msg(" packed at:", pktime, "UTC,", STAMP)
@@ -526,33 +433,13 @@ def main():
     signal.signal(signal.SIGTERM, bye)
 
     tmp = unpack()
-    fp_py = os.path.join(tmp, "py")
-    if os.path.exists(fp_py):
-        with open(fp_py, "rb") as f:
-            py = f.read().decode("utf-8").rstrip()
 
-        return run(tmp, py)
+    try:
+        from jinja2 import __version__ as j2ver
+    except:
+        j2ver = None
 
-    pys = get_pys()
+    return run(tmp, j2ver)
-    pys.sort(reverse=True)
-    j2, ver, py = pys[0]
-    if j2:
-        try:
-            os.rename(os.path.join(tmp, "jinja2"), os.path.join(tmp, "x.jinja2"))
-        except:
-            pass
-
-        return run(tmp, py)
-
-    msg("\n  could not find jinja2; will use py2 + the bundled version\n")
-    for _, ver, py in pys:
-        if ver > [2, 7] and ver < [3, 0]:
-            return run(tmp, py)
-
-    m = "\033[1;31m\n\n\ncould not find a python with jinja2 installed; please do one of these:\n\n  pip install --user jinja2\n\n  install python2\n\n\033[0m"
-    msg(m)
-    confirm()
-    sys.exit(1)
 
 
 if __name__ == "__main__":

scripts/uncomment.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import io
+import sys
+import tokenize
+
+
+def uncomment(fpath):
+    """ modified https://stackoverflow.com/a/62074206 """
+
+    with open(fpath, "rb") as f:
+        orig = f.read().decode("utf-8")
+
+    out = ""
+    for ln in orig.split("\n"):
+        if not ln.startswith("#"):
+            break
+
+        out += ln + "\n"
+
+    io_obj = io.StringIO(orig)
+    prev_toktype = tokenize.INDENT
+    last_lineno = -1
+    last_col = 0
+    for tok in tokenize.generate_tokens(io_obj.readline):
+        # print(repr(tok))
+        token_type = tok[0]
+        token_string = tok[1]
+        start_line, start_col = tok[2]
+        end_line, end_col = tok[3]
+
+        if start_line > last_lineno:
+            last_col = 0
+
+        if start_col > last_col:
+            out += " " * (start_col - last_col)
+
+        is_legalese = (
+            "copyright" in token_string.lower() or "license" in token_string.lower()
+        )
+
+        if token_type == tokenize.STRING:
+            if (
+                prev_toktype != tokenize.INDENT
+                and prev_toktype != tokenize.NEWLINE
+                and start_col > 0
+                or is_legalese
+            ):
+                out += token_string
+            else:
+                out += '"a"'
+        elif token_type != tokenize.COMMENT or is_legalese:
+            out += token_string
+
+        prev_toktype = token_type
+        last_lineno = end_line
+        last_col = end_col
+
+    # out = "\n".join(x for x in out.splitlines() if x.strip())
+
+    with open(fpath, "wb") as f:
+        f.write(out.encode("utf-8"))
+
+
+def main():
+    print("uncommenting", end="")
+    for f in sys.argv[1:]:
+        print(".", end="")
+        uncomment(f)
+
+    print("k")
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -2,10 +2,8 @@
 # coding: utf-8
 from __future__ import print_function
 
-import io
 import os
 import sys
-from glob import glob
 from shutil import rmtree
 
 setuptools_available = True
@@ -49,7 +47,7 @@ with open(here + "/README.md", "rb") as f:
 about = {}
 if not VERSION:
     with open(os.path.join(here, NAME, "__version__.py"), "rb") as f:
-        exec(f.read().decode("utf-8").split("\n\n", 1)[1], about)
+        exec (f.read().decode("utf-8").split("\n\n", 1)[1], about)
 else:
     about["__version__"] = VERSION
 
@@ -110,13 +108,13 @@ args = {
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Environment :: Console",