v0.8.3

.vscode/launch.json

@@ -12,8 +12,7 @@
                 //"-nw",
                 "-ed",
                 "-emp",
-                "-e2d",
-                "-e2s",
+                "-e2dsa",
                 "-a",
                 "ed:wark",
                 "-v",

.vscode/tasks.json

@@ -8,7 +8,7 @@
         },
         {
             "label": "no_dbg",
-            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2d -e2s -a ed:wark -v srv::r:aed:cnodupe ;exit 1",
+            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -a ed:wark -v srv::r:aed:cnodupe ;exit 1",
             "type": "shell"
         }
     ]

README.md

@@ -36,24 +36,55 @@ you may also want these, especially on servers:
 
 ## status
 
-* [x] sanic multipart parser
-* [x] load balancer (multiprocessing)
-* [x] upload (plain multipart, ie6 support)
-* [x] upload (js, resumable, multithreaded)
-* [x] download
-* [x] browser
-* [x] media player
-* [ ] thumbnails
-* [ ] download as zip
-* [x] volumes
-* [x] accounts
-* [x] markdown viewer
-* [x] markdown editor
-* [x] FUSE client (read-only)
+* backend stuff
+  * ☑ sanic multipart parser
+  * ☑ load balancer (multiprocessing)
+  * ☑ volumes (mountpoints)
+  * ☑ accounts
+* upload
+  * ☑ basic: plain multipart, ie6 support
+  * ☑ up2k: js, resumable, multithreaded
+  * ☑ stash: simple PUT filedropper
+  * ☑ symlink/discard existing files (content-matching)
+* download
+  * ☑ single files in browser
+  * ✖ folders as zip files
+  * ☑ FUSE client (read-only)
+* browser
+  * ☑ tree-view
+  * ☑ media player
+  * ✖ thumbnails
+  * ✖ SPA (browse while uploading)
+    * currently safe using the file-tree on the left only, not folders in the file list
+* server indexing
+  * ☑ locate files by contents
+  * ☑ search by name/path/date/size
+  * ✖ search by ID3-tags etc.
+* markdown
+  * ☑ viewer
+  * ☑ editor (sure why not)
 
 summary: it works! you can use it! (but technically not even close to beta)
 
 
+# bugs
+
+* probably, pls let me know
+
+
+# searching
+
+when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
+* make search queries by `size`/`date`/`directory-path`/`filename`, or...
+* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+
+path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
+* path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
+* name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9)
+
+other metadata (like song tags etc) are not yet indexed for searching
+
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)

bin/copyparty-fuse.py

@@ -33,6 +33,7 @@ import re
 import os
 import sys
 import time
+import json
 import stat
 import errno
 import struct
@@ -323,7 +324,7 @@ class Gateway(object):
         if bad_good:
             path = dewin(path)
 
-        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
+        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
         r = self.sendreq("GET", web_path)
         if r.status != 200:
             self.closeconn()
@@ -334,12 +335,17 @@ class Gateway(object):
             )
             raise FuseOSError(errno.ENOENT)
 
-        if not r.getheader("Content-Type", "").startswith("text/html"):
+        ctype = r.getheader("Content-Type", "")
+        if ctype == "application/json":
+            parser = self.parse_jls
+        elif ctype.startswith("text/html"):
+            parser = self.parse_html
+        else:
             log("listdir on file: {}".format(path))
             raise FuseOSError(errno.ENOENT)
 
         try:
-            return self.parse_html(r)
+            return parser(r)
         except:
             info(repr(path) + "\n" + traceback.format_exc())
             raise
@@ -367,6 +373,29 @@ class Gateway(object):
 
         return r.read()
 
+    def parse_jls(self, datasrc):
+        rsp = b""
+        while True:
+            buf = datasrc.read(1024 * 32)
+            if not buf:
+                break
+
+            rsp += buf
+
+        rsp = json.loads(rsp.decode("utf-8"))
+        ret = []
+        for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
+            for n in nodes:
+                fname = unquote(n["href"]).rstrip(b"/")
+                fname = fname.decode("wtf-8")
+                if bad_good:
+                    fname = enwin(fname)
+
+                fun = self.stat_dir if is_dir else self.stat_file
+                ret.append([fname, fun(n["ts"], n["sz"]), 0])
+
+        return ret
+
     def parse_html(self, datasrc):
         ret = []
         remainder = b""
@@ -818,9 +847,9 @@ class CPPF(Operations):
                 return cache_stat
 
         fun = info
-        if MACOS and path.split('/')[-1].startswith('._'):
+        if MACOS and path.split("/")[-1].startswith("._"):
             fun = dbg
-        
+
         fun("=ENOENT ({})".format(hexler(path)))
         raise FuseOSError(errno.ENOENT)
 

copyparty/__main__.py

@@ -174,6 +174,18 @@ def main():
     if HAVE_SSL:
         ensure_cert()
 
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = sys.argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        sys.argv[idx] = nk
+        time.sleep(2)
+
     ap = argparse.ArgumentParser(
         formatter_class=RiceFormatter,
         prog="copyparty",
@@ -228,13 +240,15 @@ def main():
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-e2d", action="store_true", help="enable up2k database")
-    ap.add_argument("-e2s", action="store_true", help="enable up2k db-scanner")
+    ap.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
+    ap.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
+    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 
     ap2 = ap.add_argument_group('SSL/TLS options')
     ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
@@ -246,6 +260,12 @@ def main():
     al = ap.parse_args()
     # fmt: on
 
+    if al.e2dsa:
+        al.e2ds = True
+
+    if al.e2ds:
+        al.e2d = True
+
     al.i = al.i.split(",")
     try:
         if "-" in al.p:

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 7, 7)
-CODENAME = "keeping track"
-BUILD_DT = (2021, 2, 14)
+VERSION = (0, 8, 3)
+CODENAME = "discovery"
+BUILD_DT = (2021, 2, 22)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -19,6 +19,11 @@ class VFS(object):
         self.uwrite = uwrite  # users who can write this
         self.flags = flags  # config switches
         self.nodes = {}  # child nodes
+        self.all_vols = {vpath: self}  # flattened recursive
+
+    def _trk(self, vol):
+        self.all_vols[vol.vpath] = vol
+        return vol
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""
@@ -30,7 +35,7 @@ class VFS(object):
             name, dst = dst.split("/", 1)
             if name in self.nodes:
                 # exists; do not manipulate permissions
-                return self.nodes[name].add(src, dst)
+                return self._trk(self.nodes[name].add(src, dst))
 
             vn = VFS(
                 "{}/{}".format(self.realpath, name),
@@ -40,7 +45,7 @@ class VFS(object):
                 self.flags,
             )
             self.nodes[name] = vn
-            return vn.add(src, dst)
+            return self._trk(vn.add(src, dst))
 
         if dst in self.nodes:
             # leaf exists; return as-is
@@ -50,7 +55,7 @@ class VFS(object):
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
         self.nodes[dst] = vn
-        return vn
+        return self._trk(vn)
 
     def _find(self, vpath):
         """return [vfs,remainder]"""
@@ -257,7 +262,6 @@ class AuthSrv(object):
                 with open(cfg_fn, "rb") as f:
                     self._parse_config_file(f, user, mread, mwrite, mflags, mount)
 
-        self.all_writable = []
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
@@ -280,11 +284,6 @@ class AuthSrv(object):
             v.uread = mread[dst]
             v.uwrite = mwrite[dst]
             v.flags = mflags[dst]
-            if v.uwrite:
-                self.all_writable.append(v)
-
-        if vfs.uwrite and vfs not in self.all_writable:
-            self.all_writable.append(vfs)
 
         missing_users = {}
         for d in [mread, mwrite]:

copyparty/httpcli.py

@@ -5,6 +5,7 @@ import os
 import stat
 import gzip
 import time
+import copy
 import json
 import socket
 import ctypes
@@ -76,6 +77,8 @@ class HttpCli(object):
             self.loud_reply(str(ex), status=ex.code)
             return self.keepalive
 
+        # time.sleep(0.4)
+
         # normalize incoming headers to lowercase;
         # outgoing headers however are Correct-Case
         for header_line in headerlines[1:]:
@@ -125,15 +128,15 @@ class HttpCli(object):
                     k, v = k.split("=", 1)
                     uparam[k.lower()] = v.strip()
                 else:
-                    uparam[k.lower()] = True
+                    uparam[k.lower()] = False
 
         self.uparam = uparam
         self.vpath = unquotep(vpath)
 
         ua = self.headers.get("user-agent", "")
         if ua.startswith("rclone/"):
-            uparam["raw"] = True
-            uparam["dots"] = True
+            uparam["raw"] = False
+            uparam["dots"] = False
 
         try:
             if self.mode in ["GET", "HEAD"]:
@@ -237,12 +240,15 @@ class HttpCli(object):
         )
         if not self.readable and not self.writable:
             self.log("inaccessible: [{}]".format(self.vpath))
-            self.uparam = {"h": True}
+            self.uparam = {"h": False}
 
         if "h" in self.uparam:
             self.vpath = None
             return self.tx_mounts()
 
+        if "tree" in self.uparam:
+            return self.tx_tree()
+
         return self.tx_browser()
 
     def handle_options(self):
@@ -307,7 +313,7 @@ class HttpCli(object):
                 reader, _ = self.get_body_reader()
                 for buf in reader:
                     buf = buf.decode("utf-8", "replace")
-                    self.log("urlform:\n  {}\n".format(buf))
+                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
 
             if "get" in opt:
                 return self.handle_get()
@@ -401,6 +407,9 @@ class HttpCli(object):
         except:
             raise Pebkac(422, "you POSTed invalid json")
 
+        if "srch" in self.uparam or "srch" in body:
+            return self.handle_search(body)
+
         # prefer this over undot; no reason to allow traversion
         if "/" in body["name"]:
             raise Pebkac(400, "folders verboten")
@@ -426,6 +435,30 @@ class HttpCli(object):
         self.reply(response.encode("utf-8"), mime="application/json")
         return True
 
+    def handle_search(self, body):
+        vols = []
+        for vtop in self.rvol:
+            vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
+            vols.append([vfs.vpath, vfs.realpath, vfs.flags])
+
+        idx = self.conn.get_u2idx()
+        if "srch" in body:
+            # search by up2k hashlist
+            vbody = copy.deepcopy(body)
+            vbody["hash"] = len(vbody["hash"])
+            self.log("qj: " + repr(vbody))
+            hits = idx.fsearch(vols, body)
+            self.log("q#: " + repr(hits))
+        else:
+            # search by query params
+            self.log("qj: " + repr(body))
+            hits = idx.search(vols, body)
+            self.log("q#: " + str(len(hits)))
+
+        r = json.dumps(hits).encode("utf-8")
+        self.reply(r, mime="application/json")
+        return True
+
     def handle_post_binary(self):
         try:
             remains = int(self.headers["content-length"])
@@ -1037,6 +1070,60 @@ class HttpCli(object):
         self.reply(html.encode("utf-8"))
         return True
 
+    def tx_tree(self):
+        top = self.uparam["tree"] or ""
+        dst = self.vpath
+        if top in [".", ".."]:
+            top = undot(self.vpath + "/" + top)
+
+        if top == dst:
+            dst = ""
+        elif top:
+            if not dst.startswith(top + "/"):
+                raise Pebkac(400, "arg funk")
+
+            dst = dst[len(top) + 1 :]
+
+        ret = self.gen_tree(top, dst)
+        ret = json.dumps(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
+        return True
+
+    def gen_tree(self, top, target):
+        ret = {}
+        excl = None
+        if target:
+            excl, target = (target.split("/", 1) + [""])[:2]
+            ret["k" + excl] = self.gen_tree("/".join([top, excl]).strip("/"), target)
+
+        try:
+            vn, rem = self.auth.vfs.get(top, self.uname, True, False)
+            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)
+        except:
+            vfs_ls = []
+            vfs_virt = {}
+            for v in self.rvol:
+                d1, d2 = v.rsplit("/", 1) if "/" in v else ["", v]
+                if d1 == top:
+                    vfs_virt[d2] = 0
+
+        dirs = []
+
+        if not self.args.ed or "dots" not in self.uparam:
+            vfs_ls = exclude_dotfiles(vfs_ls)
+
+        for fn in [x for x in vfs_ls if x != excl]:
+            abspath = os.path.join(fsroot, fn)
+            if os.path.isdir(abspath):
+                dirs.append(fn)
+
+        for x in vfs_virt.keys():
+            if x != excl:
+                dirs.append(x)
+
+        ret["a"] = dirs
+        return ret
+
     def tx_browser(self):
         vpath = ""
         vpnodes = [["", "/"]]
@@ -1062,8 +1149,7 @@ class HttpCli(object):
             if abspath.endswith(".md") and "raw" not in self.uparam:
                 return self.tx_md(abspath)
 
-            bad = "{0}.hist{0}up2k.".format(os.sep)
-            if abspath.endswith(bad + "db") or abspath.endswith(bad + "snap"):
+            if rem.startswith(".hist/up2k."):
                 raise Pebkac(403)
 
             return self.tx_file(abspath)
@@ -1092,21 +1178,23 @@ class HttpCli(object):
             vfs_ls = exclude_dotfiles(vfs_ls)
 
         hidden = []
-        if fsroot.endswith(str(os.sep) + ".hist"):
-            hidden = ["up2k.db", "up2k.snap"]
+        if rem == ".hist":
+            hidden = ["up2k."]
+
+        is_ls = "ls" in self.uparam
 
         dirs = []
         files = []
         for fn in vfs_ls:
             base = ""
             href = fn
-            if self.absolute_urls and vpath:
+            if not is_ls and self.absolute_urls and vpath:
                 base = "/" + vpath + "/"
                 href = base + fn
 
             if fn in vfs_virt:
                 fspath = vfs_virt[fn].realpath
-            elif fn in hidden:
+            elif hidden and any(fn.startswith(x) for x in hidden):
                 continue
             else:
                 fspath = fsroot + "/" + fn
@@ -1137,30 +1225,20 @@ class HttpCli(object):
             except:
                 ext = "%"
 
-            item = [margin, quotep(href), html_escape(fn), sz, ext, dt]
+            item = {
+                "lead": margin,
+                "href": quotep(href),
+                "name": fn,
+                "sz": sz,
+                "ext": ext,
+                "dt": dt,
+                "ts": inf.st_mtime,
+            }
             if is_dir:
                 dirs.append(item)
             else:
                 files.append(item)
 
-        logues = [None, None]
-        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
-            fn = os.path.join(abspath, fn)
-            if os.path.exists(fsenc(fn)):
-                with open(fsenc(fn), "rb") as f:
-                    logues[n] = f.read().decode("utf-8")
-
-        if False:
-            # this is a mistake
-            md = None
-            for fn in [x[2] for x in files]:
-                if fn.lower() == "readme.md":
-                    fn = os.path.join(abspath, fn)
-                    with open(fn, "rb") as f:
-                        md = f.read().decode("utf-8")
-
-                    break
-
         srv_info = []
 
         try:
@@ -1189,21 +1267,49 @@ class HttpCli(object):
         except:
             pass
 
+        srv_info = "</span> /// <span>".join(srv_info)
+
+        perms = []
+        if self.readable:
+            perms.append("read")
+        if self.writable:
+            perms.append("write")
+
+        logues = ["", ""]
+        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
+            fn = os.path.join(abspath, fn)
+            if os.path.exists(fsenc(fn)):
+                with open(fsenc(fn), "rb") as f:
+                    logues[n] = f.read().decode("utf-8")
+
+        if is_ls:
+            [x.pop(k) for k in ["name", "dt"] for y in [dirs, files] for x in y]
+            ret = {
+                "dirs": dirs,
+                "files": files,
+                "srvinf": srv_info,
+                "perms": perms,
+                "logues": logues,
+            }
+            ret = json.dumps(ret)
+            self.reply(ret.encode("utf-8", "replace"), mime="application/json")
+            return True
+
         ts = ""
         # ts = "?{}".format(time.time())
 
         dirs.extend(files)
+
         html = self.conn.tpl_browser.render(
             vdir=quotep(self.vpath),
             vpnodes=vpnodes,
             files=dirs,
-            can_upload=self.writable,
-            can_read=self.readable,
             ts=ts,
-            prologue=logues[0],
-            epilogue=logues[1],
+            perms=json.dumps(perms),
+            have_up2k_idx=self.args.e2d,
+            logues=logues,
             title=html_escape(self.vpath),
-            srv_info="</span> /// <span>".join(srv_info),
+            srv_info=srv_info,
         )
         self.reply(html.encode("utf-8", "replace"))
         return True

copyparty/httpconn.py

@@ -30,6 +30,7 @@ except ImportError:
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
+from .u2idx import U2idx
 
 
 class HttpConn(object):
@@ -50,6 +51,7 @@ class HttpConn(object):
         self.t0 = time.time()
         self.nbyte = 0
         self.workload = 0
+        self.u2idx = None
         self.log_func = hsrv.log
         self.set_rproxy()
 
@@ -80,6 +82,12 @@ class HttpConn(object):
     def log(self, msg):
         self.log_func(self.log_src, msg)
 
+    def get_u2idx(self):
+        if not self.u2idx:
+            self.u2idx = U2idx(self.args, self.log_func)
+
+        return self.u2idx
+
     def _detect_https(self):
         method = None
         if self.cert_path:

copyparty/svchub.py

@@ -39,9 +39,13 @@ class SvcHub(object):
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
-        if self.args.e2d and self.args.e2s:
+        if self.args.e2ds:
             auth = AuthSrv(self.args, self.log, False)
-            self.up2k.build_indexes(auth.all_writable)
+            vols = auth.vfs.all_vols.values()
+            if not self.args.e2dsa:
+                vols = [x for x in vols if x.uwrite]
+
+            self.up2k.build_indexes(vols)
 
         # decide which worker impl to use
         if self.check_mp_enable():
@@ -79,7 +83,7 @@ class SvcHub(object):
             now = time.time()
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
-                print("\033[36m{}\033[0m".format(dt.strftime("%Y-%m-%d")))
+                print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
 
                 # unix timestamp of next 00:00:00 (leap-seconds safe)
                 day_now = dt.day
@@ -89,9 +93,9 @@ class SvcHub(object):
                 dt = dt.replace(hour=0, minute=0, second=0)
                 self.next_day = calendar.timegm(dt.utctimetuple())
 
-            fmt = "\033[36m{} \033[33m{:21} \033[0m{}"
+            fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
-                fmt = "{} {:21} {}"
+                fmt = "{} {:21} {}\n"
                 if "\033" in msg:
                     msg = self.ansi_re.sub("", msg)
                 if "\033" in src:
@@ -100,12 +104,12 @@ class SvcHub(object):
             ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             msg = fmt.format(ts, src, msg)
             try:
-                print(msg)
+                print(msg, end="")
             except UnicodeEncodeError:
                 try:
-                    print(msg.encode("utf-8", "replace").decode())
+                    print(msg.encode("utf-8", "replace").decode(), end="")
                 except:
-                    print(msg.encode("ascii", "replace").decode())
+                    print(msg.encode("ascii", "replace").decode(), end="")
 
     def check_mp_support(self):
         vmin = sys.version_info[1]

copyparty/u2idx.py

@@ -0,0 +1,148 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+from datetime import datetime
+
+from .util import u8safe
+from .up2k import up2k_wark_from_hashlist
+
+
+try:
+    HAVE_SQLITE3 = True
+    import sqlite3
+except:
+    HAVE_SQLITE3 = False
+
+
+class U2idx(object):
+    def __init__(self, args, log_func):
+        self.args = args
+        self.log_func = log_func
+
+        if not HAVE_SQLITE3:
+            self.log("could not load sqlite3; searchign wqill be disabled")
+            return
+
+        self.dbs = {}
+
+    def log(self, msg):
+        self.log_func("u2idx", msg)
+
+    def fsearch(self, vols, body):
+        """search by up2k hashlist"""
+        if not HAVE_SQLITE3:
+            return []
+
+        fsize = body["size"]
+        fhash = body["hash"]
+        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
+        return self.run_query(vols, "select * from up where w = ?", [wark])
+
+    def search(self, vols, body):
+        """search by query params"""
+        if not HAVE_SQLITE3:
+            return []
+
+        qobj = {}
+        _conv_sz(qobj, body, "sz_min", "sz >= ?")
+        _conv_sz(qobj, body, "sz_max", "sz <= ?")
+        _conv_dt(qobj, body, "dt_min", "mt >= ?")
+        _conv_dt(qobj, body, "dt_max", "mt <= ?")
+        for seg, dk in [["path", "rd"], ["name", "fn"]]:
+            if seg in body:
+                _conv_txt(qobj, body, seg, dk)
+
+        qstr = "select * from up"
+        qv = []
+        if qobj:
+            qk = []
+            for k, v in sorted(qobj.items()):
+                qk.append(k.split("\n")[0])
+                qv.append(v)
+
+            qstr = " and ".join(qk)
+            qstr = "select * from up where " + qstr
+
+        return self.run_query(vols, qstr, qv)
+
+    def run_query(self, vols, qstr, qv):
+        qv = tuple(qv)
+        self.log("qs: {} {}".format(qstr, repr(qv)))
+
+        ret = []
+        lim = 100
+        for (vtop, ptop, flags) in vols:
+            db = self.dbs.get(ptop)
+            if not db:
+                db = _open(ptop)
+                if not db:
+                    continue
+
+                self.dbs[ptop] = db
+                # self.log("idx /{} @ {} {}".format(vtop, ptop, flags))
+
+            c = db.execute(qstr, qv)
+            for _, ts, sz, rd, fn in c:
+                lim -= 1
+                if lim <= 0:
+                    break
+
+                rp = os.path.join(vtop, rd, fn).replace("\\", "/")
+                ret.append({"ts": int(ts), "sz": sz, "rp": rp})
+
+        return ret
+
+
+def _open(ptop):
+    db_path = os.path.join(ptop, ".hist", "up2k.db")
+    if os.path.exists(db_path):
+        return sqlite3.connect(db_path)
+
+
+def _conv_sz(q, body, k, sql):
+    if k in body:
+        q[sql] = int(float(body[k]) * 1024 * 1024)
+
+
+def _conv_dt(q, body, k, sql):
+    if k not in body:
+        return
+
+    v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ")
+    while "  " in v:
+        v = v.replace("  ", " ")
+
+    for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]:
+        try:
+            ts = datetime.strptime(v, fmt).timestamp()
+            break
+        except:
+            ts = None
+
+    if ts:
+        q[sql] = ts
+
+
+def _conv_txt(q, body, k, sql):
+    for v in body[k].split(" "):
+        inv = ""
+        if v.startswith("-"):
+            inv = "not"
+            v = v[1:]
+
+        if not v:
+            continue
+
+        head = "'%'||"
+        if v.startswith("^"):
+            head = ""
+            v = v[1:]
+
+        tail = "||'%'"
+        if v.endswith("$"):
+            tail = ""
+            v = v[:-1]
+
+        qk = "{} {} like {}?{}".format(sql, inv, head, tail)
+        q[qk + "\n" + v] = u8safe(v)

copyparty/up2k.py

@@ -1,10 +1,8 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-
 import re
 import os
-import sys
 import time
 import math
 import json
@@ -17,15 +15,24 @@ import threading
 from copy import deepcopy
 
 from .__init__ import WINDOWS
-from .util import Pebkac, Queue, fsdec, fsenc, sanitize_fn, ren_open, atomic_move
+from .util import (
+    Pebkac,
+    Queue,
+    ProgressPrinter,
+    fsdec,
+    fsenc,
+    sanitize_fn,
+    ren_open,
+    atomic_move,
+    w8b64enc,
+    w8b64dec,
+)
 
-HAVE_SQLITE3 = False
 try:
-    import sqlite3
-
     HAVE_SQLITE3 = True
+    import sqlite3
 except:
-    pass
+    HAVE_SQLITE3 = False
 
 
 class Up2k(object):
@@ -39,17 +46,24 @@ class Up2k(object):
     def __init__(self, broker):
         self.broker = broker
         self.args = broker.args
-        self.log = broker.log
+        self.log_func = broker.log
         self.persist = self.args.e2d
 
         # config
-        self.salt = "hunter2"  # TODO: config
+        self.salt = broker.args.salt
 
         # state
         self.mutex = threading.Lock()
         self.registry = {}
         self.db = {}
 
+        self.mem_db = None
+        if HAVE_SQLITE3:
+            # mojibake detector
+            self.mem_db = sqlite3.connect(":memory:", check_same_thread=False)
+            self.mem_db.execute(r"create table a (b text)")
+            self.mem_db.commit()
+
         if WINDOWS:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
@@ -66,8 +80,33 @@ class Up2k(object):
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$")
 
         if self.persist and not HAVE_SQLITE3:
-            m = "could not initialize sqlite3, will use in-memory registry only"
-            self.log("up2k", m)
+            self.log("could not initialize sqlite3, will use in-memory registry only")
+
+    def log(self, msg):
+        self.log_func("up2k", msg + "\033[K")
+
+    def w8enc(self, rd, fn):
+        ret = []
+        for k, v in [["d", rd], ["f", fn]]:
+            try:
+                self.mem_db.execute("select * from a where b = ?", (v,))
+                ret.append(v)
+            except:
+                ret.append("//" + w8b64enc(v))
+                # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+        return tuple(ret)
+
+    def w8dec(self, rd, fn):
+        ret = []
+        for k, v in [["d", rd], ["f", fn]]:
+            if v.startswith("//"):
+                ret.append(w8b64dec(v[2:]))
+                # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+            else:
+                ret.append(v)
+
+        return tuple(ret)
 
     def _vis_job_progress(self, job):
         perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
@@ -98,7 +137,7 @@ class Up2k(object):
 
                 m = "loaded snap {} |{}|".format(path, len(reg.keys()))
                 m = [m] + self._vis_reg_progress(reg)
-                self.log("up2k", "\n".join(m))
+                self.log("\n".join(m))
 
             self.registry[ptop] = reg
             if not self.persist or not HAVE_SQLITE3:
@@ -119,57 +158,87 @@ class Up2k(object):
                 self.db[ptop] = db
                 return db
             except Exception as ex:
-                m = "failed to open [{}]: {}".format(ptop, repr(ex))
-                self.log("up2k", m)
+                self.log("cannot use database at [{}]: {}".format(ptop, repr(ex)))
 
             return None
 
     def build_indexes(self, writeables):
         tops = [d.realpath for d in writeables]
+        self.pp = ProgressPrinter()
+        t0 = time.time()
         for top in tops:
             db = self.register_vpath(top)
-            if db:
-                # can be symlink so don't `and d.startswith(top)``
-                excl = set([d for d in tops if d != top])
-                dbw = [db, 0, time.time()]
-                self._build_dir(dbw, top, excl, top)
-                self._drop_lost(db, top)
-                if dbw[1]:
-                    self.log("up2k", "commit {} new files".format(dbw[1]))
+            if not db:
+                continue
 
-                db.commit()
+            self.pp.n = next(db.execute("select count(w) from up"))[0]
+            db_path = os.path.join(top, ".hist", "up2k.db")
+            sz0 = os.path.getsize(db_path) // 1024
+
+            # can be symlink so don't `and d.startswith(top)``
+            excl = set([d for d in tops if d != top])
+            dbw = [db, 0, time.time()]
+
+            n_add = self._build_dir(dbw, top, excl, top)
+            n_rm = self._drop_lost(db, top)
+            if dbw[1]:
+                self.log("commit {} new files".format(dbw[1]))
+
+            db.commit()
+            if n_add or n_rm:
+                db_path = os.path.join(top, ".hist", "up2k.db")
+                sz1 = os.path.getsize(db_path) // 1024
+                db.execute("vacuum")
+                sz2 = os.path.getsize(db_path) // 1024
+                msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format(
+                    n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2
+                )
+                self.log(msg)
+
+        self.pp.end = True
+        self.log("{} volumes in {:.2f} sec".format(len(tops), time.time() - t0))
 
     def _build_dir(self, dbw, top, excl, cdir):
         try:
             inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
         except Exception as ex:
-            self.log("up2k", "listdir: {} @ [{}]".format(repr(ex), cdir))
-            return
+            self.log("listdir: {} @ [{}]".format(repr(ex), cdir))
+            return 0
 
+        self.pp.msg = "a{} {}".format(self.pp.n, cdir)
         histdir = os.path.join(top, ".hist")
+        ret = 0
         for inode in inodes:
             abspath = os.path.join(cdir, inode)
             try:
                 inf = os.stat(fsenc(abspath))
             except Exception as ex:
-                self.log("up2k", "stat: {} @ [{}]".format(repr(ex), abspath))
+                self.log("stat: {} @ [{}]".format(repr(ex), abspath))
                 continue
 
             if stat.S_ISDIR(inf.st_mode):
                 if abspath in excl or abspath == histdir:
                     continue
-                # self.log("up2k", " dir: {}".format(abspath))
-                self._build_dir(dbw, top, excl, abspath)
+                # self.log(" dir: {}".format(abspath))
+                ret += self._build_dir(dbw, top, excl, abspath)
             else:
-                # self.log("up2k", "file: {}".format(abspath))
+                # self.log("file: {}".format(abspath))
                 rp = abspath[len(top) :].replace("\\", "/").strip("/")
-                c = dbw[0].execute("select * from up where rp = ?", (rp,))
+                rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
+                sql = "select * from up where rd = ? and fn = ?"
+                try:
+                    c = dbw[0].execute(sql, (rd, fn))
+                except:
+                    c = dbw[0].execute(sql, self.w8enc(rd, fn))
+
                 in_db = list(c.fetchall())
                 if in_db:
-                    _, dts, dsz, _ = in_db[0]
+                    self.pp.n -= 1
+                    _, dts, dsz, _, _ = in_db[0]
                     if len(in_db) > 1:
-                        m = "WARN: multiple entries: [{}] => [{}] ({})"
-                        self.log("up2k", m.format(top, rp, len(in_db)))
+                        m = "WARN: multiple entries: [{}] => [{}] |{}|\n{}"
+                        rep_db = "\n".join([repr(x) for x in in_db])
+                        self.log(m.format(top, rp, len(in_db), rep_db))
                         dts = -1
 
                     if dts == inf.st_mtime and dsz == inf.st_size:
@@ -178,68 +247,84 @@ class Up2k(object):
                     m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format(
                         top, rp, dts, inf.st_mtime, dsz, inf.st_size
                     )
-                    self.log("up2k", m)
-                    self.db_rm(dbw[0], rp)
+                    self.log(m)
+                    self.db_rm(dbw[0], rd, fn)
+                    ret += 1
                     dbw[1] += 1
                     in_db = None
 
-                self.log("up2k", "file: {}".format(abspath))
+                self.pp.msg = "a{} {}".format(self.pp.n, abspath)
+                if inf.st_size > 1024 * 1024:
+                    self.log("file: {}".format(abspath))
+
                 try:
                     hashes = self._hashlist_from_file(abspath)
                 except Exception as ex:
-                    self.log("up2k", "hash: {} @ [{}]".format(repr(ex), abspath))
+                    self.log("hash: {} @ [{}]".format(repr(ex), abspath))
                     continue
 
-                wark = self._wark_from_hashlist(inf.st_size, hashes)
-                self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size)
+                wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
+                self.db_add(dbw[0], wark, rd, fn, inf.st_mtime, inf.st_size)
                 dbw[1] += 1
+                ret += 1
                 td = time.time() - dbw[2]
-                if dbw[1] > 1024 or td > 60:
-                    self.log("up2k", "commit {} new files".format(dbw[1]))
+                if dbw[1] >= 4096 or td >= 60:
+                    self.log("commit {} new files".format(dbw[1]))
                     dbw[0].commit()
                     dbw[1] = 0
                     dbw[2] = time.time()
+        return ret
 
     def _drop_lost(self, db, top):
         rm = []
+        nchecked = 0
+        nfiles = next(db.execute("select count(w) from up"))[0]
         c = db.execute("select * from up")
-        for dwark, dts, dsz, drp in c:
-            abspath = os.path.join(top, drp)
+        for dwark, dts, dsz, drd, dfn in c:
+            nchecked += 1
+            if drd.startswith("//") or dfn.startswith("//"):
+                drd, dfn = self.w8dec(drd, dfn)
+
+            abspath = os.path.join(top, drd, dfn)
+            # almost zero overhead dw
+            self.pp.msg = "b{} {}".format(nfiles - nchecked, abspath)
             try:
                 if not os.path.exists(fsenc(abspath)):
-                    rm.append(drp)
+                    rm.append([drd, dfn])
             except Exception as ex:
-                self.log("up2k", "stat-rm: {} @ [{}]".format(repr(ex), abspath))
+                self.log("stat-rm: {} @ [{}]".format(repr(ex), abspath))
 
-        if not rm:
-            return
+        if rm:
+            self.log("forgetting {} deleted files".format(len(rm)))
+            for rd, fn in rm:
+                # self.log("{} / {}".format(rd, fn))
+                self.db_rm(db, rd, fn)
 
-        self.log("up2k", "forgetting {} deleted files".format(len(rm)))
-        for rp in rm:
-            self.db_rm(db, rp)
+        return len(rm)
 
     def _open_db(self, db_path):
+        existed = os.path.exists(db_path)
         conn = sqlite3.connect(db_path, check_same_thread=False)
         try:
-            c = conn.execute(r"select * from kv where k = 'sver'")
-            rows = c.fetchall()
-            if rows:
-                ver = rows[0][1]
-            else:
-                self.log("up2k", "WARN: no sver in kv, DB corrupt?")
-                ver = "unknown"
+            ver = self._read_ver(conn)
 
-            if ver == "1":
+            if ver == 1:
+                conn = self._upgrade_v1(conn, db_path)
+                ver = self._read_ver(conn)
+
+            if ver == 2:
                 try:
                     nfiles = next(conn.execute("select count(w) from up"))[0]
-                    self.log("up2k", "found DB at {} |{}|".format(db_path, nfiles))
+                    self.log("found DB at {} |{}|".format(db_path, nfiles))
                     return conn
                 except Exception as ex:
-                    m = "WARN: could not list files, DB corrupt?\n  " + repr(ex)
-                    self.log("up2k", m)
+                    self.log("WARN: could not list files, DB corrupt?\n  " + repr(ex))
+
+            if ver is not None:
+                self.log("REPLACING unsupported DB (v.{}) at {}".format(ver, db_path))
+            elif not existed:
+                raise Exception("whatever")
 
-            m = "REPLACING unsupported DB (v.{}) at {}".format(ver, db_path)
-            self.log("up2k", m)
             conn.close()
             os.unlink(db_path)
             conn = sqlite3.connect(db_path, check_same_thread=False)
@@ -247,17 +332,58 @@ class Up2k(object):
             pass
 
         # sqlite is variable-width only, no point in using char/nchar/varchar
+        self._create_v2(conn)
+        conn.commit()
+        self.log("created DB at {}".format(db_path))
+        return conn
+
+    def _read_ver(self, conn):
+        for tab in ["ki", "kv"]:
+            try:
+                c = conn.execute(r"select v from {} where k = 'sver'".format(tab))
+            except:
+                continue
+
+            rows = c.fetchall()
+            if rows:
+                return int(rows[0][0])
+
+    def _create_v2(self, conn):
         for cmd in [
-            r"create table kv (k text, v text)",
-            r"create table up (w text, mt int, sz int, rp text)",
-            r"insert into kv values ('sver', '1')",
+            r"create table ks (k text, v text)",
+            r"create table ki (k text, v int)",
+            r"create table up (w text, mt int, sz int, rd text, fn text)",
+            r"insert into ki values ('sver', 2)",
             r"create index up_w on up(w)",
+            r"create index up_rd on up(rd)",
+            r"create index up_fn on up(fn)",
         ]:
             conn.execute(cmd)
 
-        conn.commit()
-        self.log("up2k", "created DB at {}".format(db_path))
-        return conn
+    def _upgrade_v1(self, odb, db_path):
+        self.log("\033[33mupgrading v1 to v2:\033[0m {}".format(db_path))
+
+        npath = db_path + ".next"
+        if os.path.exists(npath):
+            os.unlink(npath)
+
+        ndb = sqlite3.connect(npath, check_same_thread=False)
+        self._create_v2(ndb)
+
+        c = odb.execute("select * from up")
+        for wark, ts, sz, rp in c:
+            rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
+            v = (wark, ts, sz, rd, fn)
+            ndb.execute("insert into up values (?,?,?,?,?)", v)
+
+        ndb.commit()
+        ndb.close()
+        odb.close()
+        bpath = db_path + ".bak.v1"
+        self.log("success; backup at: " + bpath)
+        atomic_move(db_path, bpath)
+        atomic_move(npath, db_path)
+        return sqlite3.connect(db_path, check_same_thread=False)
 
     def handle_json(self, cj):
         self.register_vpath(cj["ptop"])
@@ -271,19 +397,16 @@ class Up2k(object):
             reg = self.registry[cj["ptop"]]
             if db:
                 cur = db.execute(r"select * from up where w = ?", (wark,))
-                for _, dtime, dsize, dp_rel in cur:
-                    dp_abs = os.path.join(cj["ptop"], dp_rel).replace("\\", "/")
+                for _, dtime, dsize, dp_dir, dp_fn in cur:
+                    if dp_dir.startswith("//") or dp_fn.startswith("//"):
+                        dp_dir, dp_fn = self.w8dec(dp_dir, dp_fn)
+
+                    dp_abs = os.path.join(cj["ptop"], dp_dir, dp_fn).replace("\\", "/")
                     # relying on path.exists to return false on broken symlinks
                     if os.path.exists(fsenc(dp_abs)):
-                        try:
-                            prel, name = dp_rel.rsplit("/", 1)
-                        except:
-                            prel = ""
-                            name = dp_rel
-
                         job = {
-                            "name": name,
-                            "prel": prel,
+                            "name": dp_fn,
+                            "prel": dp_dir,
                             "vtop": cj["vtop"],
                             "ptop": cj["ptop"],
                             "flag": cj["flag"],
@@ -319,13 +442,13 @@ class Up2k(object):
                     vsrc = os.path.join(job["vtop"], job["prel"], job["name"])
                     vsrc = vsrc.replace("\\", "/")  # just for prints anyways
                     if job["need"]:
-                        self.log("up2k", "unfinished:\n  {0}\n  {1}".format(src, dst))
+                        self.log("unfinished:\n  {0}\n  {1}".format(src, dst))
                         err = "partial upload exists at a different location; please resume uploading here instead:\n"
-                        err += vsrc + " "
+                        err += "/" + vsrc + " "
                         raise Pebkac(400, err)
                     elif "nodupe" in job["flag"]:
-                        self.log("up2k", "dupe-reject:\n  {0}\n  {1}".format(src, dst))
-                        err = "upload rejected, file already exists:\n " + vsrc + " "
+                        self.log("dupe-reject:\n  {0}\n  {1}".format(src, dst))
+                        err = "upload rejected, file already exists:\n/" + vsrc + " "
                         raise Pebkac(400, err)
                     else:
                         # symlink to the client-provided name,
@@ -389,7 +512,7 @@ class Up2k(object):
 
     def _symlink(self, src, dst):
         # TODO store this in linktab so we never delete src if there are links to it
-        self.log("up2k", "linking dupe:\n  {0}\n  {1}".format(src, dst))
+        self.log("linking dupe:\n  {0}\n  {1}".format(src, dst))
         try:
             lsrc = src
             ldst = dst
@@ -412,7 +535,7 @@ class Up2k(object):
                     lsrc = "../" * (len(lsrc) - 1) + "/".join(lsrc)
             os.symlink(fsenc(lsrc), fsenc(ldst))
         except (AttributeError, OSError) as ex:
-            self.log("up2k", "cannot symlink; creating copy: " + repr(ex))
+            self.log("cannot symlink; creating copy: " + repr(ex))
             shutil.copy2(fsenc(src), fsenc(dst))
 
     def handle_chunk(self, ptop, wark, chash):
@@ -430,7 +553,7 @@ class Up2k(object):
 
         job["poke"] = time.time()
 
-        chunksize = self._get_chunksize(job["size"])
+        chunksize = up2k_chunksize(job["size"])
         ofs = [chunksize * x for x in nchunk]
 
         path = os.path.join(job["ptop"], job["prel"], job["tnam"])
@@ -463,33 +586,31 @@ class Up2k(object):
 
             db = self.db.get(job["ptop"], None)
             if db:
-                rp = os.path.join(job["prel"], job["name"]).replace("\\", "/")
-                self.db_rm(db, rp)
-                self.db_add(db, job["wark"], rp, job["lmod"], job["size"])
+                j = job
+                self.db_rm(db, j["prel"], j["name"])
+                self.db_add(db, j["wark"], j["prel"], j["name"], j["lmod"], j["size"])
                 db.commit()
                 del self.registry[ptop][wark]
                 # in-memory registry is reserved for unfinished uploads
 
             return ret, dst
 
-    def _get_chunksize(self, filesize):
-        chunksize = 1024 * 1024
-        stepsize = 512 * 1024
-        while True:
-            for mul in [1, 2]:
-                nchunks = math.ceil(filesize * 1.0 / chunksize)
-                if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
-                    return chunksize
+    def db_rm(self, db, rd, fn):
+        sql = "delete from up where rd = ? and fn = ?"
+        try:
+            db.execute(sql, (rd, fn))
+        except:
+            db.execute(sql, self.w8enc(rd, fn))
 
-                chunksize += stepsize
-                stepsize *= mul
-
-    def db_rm(self, db, rp):
-        db.execute("delete from up where rp = ?", (rp,))
-
-    def db_add(self, db, wark, rp, ts, sz):
-        v = (wark, ts, sz, rp)
-        db.execute("insert into up values (?,?,?,?)", v)
+    def db_add(self, db, wark, rd, fn, ts, sz):
+        sql = "insert into up values (?,?,?,?,?)"
+        v = (wark, ts, sz, rd, fn)
+        try:
+            db.execute(sql, v)
+        except:
+            rd, fn = self.w8enc(rd, fn)
+            v = (wark, ts, sz, rd, fn)
+            db.execute(sql, v)
 
     def _get_wark(self, cj):
         if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024:  # 16TiB
@@ -507,36 +628,17 @@ class Up2k(object):
         except:
             cj["lmod"] = int(time.time())
 
-        wark = self._wark_from_hashlist(cj["size"], cj["hash"])
+        wark = up2k_wark_from_hashlist(self.salt, cj["size"], cj["hash"])
         return wark
 
-    def _wark_from_hashlist(self, filesize, hashes):
-        """ server-reproducible file identifier, independent of name or location """
-        ident = [self.salt, str(filesize)]
-        ident.extend(hashes)
-        ident = "\n".join(ident)
-
-        hasher = hashlib.sha512()
-        hasher.update(ident.encode("utf-8"))
-        digest = hasher.digest()[:32]
-
-        wark = base64.urlsafe_b64encode(digest)
-        return wark.decode("utf-8").rstrip("=")
-
     def _hashlist_from_file(self, path):
         fsz = os.path.getsize(path)
-        csz = self._get_chunksize(fsz)
+        csz = up2k_chunksize(fsz)
         ret = []
         last_print = time.time()
         with open(path, "rb", 512 * 1024) as f:
             while fsz > 0:
-                now = time.time()
-                td = now - last_print
-                if td >= 0.1:
-                    last_print = now
-                    msg = " {} MB   \r".format(int(fsz / 1024 / 1024))
-                    print(msg, end="", file=sys.stderr)
-
+                self.pp.msg = msg = "{} MB".format(int(fsz / 1024 / 1024))
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
                 fsz -= rem
@@ -574,14 +676,14 @@ class Up2k(object):
             while not self.lastmod_q.empty():
                 ready.append(self.lastmod_q.get())
 
-            # self.log("lmod", "got {}".format(len(ready)))
+            # self.log("lmod: got {}".format(len(ready)))
             time.sleep(5)
             for path, times in ready:
-                self.log("lmod", "setting times {} on {}".format(times, path))
+                self.log("lmod: setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:
-                    self.log("lmod", "failed to utime ({}, {})".format(path, times))
+                    self.log("lmod: failed to utime ({}, {})".format(path, times))
 
     def _snapshot(self):
         persist_interval = 30  # persist unfinished uploads index every 30 sec
@@ -599,7 +701,7 @@ class Up2k(object):
         if rm:
             m = "dropping {} abandoned uploads in {}".format(len(rm), k)
             vis = [self._vis_job_progress(x) for x in rm]
-            self.log("up2k", "\n".join([m] + vis))
+            self.log("\n".join([m] + vis))
             for job in rm:
                 del reg[job["wark"]]
                 try:
@@ -628,6 +730,11 @@ class Up2k(object):
         if etag == prev.get(k, None):
             return
 
+        try:
+            os.mkdir(os.path.join(k, ".hist"))
+        except:
+            pass
+
         path2 = "{}.{}".format(path, os.getpid())
         j = json.dumps(reg, indent=2, sort_keys=True).encode("utf-8")
         with gzip.GzipFile(path2, "wb") as f:
@@ -635,5 +742,32 @@ class Up2k(object):
 
         atomic_move(path2, path)
 
-        self.log("up2k", "snap: {} |{}|".format(path, len(reg.keys())))
+        self.log("snap: {} |{}|".format(path, len(reg.keys())))
         prev[k] = etag
+
+
+def up2k_chunksize(filesize):
+    chunksize = 1024 * 1024
+    stepsize = 512 * 1024
+    while True:
+        for mul in [1, 2]:
+            nchunks = math.ceil(filesize * 1.0 / chunksize)
+            if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
+                return chunksize
+
+            chunksize += stepsize
+            stepsize *= mul
+
+
+def up2k_wark_from_hashlist(salt, filesize, hashes):
+    """ server-reproducible file identifier, independent of name or location """
+    ident = [salt, str(filesize)]
+    ident.extend(hashes)
+    ident = "\n".join(ident)
+
+    hasher = hashlib.sha512()
+    hasher.update(ident.encode("utf-8"))
+    digest = hasher.digest()[:32]
+
+    wark = base64.urlsafe_b64encode(digest)
+    return wark.decode("utf-8").rstrip("=")

copyparty/util.py

@@ -99,6 +99,39 @@ class Unrecv(object):
         self.buf = buf + self.buf
 
 
+class ProgressPrinter(threading.Thread):
+    """
+    periodically print progress info without linefeeds
+    """
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.msg = None
+        self.end = False
+        self.start()
+
+    def run(self):
+        msg = None
+        while not self.end:
+            time.sleep(0.1)
+            if msg == self.msg or self.end:
+                continue
+
+            msg = self.msg
+            m = " {}\033[K\r".format(msg)
+            try:
+                print(m, end="")
+            except UnicodeEncodeError:
+                try:
+                    print(m.encode("utf-8", "replace").decode(), end="")
+                except:
+                    print(m.encode("ascii", "replace").decode(), end="")
+
+        print("\033[K", end="")
+        sys.stdout.flush()  # necessary on win10 even w/ stderr btw
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -146,7 +179,7 @@ def ren_open(fname, *args, **kwargs):
 
         except OSError as ex_:
             ex = ex_
-            if ex.errno != 36:
+            if ex.errno not in [36, 63] and (not WINDOWS or ex.errno != 22):
                 raise
 
         if not b64:
@@ -480,6 +513,13 @@ def sanitize_fn(fn):
     return fn.strip()
 
 
+def u8safe(txt):
+    try:
+        return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
+    except:
+        return txt.encode("utf-8", "replace").decode("utf-8", "replace")
+
+
 def exclude_dotfiles(filepaths):
     for fpath in filepaths:
         if not fpath.split("/")[-1].startswith("."):
@@ -536,6 +576,16 @@ def w8enc(txt):
     return txt.encode(FS_ENCODING, "surrogateescape")
 
 
+def w8b64dec(txt):
+    """decodes base64(filesystem-bytes) to wtf8"""
+    return w8dec(base64.urlsafe_b64decode(txt.encode("ascii")))
+
+
+def w8b64enc(txt):
+    """encodes wtf8 to base64(filesystem-bytes)"""
+    return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii")
+
+
 if PY2 and WINDOWS:
     # moonrunes become \x3f with bytestrings,
     # losing mojibake support is worth

copyparty/web/browser.css

@@ -39,15 +39,27 @@ body {
 	margin: 1.3em 0 0 0;
 	font-size: 1.4em;
 }
+#path #entree {
+	margin-left: -.7em;
+}
+#treetab {
+	display: none;
+}
 #files {
 	border-collapse: collapse;
 	margin-top: 2em;
+	z-index: 1;
+	position: relative;
 }
 #files tbody a {
 	display: block;
 	padding: .3em 0;
 }
-a {
+#files[ts] tbody div a {
+	color: #f5a;
+}
+a,
+#files[ts] tbody div a:last-child {
 	color: #fc5;
 	padding: .2em;
 	text-decoration: none;
@@ -142,10 +154,12 @@ a {
 #srv_info span {
 	color: #fff;
 }
-a.play {
+#files tbody a.play {
 	color: #e70;
+	padding: .2em;
+	margin: -.2em;
 }
-a.play.act {
+#files tbody a.play.act {
 	color: #af0;
 }
 #blocked {
@@ -156,7 +170,7 @@ a.play.act {
 	height: 100%;
 	background: #333;
 	font-size: 2.5em;
-	z-index:99;
+	z-index: 99;
 }
 #blk_play,
 #blk_abrt {
@@ -190,6 +204,7 @@ a.play.act {
 	bottom: -6em;
 	height: 6em;
 	width: 100%;
+	z-index: 3;
 	transition: bottom 0.15s;
 }
 #widget.open {
@@ -214,6 +229,9 @@ a.play.act {
 	75% {cursor: url(/.cpr/dd/5.png), pointer}
 	85% {cursor: url(/.cpr/dd/1.png), pointer}
 }
+@keyframes spin {
+	100% {transform: rotate(360deg)}
+}
 #wtoggle {
 	position: absolute;
 	top: -1.2em;
@@ -273,3 +291,201 @@ a.play.act {
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
+
+
+
+.opview {
+	display: none;
+}
+.opview.act {
+	display: block;
+}
+#ops a {
+	color: #fc5;
+	font-size: 1.5em;
+	padding: .25em .3em;
+	margin: 0;
+	outline: none;
+}
+#ops a.act {
+	background: #281838;
+	border-radius: 0 0 .2em .2em;
+	border-bottom: .3em solid #d90;
+	box-shadow: 0 -.15em .2em #000 inset;
+	padding-bottom: .3em;
+}
+#ops i {
+	font-size: 1.5em;
+}
+#ops i:before {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #01a7e1;
+	position: relative;
+}
+#ops i:after {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #ff3f1a;
+	margin-left: -.35em;
+	font-size: 1.05em;
+}
+#ops,
+.opbox {
+	border: 1px solid #3a3a3a;
+	box-shadow: 0 0 1em #222 inset;
+}
+#ops {
+	background: #333;
+	margin: 1.7em 1.5em 0 1.5em;
+	padding: .3em .6em;
+	border-radius: .3em;
+	border-width: .15em 0;
+}
+.opbox {
+	background: #2d2d2d;
+	margin: 1.5em 0 0 0;
+	padding: .5em;
+	border-radius: 0 1em 1em 0;
+	border-width: .15em .3em .3em 0;
+	max-width: 40em;
+}
+.opbox input {
+	margin: .5em;
+}
+.opview input[type=text] {
+	color: #fff;
+	background: #383838;
+	border: none;
+	box-shadow: 0 0 .3em #222;
+	border-bottom: 1px solid #fc5;
+	border-radius: .2em;
+	padding: .2em .3em;
+}
+input[type="checkbox"]+label {
+	color: #f5a;
+}
+input[type="checkbox"]:checked+label {
+	color: #fc5;
+}
+
+
+
+#op_search table {
+	border: 1px solid #3a3a3a;
+	box-shadow: 0 0 1em #222 inset;
+	background: #2d2d2d;
+	border-radius: .4em;
+	margin: 1.4em;
+	margin-bottom: 0;
+	padding: 0 .5em .5em 0;
+}
+#srch_form td {
+	padding: .6em .6em;
+}
+#op_search input {
+	margin: 0;
+}
+#srch_q {
+	white-space: pre;
+}
+#files td div span {
+	color: #fff;
+	padding: 0 .4em;
+	font-weight: bold;
+	font-style: italic;
+}
+#files td div a:hover {
+	background: #444;
+	color: #fff;
+}
+#files td div a {
+	display: table-cell;
+	white-space: nowrap;
+}
+#files td div a:last-child {
+	width: 100%;
+}
+#files td div {
+	display: table;
+	border-collapse: collapse;
+	width: 100%;
+}
+#files td div a:last-child {
+	width: 100%;
+}
+#tree,
+#treefiles {
+	vertical-align: top;
+}
+#tree {
+	padding-top: 2em;
+}
+#detree {
+	padding: .3em .5em;
+	font-size: 1.5em;
+	display: inline-block;
+	min-width: 12em;
+	width: 100%;
+}
+#treefiles #files tbody {
+	border-radius: 0 .7em 0 .7em;
+}
+#treefiles #files thead th:nth-child(1) {
+	border-radius: .7em 0 0 0;
+}
+#tree ul,
+#tree li {
+	padding: 0;
+	margin: 0;
+}
+#tree ul {
+	border-left: .2em solid #444;
+}
+#tree li {
+	margin-left: 1em;
+	list-style: none;
+	white-space: nowrap;
+}
+#tree a.hl {
+	color: #400;
+	background: #fc4;
+	border-radius: .3em;
+	text-shadow: none;
+}
+#tree a {
+	display: inline-block;
+}
+#tree a+a {
+	width: calc(100% - 2em);
+	background: #333;
+}
+#tree a+a:hover {
+	background: #222;
+	color: #fff;
+}
+#treeul {
+	position: relative;
+	overflow: hidden;
+	left: -1.7em;
+}
+#treeul:hover {
+	z-index: 2;
+	overflow: visible;
+}
+#treeul:hover a+a {
+	width: auto;
+	min-width: calc(100% - 2em);
+}
+#treeul a:first-child {
+	font-family: monospace, monospace;
+}
+.dumb_loader_thing {
+	display: inline-block;
+	margin: 1em .3em 1em 1em;
+	padding: 0 1.2em 0 0;
+	font-size: 4em;
+	animation: spin 1s linear infinite;
+	position: absolute;
+	z-index: 9;
+}

copyparty/web/browser.html

@@ -7,26 +7,48 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
-    {%- if can_upload %}
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
-    {%- endif %}
 </head>
 
 <body>
-    {%- if can_upload %}
+    <div id="ops">
+        <a href="#" data-dest="">---</a>
+        <a href="#" data-perm="read" data-dest="search">🔎</a>
+        {%- if have_up2k_idx %}
+        <a href="#" data-dest="up2k">🚀</a>
+        {%- else %}
+        <a href="#" data-perm="write" data-dest="up2k">🚀</a>
+        {%- endif %}
+        <a href="#" data-perm="write" data-dest="bup">🎈</a>
+        <a href="#" data-perm="write" data-dest="mkdir">📂</a>
+        <a href="#" data-perm="write" data-dest="new_md">📝</a>
+        <a href="#" data-perm="write" data-dest="msg">📟</a>
+    </div>
+
+    <div id="op_search" class="opview">
+        <table id="srch_form"></table>
+        <div id="srch_q"></div>
+    </div>
     {%- include 'upload.html' %}
-    {%- endif %}
     
     <h1 id="path">
+        <a href="#" id="entree">🌲</a>
         {%- for n in vpnodes %}
         <a href="/{{ n[0] }}">{{ n[1] }}</a>
         {%- endfor %}
     </h1>
     
-    {%- if can_read %}
-    {%- if prologue %}
-    <div id="pro" class="logue">{{ prologue }}</div>
-    {%- endif %}
+    <div id="pro" class="logue">{{ logues[0] }}</div>
+
+    <table id="treetab">
+        <tr>
+            <td id="tree">
+                <a href="#" id="detree">🍞...</a>
+                <ul id="treeul"></ul>
+            </td>
+            <td id="treefiles"></td>
+        </tr>
+    </table>
 
     <table id="files">
         <thead>
@@ -41,16 +63,13 @@
         <tbody>
 
 {%- for f in files %}
-<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td><td>{{ f[5] }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
         </tbody>
     </table>
     
-    {%- if epilogue %}
-    <div id="epi" class="logue">{{ epilogue }}</div>
-    {%- endif %}
-    {%- endif %}
+    <div id="epi" class="logue">{{ logues[1] }}</div>
 
     <h2><a href="?h">control-panel</a></h2>
 
@@ -69,14 +88,11 @@
     </div>
     
     <script src="/.cpr/util.js{{ ts }}"></script>
-
-    {%- if can_read %}
     <script src="/.cpr/browser.js{{ ts }}"></script>
-    {%- endif %}
-    
-    {%- if can_upload %}
     <script src="/.cpr/up2k.js{{ ts }}"></script>
-    {%- endif %}
+    <script>
+        apply_perms({{ perms }});
+    </script>
 </body>
 
 </html>

copyparty/web/browser.js

@@ -8,6 +8,8 @@ function dbg(msg) {
 
 function ev(e) {
 	e = e || window.event;
+	if (!e)
+		return;
 
 	if (e.preventDefault)
 		e.preventDefault()
@@ -23,7 +25,7 @@ makeSortable(ebi('files'));
 
 
 // extract songs + add play column
-var mp = (function () {
+function init_mp() {
 	var tracks = [];
 	var ret = {
 		'au': null,
@@ -37,7 +39,8 @@ var mp = (function () {
 	var trs = ebi('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr');
 	for (var a = 0, aa = trs.length; a < aa; a++) {
 		var tds = trs[a].getElementsByTagName('td');
-		var link = tds[1].getElementsByTagName('a')[0];
+		var link = tds[1].getElementsByTagName('a');
+		link = link[link.length - 1];
 		var url = link.getAttribute('href');
 
 		var m = re_audio.exec(url);
@@ -71,7 +74,8 @@ var mp = (function () {
 	};
 
 	return ret;
-})();
+}
+var mp = init_mp();
 
 
 // toggle player widget
@@ -466,7 +470,13 @@ function play(tid, call_depth) {
 
 		var o = ebi(oid);
 		o.setAttribute('id', 'thx_js');
-		location.hash = oid;
+		if (window.history && history.replaceState) {
+			var nurl = (document.location + '').split('#')[0] + '#' + oid;
+			history.replaceState(ebi('files').tBodies[0].innerHTML, nurl, nurl);
+		}
+		else {
+			document.location.hash = oid;
+		}
 		o.setAttribute('id', oid);
 
 		pbar.drawbuf();
@@ -561,3 +571,459 @@ function autoplay_blocked() {
 
 
 //widget.open();
+
+
+// search
+(function () {
+	var sconf = [
+		["size",
+			["szl", "sz_min", "minimum MiB", ""],
+			["szu", "sz_max", "maximum MiB", ""]
+		],
+		["date",
+			["dtl", "dt_min", "min. iso8601", ""],
+			["dtu", "dt_max", "max. iso8601", ""]
+		],
+		["path",
+			["path", "path", "path contains &nbsp; (space-separated)", "46"]
+		],
+		["name",
+			["name", "name", "name contains &nbsp; (negate with -nope)", "46"]
+		]
+	];
+	var html = [];
+	var orig_html = null;
+	for (var a = 0; a < sconf.length; a++) {
+		html.push('<tr><td><br />' + sconf[a][0] + '</td>');
+		for (var b = 1; b < 3; b++) {
+			var hn = "srch_" + sconf[a][b][0];
+			var csp = (sconf[a].length == 2) ? 2 : 1;
+			html.push(
+				'<td colspan="' + csp + '"><input id="' + hn + 'c" type="checkbox">\n' +
+				'<label for="' + hn + 'c">' + sconf[a][b][2] + '</label>\n' +
+				'<br /><input id="' + hn + 'v" type="text" size="' + sconf[a][b][3] +
+				'" name="' + sconf[a][b][1] + '" /></td>');
+			if (csp == 2)
+				break;
+		}
+		html.push('</tr>');
+	}
+	ebi('srch_form').innerHTML = html.join('\n');
+
+	var o = document.querySelectorAll('#op_search input[type="text"]');
+	for (var a = 0; a < o.length; a++) {
+		o[a].oninput = ev_search_input;
+	}
+
+	var search_timeout;
+
+	function ev_search_input() {
+		var v = this.value;
+		var chk = ebi(this.getAttribute('id').slice(0, -1) + 'c');
+		chk.checked = ((v + '').length > 0);
+		clearTimeout(search_timeout);
+		search_timeout = setTimeout(do_search, 100);
+	}
+
+	function do_search() {
+		clearTimeout(search_timeout);
+		var params = {};
+		var o = document.querySelectorAll('#op_search input[type="text"]');
+		for (var a = 0; a < o.length; a++) {
+			var chk = ebi(o[a].getAttribute('id').slice(0, -1) + 'c');
+			if (!chk.checked)
+				continue;
+
+			params[o[a].getAttribute('name')] = o[a].value;
+		}
+		// ebi('srch_q').textContent = JSON.stringify(params, null, 4);
+		var xhr = new XMLHttpRequest();
+		xhr.open('POST', '/?srch', true);
+		xhr.onreadystatechange = xhr_search_results;
+		xhr.ts = new Date().getTime();
+		xhr.send(JSON.stringify(params));
+	}
+
+	function xhr_search_results() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			alert('ah fug\n' + this.status + ": " + this.responseText);
+			return;
+		}
+
+		var ofiles = ebi('files');
+		if (ofiles.getAttribute('ts') > this.ts)
+			return;
+
+		ebi('path').style.display = 'none';
+		ebi('tree').style.display = 'none';
+
+		var html = ['<tr><td>-</td><td colspan="4"><a href="#" id="unsearch">close search results</a></td></tr>'];
+		var res = JSON.parse(this.responseText);
+		for (var a = 0; a < res.length; a++) {
+			var r = res[a],
+				ts = parseInt(r.ts),
+				sz = esc(r.sz + ''),
+				rp = esc(r.rp + ''),
+				ext = rp.lastIndexOf('.') > 0 ? rp.split('.').slice(-1)[0] : '%',
+				links = linksplit(rp);
+
+			if (ext.length > 8)
+				ext = '%';
+
+			links = links.join('');
+			html.push('<tr><td>-</td><td><div>' + links + '</div></td><td>' + sz +
+				'</td><td>' + ext + '</td><td>' + unix2iso(ts) + '</td></tr>');
+		}
+
+		if (!orig_html)
+			orig_html = ebi('files').tBodies[0].innerHTML;
+
+		ofiles.tBodies[0].innerHTML = html.join('\n');
+		ofiles.setAttribute("ts", this.ts);
+		reload_browser();
+
+		ebi('unsearch').onclick = unsearch;
+	}
+
+	function unsearch(e) {
+		ev(e);
+		ebi('path').style.display = 'inline-block';
+		ebi('tree').style.display = 'block';
+		ebi('files').tBodies[0].innerHTML = orig_html;
+		orig_html = null;
+		reload_browser();
+	}
+})();
+
+
+// tree
+(function () {
+	var treedata = null;
+
+	function entree(e) {
+		ev(e);
+		ebi('path').style.display = 'none';
+
+		var treetab = ebi('treetab');
+		var treefiles = ebi('treefiles');
+
+		treetab.style.display = 'table';
+
+		treefiles.appendChild(ebi('pro'));
+		treefiles.appendChild(ebi('files'));
+		treefiles.appendChild(ebi('epi'));
+
+		localStorage.setItem('entreed', 'tree');
+		get_tree("", get_vpath());
+	}
+
+	function get_tree(top, dst) {
+		var xhr = new XMLHttpRequest();
+		xhr.top = top;
+		xhr.dst = dst;
+		xhr.open('GET', dst + '?tree=' + top, true);
+		xhr.onreadystatechange = recvtree;
+		xhr.send();
+		enspin('#tree');
+	}
+
+	function recvtree() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			alert('ah fug\n' + this.status + ": " + this.responseText);
+			return;
+		}
+
+		var top = this.top == '.' ? this.dst : this.top,
+			name = top.split('/').slice(-2)[0],
+			rtop = top.replace(/^\/+/, "");
+
+		try {
+			var res = JSON.parse(this.responseText);
+		}
+		catch (ex) {
+			return;
+		}
+		var html = parsetree(res, rtop);
+		if (!this.top) {
+			html = '<li><a href="#">-</a><a href="/">[root]</a>\n<ul>' + html;
+			if (!ebi('treeul').getElementsByTagName('li').length)
+				ebi('treeul').innerHTML = html + '</ul></li>';
+		}
+		else {
+			html = '<a href="#">-</a><a href="' +
+				esc(top) + '">' + esc(name) +
+				"</a>\n<ul>\n" + html + "</ul>";
+
+			var links = document.querySelectorAll('#tree a+a');
+			for (var a = 0, aa = links.length; a < aa; a++) {
+				if (links[a].getAttribute('href') == top) {
+					var o = links[a].parentNode;
+					if (!o.getElementsByTagName('li').length)
+						o.innerHTML = html;
+					//else
+					//	links[a].previousSibling.textContent = '-';
+				}
+			}
+		}
+		document.querySelector('#treeul>li>a+a').textContent = '[root]';
+		despin('#tree');
+		reload_tree();
+
+		var q = '#tree';
+		var nq = 0;
+		while (true) {
+			nq++;
+			q += '>ul>li';
+			if (!document.querySelector(q))
+				break;
+		}
+		ebi('treeul').style.width = (24 + nq) + 'em';
+	}
+
+	function reload_tree() {
+		var cdir = get_vpath();
+		var links = document.querySelectorAll('#tree a+a');
+		for (var a = 0, aa = links.length; a < aa; a++) {
+			var href = links[a].getAttribute('href');
+			links[a].setAttribute('class', href == cdir ? 'hl' : '');
+			links[a].onclick = treego;
+		}
+		links = document.querySelectorAll('#tree li>a:first-child');
+		for (var a = 0, aa = links.length; a < aa; a++) {
+			links[a].setAttribute('dst', links[a].nextSibling.getAttribute('href'));
+			links[a].onclick = treegrow;
+		}
+	}
+
+	function treego(e) {
+		ev(e);
+		if (this.getAttribute('class') == 'hl' &&
+			this.previousSibling.textContent == '-') {
+			treegrow.call(this.previousSibling, e);
+			return;
+		}
+		var xhr = new XMLHttpRequest();
+		xhr.top = this.getAttribute('href');
+		xhr.open('GET', xhr.top + '?ls', true);
+		xhr.onreadystatechange = recvls;
+		xhr.send();
+		get_tree('.', xhr.top);
+		enspin('#files');
+	}
+
+	function treegrow(e) {
+		ev(e);
+		if (this.textContent == '-') {
+			while (this.nextSibling.nextSibling) {
+				var rm = this.nextSibling.nextSibling;
+				rm.parentNode.removeChild(rm);
+			}
+			this.textContent = '+';
+			return;
+		}
+		var dst = this.getAttribute('dst');
+		get_tree('.', dst);
+	}
+
+	function recvls() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			alert('ah fug\n' + this.status + ": " + this.responseText);
+			return;
+		}
+
+		try {
+			var res = JSON.parse(this.responseText);
+		}
+		catch (ex) {
+			window.location = this.top;
+			return;
+		}
+
+		ebi('srv_info').innerHTML = '<span>' + res.srvinf + '</span>';
+		var nodes = res.dirs.concat(res.files);
+		var top = this.top;
+		var html = [];
+		for (var a = 0; a < nodes.length; a++) {
+			var r = nodes[a],
+				ln = '<tr><td>' + r.lead + '</td><td><a href="' +
+					top + r.href + '">' + esc(decodeURIComponent(r.href)) + '</a>';
+
+			ln = [ln, r.sz, r.ext, unix2iso(r.ts)].join('</td><td>');
+			html.push(ln + '</td></tr>');
+		}
+		html = html.join('\n');
+		ebi('files').tBodies[0].innerHTML = html;
+		history.pushState(html, this.top, this.top);
+		apply_perms(res.perms);
+		despin('#files');
+
+		ebi('pro').innerHTML = res.logues ? res.logues[0] || "" : "";
+		ebi('epi').innerHTML = res.logues ? res.logues[1] || "" : "";
+
+		reload_tree();
+		reload_browser();
+	}
+
+	function parsetree(res, top) {
+		var ret = '';
+		for (var a = 0; a < res.a.length; a++) {
+			if (res.a[a] !== '')
+				res['k' + res.a[a]] = 0;
+		}
+		delete res['a'];
+		var keys = Object.keys(res);
+		keys.sort();
+		for (var a = 0; a < keys.length; a++) {
+			var kk = keys[a],
+				k = kk.slice(1),
+				url = '/' + (top ? top + k : k) + '/',
+				ek = esc(k),
+				sym = res[kk] ? '-' : '+',
+				link = '<a href="#">' + sym + '</a><a href="' +
+					esc(url) + '">' + ek + '</a>';
+
+			if (res[kk]) {
+				var subtree = parsetree(res[kk], url.slice(1));
+				ret += '<li>' + link + '\n<ul>\n' + subtree + '</ul></li>\n';
+			}
+			else {
+				ret += '<li>' + link + '</li>\n';
+			}
+		}
+		return ret;
+	}
+
+	function detree(e) {
+		ev(e);
+		var treetab = ebi('treetab');
+
+		treetab.parentNode.insertBefore(ebi('pro'), treetab);
+		treetab.parentNode.insertBefore(ebi('files'), treetab.nextSibling);
+		treetab.parentNode.insertBefore(ebi('epi'), ebi('files').nextSibling);
+
+		ebi('path').style.display = 'inline-block';
+		treetab.style.display = 'none';
+
+		localStorage.setItem('entreed', 'na');
+	}
+
+	ebi('entree').onclick = entree;
+	ebi('detree').onclick = detree;
+	if (window.localStorage && localStorage.getItem('entreed') == 'tree')
+		entree();
+
+	window.onpopstate = function (e) {
+		console.log(e.url + ' ,, ' + ((e.state + '').slice(0, 64)));
+		if (e.state) {
+			ebi('files').tBodies[0].innerHTML = e.state;
+			reload_tree();
+			reload_browser();
+		}
+	};
+
+	if (window.history && history.pushState) {
+		var u = get_vpath();
+		history.replaceState(ebi('files').tBodies[0].innerHTML, u, u);
+	}
+})();
+
+
+function enspin(sel) {
+	despin(sel);
+	var d = document.createElement('div');
+	d.setAttribute('class', 'dumb_loader_thing');
+	d.innerHTML = '🌲';
+	var tgt = document.querySelector(sel);
+	tgt.insertBefore(d, tgt.childNodes[0]);
+}
+
+
+function despin(sel) {
+	var o = document.querySelectorAll(sel + '>.dumb_loader_thing');
+	for (var a = o.length - 1; a >= 0; a--)
+		o[a].parentNode.removeChild(o[a]);
+}
+
+
+function apply_perms(perms) {
+	perms = perms || [];
+
+	var o = document.querySelectorAll('#ops>a[data-perm]');
+	for (var a = 0; a < o.length; a++)
+		o[a].style.display = 'none';
+
+	for (var a = 0; a < perms.length; a++) {
+		o = document.querySelectorAll('#ops>a[data-perm="' + perms[a] + '"]');
+		for (var b = 0; b < o.length; b++)
+			o[b].style.display = 'inline';
+	}
+
+	var act = document.querySelector('#ops>a.act');
+	if (act) {
+		var areq = act.getAttribute('data-perm');
+		if (areq && !has(perms, areq))
+			goto();
+	}
+
+	document.body.setAttribute('perms', perms.join(' '));
+
+	var have_write = has(perms, "write");
+	var tds = document.querySelectorAll('#u2conf td');
+	for (var a = 0; a < tds.length; a++) {
+		tds[a].style.display =
+			(have_write || tds[a].getAttribute('data-perm') == 'read') ?
+				'table-cell' : 'none';
+	}
+
+	if (window['up2k'])
+		up2k.set_fsearch();
+}
+
+
+function reload_browser(not_mp) {
+	makeSortable(ebi('files'));
+
+	var parts = get_vpath().split('/');
+	var rm = document.querySelectorAll('#path>a+a+a');
+	for (a = rm.length - 1; a >= 0; a--)
+		rm[a].parentNode.removeChild(rm[a]);
+
+	var link = '/';
+	for (var a = 1; a < parts.length - 1; a++) {
+		link += parts[a] + '/';
+		var o = document.createElement('a');
+		o.setAttribute('href', link);
+		o.innerHTML = parts[a];
+		ebi('path').appendChild(o);
+	}
+
+	var oo = document.querySelectorAll('#files>tbody>tr>td:nth-child(3)');
+	for (var a = 0, aa = oo.length; a < aa; a++) {
+		var sz = oo[a].textContent.replace(/ /g, ""),
+			hsz = sz.replace(/\B(?=(\d{3})+(?!\d))/g, " ");
+
+		oo[a].textContent = hsz;
+	}
+
+	if (!not_mp) {
+		if (mp && mp.au) {
+			mp.au.pause();
+			mp.au = null;
+		}
+		widget.close();
+		mp = init_mp();
+	}
+
+	if (window['up2k'])
+		up2k.set_fsearch();
+}
+reload_browser(true);

copyparty/web/md2.css

@@ -124,5 +124,3 @@ html.dark #toast {
     transition: opacity 0.2s ease-in-out;
     opacity: 1;
 }
-
-# mt {opacity: .5;top:1px}

copyparty/web/up2k.js

@@ -3,51 +3,6 @@
 window.onerror = vis_exh;
 
 
-(function () {
-    var ops = document.querySelectorAll('#ops>a');
-    for (var a = 0; a < ops.length; a++) {
-        ops[a].onclick = opclick;
-    }
-})();
-
-
-function opclick(ev) {
-    if (ev) //ie
-        ev.preventDefault();
-
-    var dest = this.getAttribute('data-dest');
-    goto(dest);
-
-    // writing a blank value makes ie8 segfault w
-    if (window.localStorage)
-        localStorage.setItem('opmode', dest || '.');
-
-    var input = document.querySelector('.opview.act input:not([type="hidden"])')
-    if (input)
-        input.focus();
-}
-
-
-function goto(dest) {
-    var obj = document.querySelectorAll('.opview.act');
-    for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
-
-    obj = document.querySelectorAll('#ops>a');
-    for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
-
-    if (dest) {
-        ebi('op_' + dest).classList.add('act');
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
-
-        var fn = window['goto_' + dest];
-        if (fn)
-            fn();
-    }
-}
-
-
 function goto_up2k() {
     if (up2k === false)
         return goto('bup');
@@ -59,17 +14,6 @@ function goto_up2k() {
 }
 
 
-(function () {
-    goto();
-    if (window.localStorage) {
-        var op = localStorage.getItem('opmode');
-        if (op !== null && op !== '.')
-            goto(op);
-    }
-    ebi('ops').style.display = 'block';
-})();
-
-
 // chrome requires https to use crypto.subtle,
 // usually it's undefined but some chromes throw on invoke
 var up2k = null;
@@ -207,10 +151,6 @@ function up2k_init(have_crypto) {
         ebi('u2notbtn').innerHTML = '';
     }
 
-    var post_url = ebi('op_bup').getElementsByTagName('form')[0].getAttribute('action');
-    if (post_url && post_url.charAt(post_url.length - 1) !== '/')
-        post_url += '/';
-
     var shame = 'your browser <a href="https://www.chromium.org/blink/webcrypto">disables sha512</a> unless you <a href="' + (window.location + '').replace(':', 's:') + '">use https</a>'
     var is_https = (window.location + '').indexOf('https:') === 0;
     if (is_https)
@@ -255,7 +195,7 @@ function up2k_init(have_crypto) {
     // handle user intent to use the basic uploader instead
     ebi('u2nope').onclick = function (e) {
         e.preventDefault();
-        setmsg('');
+        setmsg();
         goto('bup');
     };
 
@@ -279,13 +219,17 @@ function up2k_init(have_crypto) {
     }
 
     function bcfg_get(name, defval) {
+        var o = ebi(name);
+        if (!o)
+            return defval;
+
         var val = localStorage.getItem(name);
         if (val === null)
             val = defval;
         else
             val = (val == '1');
 
-        ebi(name).checked = val;
+        o.checked = val;
         return val;
     }
 
@@ -293,7 +237,10 @@ function up2k_init(have_crypto) {
         localStorage.setItem(
             name, val ? '1' : '0');
 
-        ebi(name).checked = val;
+        var o = ebi(name);
+        if (o)
+            o.checked = val;
+
         return val;
     }
 
@@ -301,6 +248,7 @@ function up2k_init(have_crypto) {
     var multitask = bcfg_get('multitask', true);
     var ask_up = bcfg_get('ask_up', true);
     var flag_en = bcfg_get('flag_en', false);
+    var fsearch = bcfg_get('fsearch', false);
 
     var col_hashing = '#00bbff';
     var col_hashed = '#004466';
@@ -334,6 +282,7 @@ function up2k_init(have_crypto) {
 
     var flag = false;
     apply_flag_cfg();
+    set_fsearch();
 
     function nav() {
         ebi('file' + fdom_ctr).click();
@@ -404,7 +353,7 @@ function up2k_init(have_crypto) {
         for (var a = 0; a < good_files.length; a++)
             msg.push(good_files[a].name);
 
-        if (ask_up && !confirm(msg.join('\n')))
+        if (ask_up && !fsearch && !confirm(msg.join('\n')))
             return;
 
         for (var a = 0; a < good_files.length; a++) {
@@ -418,6 +367,8 @@ function up2k_init(have_crypto) {
                 "name": fobj.name,
                 "size": fobj.size,
                 "lmod": lmod / 1000,
+                "purl": get_vpath(),
+                "done": false,
                 "hash": []
             };
 
@@ -432,7 +383,7 @@ function up2k_init(have_crypto) {
 
             var tr = document.createElement('tr');
             tr.innerHTML = '<td id="f{0}n"></td><td id="f{0}t">hashing</td><td id="f{0}p" class="prog"></td>'.format(st.files.length);
-            tr.getElementsByTagName('td')[0].textContent = entry.name;
+            tr.getElementsByTagName('td')[0].innerHTML = fsearch ? entry.name : linksplit(esc(entry.purl + entry.name)).join(' ');
             ebi('u2tab').appendChild(tr);
 
             st.files.push(entry);
@@ -450,6 +401,19 @@ function up2k_init(have_crypto) {
     }
     more_one_file();
 
+    function u2cleanup(e) {
+        ev(e);
+        for (var a = 0; a < st.files.length; a++) {
+            var t = st.files[a];
+            if (t.done && t.name) {
+                var tr = ebi('f{0}p'.format(t.n)).parentNode;
+                tr.parentNode.removeChild(tr);
+                t.name = undefined;
+            }
+        }
+    }
+    ebi('u2cleanup').onclick = u2cleanup;
+
     /////
     ////
     ///   actuator
@@ -474,6 +438,7 @@ function up2k_init(have_crypto) {
 
     var tasker = (function () {
         var mutex = false;
+        var was_busy = false;
 
         function taskerd() {
             if (mutex)
@@ -492,16 +457,25 @@ function up2k_init(have_crypto) {
                         st.busy.upload.length;
                 }
 
-                if (flag) {
-                    var need_flag = 0 !=
-                        st.todo.hash.length +
-                        st.todo.handshake.length +
-                        st.todo.upload.length +
-                        st.busy.hash.length +
-                        st.busy.handshake.length +
-                        st.busy.upload.length;
+                var is_busy = 0 !=
+                    st.todo.hash.length +
+                    st.todo.handshake.length +
+                    st.todo.upload.length +
+                    st.busy.hash.length +
+                    st.busy.handshake.length +
+                    st.busy.upload.length;
 
-                    if (need_flag) {
+                if (was_busy != is_busy) {
+                    was_busy = is_busy;
+
+                    if (is_busy)
+                        window.addEventListener("beforeunload", warn_uploader_busy);
+                    else
+                        window.removeEventListener("beforeunload", warn_uploader_busy);
+                }
+
+                if (flag) {
+                    if (is_busy) {
                         var now = new Date().getTime();
                         flag.take(now);
                         if (!flag.ours) {
@@ -795,10 +769,38 @@ function up2k_init(have_crypto) {
             if (xhr.status == 200) {
                 var response = JSON.parse(xhr.responseText);
 
+                if (!response.name) {
+                    var msg = '';
+                    var smsg = '';
+                    if (!response || !response.length) {
+                        msg = 'not found on server';
+                        smsg = '404';
+                    }
+                    else {
+                        smsg = 'found';
+                        var hit = response[0],
+                            msg = linksplit(hit.rp).join(''),
+                            tr = unix2iso(hit.ts),
+                            tu = unix2iso(t.lmod),
+                            diff = parseInt(t.lmod) - parseInt(hit.ts),
+                            cdiff = (Math.abs(diff) <= 2) ? '3c0' : 'f0b',
+                            sdiff = '<span style="color:#' + cdiff + '">diff ' + diff;
+
+                        msg += '<br /><small>' + tr + ' (srv), ' + tu + ' (You), ' + sdiff + '</span></span>';
+                    }
+                    ebi('f{0}p'.format(t.n)).innerHTML = msg;
+                    ebi('f{0}t'.format(t.n)).innerHTML = smsg;
+                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    st.bytes.uploaded += t.size;
+                    t.done = true;
+                    tasker();
+                    return;
+                }
+
                 if (response.name !== t.name) {
                     // file exists; server renamed us
                     t.name = response.name;
-                    ebi('f{0}n'.format(t.n)).textContent = t.name;
+                    ebi('f{0}n'.format(t.n)).innerHTML = linksplit(esc(t.purl + t.name)).join(' ');
                 }
 
                 t.postlist = [];
@@ -832,6 +834,7 @@ function up2k_init(have_crypto) {
                 st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
 
                 if (done) {
+                    t.done = true;
                     st.bytes.uploaded += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
                     var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
@@ -851,6 +854,11 @@ function up2k_init(have_crypto) {
                     var ofs = err.lastIndexOf(' : ');
                     if (ofs > 0)
                         err = err.slice(0, ofs);
+
+                    ofs = err.indexOf('\n/');
+                    if (ofs !== -1) {
+                        err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2, -1)).join(' ');
+                    }
                 }
                 if (err != "") {
                     ebi('f{0}t'.format(t.n)).innerHTML = "ERROR";
@@ -867,14 +875,19 @@ function up2k_init(have_crypto) {
                     "no further information"));
             }
         };
-        xhr.open('POST', post_url + 'handshake.php', true);
-        xhr.responseType = 'text';
-        xhr.send(JSON.stringify({
+
+        var req = {
             "name": t.name,
             "size": t.size,
             "lmod": t.lmod,
             "hash": t.hash
-        }));
+        };
+        if (fsearch)
+            req.srch = 1;
+
+        xhr.open('POST', t.purl + 'handshake.php', true);
+        xhr.responseType = 'text';
+        xhr.send(JSON.stringify(req));
     }
 
     /////
@@ -930,7 +943,7 @@ function up2k_init(have_crypto) {
                         (xhr.responseText && xhr.responseText) ||
                         "no further information"));
             };
-            xhr.open('POST', post_url + 'chunkpit.php', true);
+            xhr.open('POST', t.purl + 'chunkpit.php', true);
             //xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart].substr(1) + "x");
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
@@ -966,6 +979,46 @@ function up2k_init(have_crypto) {
     ///   config ui
     //
 
+    function onresize(ev) {
+        var bar = ebi('ops'),
+            wpx = innerWidth,
+            fpx = parseInt(getComputedStyle(bar)['font-size']),
+            wem = wpx * 1.0 / fpx,
+            wide = wem > 54,
+            parent = ebi(wide ? 'u2btn_cw' : 'u2btn_ct'),
+            btn = ebi('u2btn');
+
+        //console.log([wpx, fpx, wem]);
+        if (btn.parentNode !== parent) {
+            parent.appendChild(btn);
+            ebi('u2conf').setAttribute('class', wide ? 'has_btn' : '');
+        }
+    }
+    window.onresize = onresize;
+    onresize();
+
+    function desc_show(ev) {
+        var msg = this.getAttribute('alt');
+        msg = msg.replace(/\$N/g, "<br />");
+        var cdesc = ebi('u2cdesc');
+        cdesc.innerHTML = msg;
+        cdesc.setAttribute('class', 'show');
+    }
+    function desc_hide(ev) {
+        ebi('u2cdesc').setAttribute('class', '');
+    }
+    var o = document.querySelectorAll('#u2conf *[alt]');
+    for (var a = o.length - 1; a >= 0; a--) {
+        o[a].parentNode.getElementsByTagName('input')[0].setAttribute('alt', o[a].getAttribute('alt'));
+    }
+    var o = document.querySelectorAll('#u2conf *[alt]');
+    for (var a = 0; a < o.length; a++) {
+        o[a].onfocus = desc_show;
+        o[a].onblur = desc_hide;
+        o[a].onmouseenter = desc_show;
+        o[a].onmouseleave = desc_hide;
+    }
+
     function bumpthread(dir) {
         try {
             dir.stopPropagation();
@@ -1007,6 +1060,43 @@ function up2k_init(have_crypto) {
         bcfg_set('ask_up', ask_up);
     }
 
+    function tgl_fsearch() {
+        set_fsearch(!fsearch);
+    }
+
+    function set_fsearch(new_state) {
+        var perms = document.body.getAttribute('perms');
+        var read_only = false;
+
+        if (!ebi('fsearch')) {
+            new_state = false;
+        }
+        else if (perms && perms.indexOf('write') === -1) {
+            new_state = true;
+            read_only = true;
+        }
+
+        if (new_state !== undefined) {
+            fsearch = new_state;
+            bcfg_set('fsearch', fsearch);
+        }
+
+        try {
+            document.querySelector('label[for="fsearch"]').style.opacity = read_only ? '0' : '1';
+        }
+        catch (ex) { }
+
+        try {
+            var fun = fsearch ? 'add' : 'remove';
+            ebi('op_up2k').classList[fun]('srch');
+
+            var ico = fsearch ? '🔎' : '🚀';
+            var desc = fsearch ? 'Search' : 'Upload';
+            ebi('u2bm').innerHTML = ico + ' <sup>' + desc + '</sup>';
+        }
+        catch (ex) { }
+    }
+
     function tgl_flag_en() {
         flag_en = !flag_en;
         bcfg_set('flag_en', flag_en);
@@ -1047,12 +1137,26 @@ function up2k_init(have_crypto) {
     ebi('multitask').addEventListener('click', tgl_multitask, false);
     ebi('ask_up').addEventListener('click', tgl_ask_up, false);
     ebi('flag_en').addEventListener('click', tgl_flag_en, false);
+    var o = ebi('fsearch');
+    if (o)
+        o.addEventListener('click', tgl_fsearch, false);
 
     var nodes = ebi('u2conf').getElementsByTagName('a');
     for (var a = nodes.length - 1; a >= 0; a--)
         nodes[a].addEventListener('touchend', nop, false);
 
+    set_fsearch();
     bumpthread({ "target": 1 })
-
-    return { "init_deps": init_deps }
+    return { "init_deps": init_deps, "set_fsearch": set_fsearch }
 }
+
+
+function warn_uploader_busy(e) {
+    e.preventDefault();
+    e.returnValue = '';
+    return "upload in progress, click abort and use the file-tree to navigate instead";
+}
+
+
+if (document.querySelector('#op_up2k.act'))
+    goto_up2k();

copyparty/web/upload.css

@@ -1,92 +1,4 @@
-.opview {
-	display: none;
-}
-.opview.act {
-	display: block;
-}
-#ops a {
-	color: #fc5;
-	font-size: 1.5em;
-	padding: 0 .3em;
-	margin: 0;
-	outline: none;
-}
-#ops a.act {
-	text-decoration: underline;
-}
-/*
-#ops a+a:after,
-#ops a:first-child:after {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #01a7e1;
-	margin-left: .3em;
-	position: relative;
-}
-#ops a+a:before {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #ff3f1a;
-	margin-right: .3em;
-	margin-left: -.3em;
-}
-#ops a:last-child:after {
-	content: '';
-}
-#ops a.act:before,
-#ops a.act:after {
-	text-decoration: none !important;
-}
-*/
-#ops i {
-	font-size: 1.5em;
-}
-#ops i:before {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #01a7e1;
-	position: relative;
-}
-#ops i:after {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #ff3f1a;
-	margin-left: -.35em;
-	font-size: 1.05em;
-}
-#ops,
-.opbox {
-	border: 1px solid #3a3a3a;
-	box-shadow: 0 0 1em #222 inset;
-}
-#ops {
-	display: none;
-	background: #333;
-	margin: 1.7em 1.5em 0 1.5em;
-	padding: .3em .6em;
-	border-radius: .3em;
-	border-width: .15em 0;
-}
-.opbox {
-	background: #2d2d2d;
-	margin: 1.5em 0 0 0;
-	padding: .5em;
-	border-radius: 0 1em 1em 0;
-	border-width: .15em .3em .3em 0;
-	max-width: 40em;
-}
-.opbox input {
-	margin: .5em;
-}
-.opbox input[type=text] {
-	color: #fff;
-	background: #383838;
-	border: none;
-	box-shadow: 0 0 .3em #222;
-	border-bottom: 1px solid #fc5;
-	border-radius: .2em;
-	padding: .2em .3em;
-}
+
 #op_up2k {
 	padding: 0 1em 1em 1em;
 }
@@ -94,6 +6,9 @@
 	position: absolute;
 	top: 0;
 	left: 0;
+	width: 2px;
+	height: 2px;
+	overflow: hidden;
 }
 #u2form input {
 	background: #444;
@@ -104,11 +19,6 @@
 	color: #f87;
 	padding: .5em;
 }
-#u2form {
-	width: 2px;
-	height: 2px;
-	overflow: hidden;
-}
 #u2btn {
 	color: #eee;
 	background: #555;
@@ -117,17 +27,27 @@
 	background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%);
 	filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0);
 	text-decoration: none;
-	line-height: 1.5em;
+	line-height: 1.3em;
 	border: 1px solid #222;
 	border-radius: .4em;
 	text-align: center;
-	font-size: 2em;
-	margin: 1em auto;
-	padding: 1em 0;
-	width: 12em;
+	font-size: 1.5em;
+	margin: .5em auto;
+	padding: .8em 0;
+	width: 16em;
 	cursor: pointer;
 	box-shadow: .4em .4em 0 #111;
 }
+#op_up2k.srch #u2btn {
+	background: linear-gradient(to bottom, #ca3 0%, #fd8 50%, #fc6 51%, #b92 100%);
+	text-shadow: 1px 1px 1px #fc6;
+	color: #333;
+}
+#u2conf #u2btn {
+	margin: -1.5em 0;
+	padding: .8em 0;
+	width: 100%;
+}
 #u2notbtn {
 	display: none;
 	text-align: center;
@@ -142,6 +62,9 @@
 	width: calc(100% - 2em);
 	max-width: 100em;
 }
+#u2form.srch #u2tab {
+	max-width: none;
+}
 #u2tab td {
 	border: 1px solid #ccc;
 	border-width: 0 0px 1px 0;
@@ -153,12 +76,19 @@
 #u2tab td:nth-child(3) {
 	width: 40%;
 }
+#u2form.srch #u2tab td:nth-child(3) {
+	font-family: sans-serif;
+	width: auto;
+}
 #u2tab tr+tr:hover td {
 	background: #222;
 }
 #u2conf {
 	margin: 1em auto;
-	width: 26em;
+	width: 30em;
+}
+#u2conf.has_btn {
+	width: 46em;
 }
 #u2conf * {
 	text-align: center;
@@ -194,16 +124,72 @@
 #u2conf input+a {
 	background: #d80;
 }
+#u2conf label {
+	font-size: 1.6em;
+	width: 2em;
+	height: 1em;
+	padding: .4em 0;
+	display: block;
+	user-select: none;
+	border-radius: .25em;
+}
+#u2conf input[type="checkbox"] {
+	position: relative;
+	opacity: .02;
+	top: 2em;
+}
 #u2conf input[type="checkbox"]+label {
-	color: #f5a;
+	position: relative;
+	background: #603;
+	border-bottom: .2em solid #a16;
+	box-shadow: 0 .1em .3em #a00 inset;
 }
 #u2conf input[type="checkbox"]:checked+label {
-	color: #fc5;
+	background: #6a1;
+	border-bottom: .2em solid #efa;
+	box-shadow: 0 .1em .5em #0c0;
+}
+#u2conf input[type="checkbox"]+label:hover {
+	box-shadow: 0 .1em .3em #fb0;
+	border-color: #fb0;
+}
+#op_up2k.srch #u2conf td:nth-child(1)>*,
+#op_up2k.srch #u2conf td:nth-child(2)>*,
+#op_up2k.srch #u2conf td:nth-child(3)>* {
+	background: #777;
+	border-color: #ccc;
+	box-shadow: none;
+	opacity: .2;
+}
+#u2cdesc {
+	position: absolute;
+	width: 34em;
+	left: calc(50% - 15em);
+	background: #222;
+	border: 0 solid #555;
+	text-align: center;
+	overflow: hidden;
+	margin: 0 -2em;
+	height: 0;
+	padding: 0 1em;
+	opacity: .1;
+    transition: all 0.14s ease-in-out;
+	border-radius: .4em;
+	box-shadow: 0 .2em .5em #222;
+}
+#u2cdesc.show {
+	padding: 1em;
+	height: auto;
+	border-width: .2em 0;
+	opacity: 1;
 }
 #u2foot {
 	color: #fff;
 	font-style: italic;
 }
+#u2footfoot {
+	margin-bottom: -1em;
+}
 .prog {
 	font-family: monospace;
 }
@@ -225,3 +211,13 @@
 	bottom: 0;
 	background: #0a0;
 }
+#u2tab a>span {
+	font-weight: bold;
+	font-style: italic;
+	color: #fff;
+	padding-left: .2em;
+}
+#u2cleanup {
+	float: right;
+	margin-bottom: -.3em;
+}

copyparty/web/upload.html

@@ -1,14 +1,7 @@
-    <div id="ops"><a
-        href="#" data-dest="">---</a><i></i><a
-        href="#" data-dest="up2k">up2k</a><i></i><a
-        href="#" data-dest="bup">bup</a><i></i><a
-        href="#" data-dest="mkdir">mkdir</a><i></i><a
-        href="#" data-dest="new_md">new.md</a><i></i><a
-        href="#" data-dest="msg">msg</a></div>
 
     <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
@@ -16,7 +9,7 @@
     </div>
 
     <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="mkdir" />
             <input type="text" name="name" size="30">
             <input type="submit" value="mkdir">
@@ -24,7 +17,7 @@
     </div>
 
     <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="new_md" />
             <input type="text" name="name" size="30">
             <input type="submit" value="create doc">
@@ -32,9 +25,9 @@
     </div>
 
     <div id="op_msg" class="opview opbox">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
             <input type="text" name="msg" size="30">
-            <input type="submit" value="send">
+            <input type="submit" value="send msg">
         </form>
     </div>
 
@@ -44,6 +37,25 @@
             <table id="u2conf">
                 <tr>
                     <td>parallel uploads</td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="multitask" />
+                        <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="ask_up" />
+                        <label for="ask_up" alt="ask for confirmation befofre upload starts">💭</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="flag_en" />
+                        <label for="flag_en" alt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
+                    </td>
+                {%- if have_up2k_idx %}
+                    <td data-perm="read" rowspan="2">
+                        <input type="checkbox" id="fsearch" />
+                        <label for="fsearch" alt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
+                    </td>
+                {%- endif %}
+                    <td data-perm="read" rowspan="2" id="u2btn_cw"></td>
                 </tr>
                 <tr>
                     <td>
@@ -51,36 +63,29 @@
                         <input class="txtbox" id="nthread" value="2" />
                         <a href="#" id="nthread_add">+</a>
                     </td>
-                    <td rowspan="2" style="padding-left:1.5em">
-                        <input type="checkbox" id="multitask" />
-                        <label for="multitask">hash<br />while<br />upping</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="ask_up" />
-                        <label for="ask_up">ask<br />before<br />start</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="flag_en" />
-                        <label for="flag_en">only<br />one tab<br />at once</label>
-                    </td>
                 </tr>
             </table>
 
+            <div id="u2cdesc"></div>
+
             <div id="u2notbtn"></div>
 
-            <div id="u2btn">
-                drop files here<br />
-                (or click me)
+            <div id="u2btn_ct">
+                <div id="u2btn">
+                    <span id="u2bm"></span><br />
+                    drop files here<br />
+                    (or click me)
+                </div>
             </div>
 
             <table id="u2tab">
                 <tr>
                     <td>filename</td>
                     <td>status</td>
-                    <td>progress</td>
+                    <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
                 </tr>
             </table>
 
             <p id="u2foot"></p>
-            <p>( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
+            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
     </div>

copyparty/web/util.js

@@ -85,6 +85,11 @@ function sortTable(table, col) {
     th[col].className = 'sort' + reverse;
     var stype = th[col].getAttribute('sort');
     tr = tr.sort(function (a, b) {
+        if (!a.cells[col])
+            return -1;
+        if (!b.cells[col])
+            return 1;
+
         var v1 = a.cells[col].textContent.trim();
         var v2 = b.cells[col].textContent.trim();
         if (stype == 'int') {
@@ -106,4 +111,124 @@ function makeSortable(table) {
             sortTable(table, i);
         };
     }(i));
-}
+}
+
+
+
+(function () {
+    var ops = document.querySelectorAll('#ops>a');
+    for (var a = 0; a < ops.length; a++) {
+        ops[a].onclick = opclick;
+    }
+})();
+
+
+function opclick(ev) {
+    if (ev) //ie
+        ev.preventDefault();
+
+    var dest = this.getAttribute('data-dest');
+    goto(dest);
+
+    // writing a blank value makes ie8 segfault w
+    if (window.localStorage)
+        localStorage.setItem('opmode', dest || '.');
+
+    var input = document.querySelector('.opview.act input:not([type="hidden"])')
+    if (input)
+        input.focus();
+}
+
+
+function goto(dest) {
+    var obj = document.querySelectorAll('.opview.act');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].classList.remove('act');
+
+    obj = document.querySelectorAll('#ops>a');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].classList.remove('act');
+
+    var others = ['path', 'files', 'widget'];
+    for (var a = 0; a < others.length; a++)
+        ebi(others[a]).classList.remove('hidden');
+
+    if (dest) {
+        var ui = ebi('op_' + dest);
+        ui.classList.add('act');
+        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+
+        var fn = window['goto_' + dest];
+        if (fn)
+            fn();
+    }
+}
+
+
+(function () {
+    goto();
+    if (window.localStorage) {
+        var op = localStorage.getItem('opmode');
+        if (op !== null && op !== '.')
+            goto(op);
+    }
+})();
+
+
+function linksplit(rp) {
+    var ret = [];
+    var apath = '/';
+    if (rp && rp.charAt(0) == '/')
+        rp = rp.slice(1);
+
+    while (rp) {
+        var link = rp;
+        var ofs = rp.indexOf('/');
+        if (ofs === -1) {
+            rp = null;
+        }
+        else {
+            link = rp.slice(0, ofs + 1);
+            rp = rp.slice(ofs + 1);
+        }
+        var vlink = link;
+        if (link.indexOf('/') !== -1)
+            vlink = link.slice(0, -1) + '<span>/</span>';
+
+        ret.push('<a href="' + apath + link + '">' + vlink + '</a>');
+        apath += link;
+    }
+    return ret;
+}
+
+
+function get_evpath() {
+    var ret = document.location.pathname;
+
+    if (ret.indexOf('/') !== 0)
+        ret = '/' + ret;
+
+    if (ret.lastIndexOf('/') !== ret.length - 1)
+        ret += '/';
+
+    return ret;
+}
+
+
+function get_vpath() {
+    return decodeURIComponent(get_evpath());
+}
+
+
+function unix2iso(ts) {
+    return new Date(ts * 1000).toISOString().replace("T", " ").slice(0, -5);
+}
+
+
+function has(haystack, needle) {
+    for (var a = 0; a < haystack.length; a++)
+        if (haystack[a] == needle)
+            return true;
+
+    return false;
+}

scripts/deps-docker/Dockerfile

@@ -1,12 +1,10 @@
-FROM    alpine:3.11
+FROM    alpine:3.13
 WORKDIR /z
-ENV     ver_asmcrypto=2821dd1dedd1196c378f5854037dda5c869313f3 \
-        ver_markdownit=10.0.0 \
-        ver_showdown=1.9.1 \
+ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
         ver_marked=1.1.0 \
-        ver_ogvjs=1.6.1 \
-        ver_mde=2.10.1 \
-        ver_codemirror=5.53.2 \
+        ver_ogvjs=1.8.0 \
+        ver_mde=2.14.0 \
+        ver_codemirror=5.59.3 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 
@@ -17,7 +15,7 @@ RUN     mkdir -p /z/dist/no-pk \
         && wget https://fonts.gstatic.com/s/sourcecodepro/v11/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevW.woff2 -O scp.woff2 \
         && apk add cmake make g++ git bash npm patch wget tar pigz brotli gzip unzip python3 python3-dev brotli py3-brotli \
         && wget https://github.com/brion/ogv.js/releases/download/$ver_ogvjs/ogvjs-$ver_ogvjs.zip -O ogvjs.zip \
-        && wget https://github.com/asmcrypto/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
+        && wget https://github.com/openpgpjs/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
         && wget https://github.com/markedjs/marked/archive/v$ver_marked.tar.gz -O marked.tgz \
         && wget https://github.com/Ionaru/easy-markdown-editor/archive/$ver_mde.tar.gz -O mde.tgz \
         && wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
@@ -52,6 +50,7 @@ RUN     tar -xf zopfli.tgz \
             -S . \
         && make -C build \
         && make -C build install \
+        && python3 -m ensurepip \
         && python3 -m pip install fonttools zopfli
 
 

scripts/deps-docker/codemirror.patch

@@ -1,6 +1,6 @@
-diff -NarU2 CodeMirror-orig/mode/gfm/gfm.js CodeMirror-edit/mode/gfm/gfm.js
---- CodeMirror-orig/mode/gfm/gfm.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/mode/gfm/gfm.js	2020-05-02 02:13:32.142131800 +0200
+diff -NarU2 codemirror-5.59.3-orig/mode/gfm/gfm.js codemirror-5.59.3/mode/gfm/gfm.js
+--- codemirror-5.59.3-orig/mode/gfm/gfm.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/mode/gfm/gfm.js	2021-02-21 20:42:02.166174775 +0000
 @@ -97,5 +97,5 @@
          }
        }
@@ -15,9 +15,9 @@ diff -NarU2 CodeMirror-orig/mode/gfm/gfm.js CodeMirror-edit/mode/gfm/gfm.js
 +      }*/
        stream.next();
        return null;
-diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
---- CodeMirror-orig/mode/meta.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/mode/meta.js	2020-05-02 03:56:58.852408400 +0200
+diff -NarU2 codemirror-5.59.3-orig/mode/meta.js codemirror-5.59.3/mode/meta.js
+--- codemirror-5.59.3-orig/mode/meta.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/mode/meta.js	2021-02-21 20:42:54.798742821 +0000
 @@ -13,4 +13,5 @@
  
    CodeMirror.modeInfo = [
@@ -28,7 +28,7 @@ diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
      {name: "Gas", mime: "text/x-gas", mode: "gas", ext: ["s"]},
      {name: "Gherkin", mime: "text/x-feature", mode: "gherkin", ext: ["feature"]},
 +    */
-     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history).md$/i},
+     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history)\.md$/i},
 +    /*
      {name: "Go", mime: "text/x-go", mode: "go", ext: ["go"]},
      {name: "Groovy", mime: "text/x-groovy", mode: "groovy", ext: ["groovy", "gradle"], file: /^Jenkinsfile$/},
@@ -56,16 +56,16 @@ diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
 +    /*
      {name: "XQuery", mime: "application/xquery", mode: "xquery", ext: ["xy", "xquery"]},
      {name: "Yacas", mime: "text/x-yacas", mode: "yacas", ext: ["ys"]},
-@@ -171,4 +180,5 @@
-     {name: "xu", mime: "text/x-xu", mode: "mscgen", ext: ["xu"]},
-     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]}
+@@ -172,4 +181,5 @@
+     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]},
+     {name: "WebAssembly", mime: "text/webassembly", mode: "wast", ext: ["wat", "wast"]},
 +    */
    ];
    // Ensure all modes have a mime property for backwards compatibility
-diff -NarU2 CodeMirror-orig/src/display/selection.js CodeMirror-edit/src/display/selection.js
---- CodeMirror-orig/src/display/selection.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/display/selection.js	2020-05-02 03:27:30.144662800 +0200
-@@ -83,29 +83,21 @@
+diff -NarU2 codemirror-5.59.3-orig/src/display/selection.js codemirror-5.59.3/src/display/selection.js
+--- codemirror-5.59.3-orig/src/display/selection.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/display/selection.js	2021-02-21 20:44:14.860894328 +0000
+@@ -84,29 +84,21 @@
      let order = getOrder(lineObj, doc.direction)
      iterateBidiSections(order, fromArg || 0, toArg == null ? lineLen : toArg, (from, to, dir, i) => {
 -      let ltr = dir == "ltr"
@@ -105,24 +105,24 @@ diff -NarU2 CodeMirror-orig/src/display/selection.js CodeMirror-edit/src/display
 +          botRight = openEnd && last ? rightSide : toPos.right
          add(topLeft, fromPos.top, topRight - topLeft, fromPos.bottom)
          if (fromPos.bottom < toPos.top) add(leftSide, fromPos.bottom, null, toPos.top)
-diff -NarU2 CodeMirror-orig/src/input/ContentEditableInput.js CodeMirror-edit/src/input/ContentEditableInput.js
---- CodeMirror-orig/src/input/ContentEditableInput.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/input/ContentEditableInput.js	2020-05-02 03:33:05.707995500 +0200
-@@ -391,4 +391,5 @@
+diff -NarU2 codemirror-5.59.3-orig/src/input/ContentEditableInput.js codemirror-5.59.3/src/input/ContentEditableInput.js
+--- codemirror-5.59.3-orig/src/input/ContentEditableInput.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/input/ContentEditableInput.js	2021-02-21 20:44:33.273953867 +0000
+@@ -399,4 +399,5 @@
    let info = mapFromLineView(view, line, pos.line)
  
 +  /*
    let order = getOrder(line, cm.doc.direction), side = "left"
    if (order) {
-@@ -396,4 +397,5 @@
+@@ -404,4 +405,5 @@
      side = partPos % 2 ? "right" : "left"
    }
 +  */
    let result = nodeAndOffsetInLineMap(info.map, pos.ch, side)
    result.offset = result.collapse == "right" ? result.end : result.start
-diff -NarU2 CodeMirror-orig/src/input/movement.js CodeMirror-edit/src/input/movement.js
---- CodeMirror-orig/src/input/movement.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/input/movement.js	2020-05-02 03:31:19.710773500 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/input/movement.js codemirror-5.59.3/src/input/movement.js
+--- codemirror-5.59.3-orig/src/input/movement.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/input/movement.js	2021-02-21 20:45:12.763093671 +0000
 @@ -15,4 +15,5 @@
  
  export function endOfLine(visually, cm, lineObj, lineNo, dir) {
@@ -146,9 +146,9 @@ diff -NarU2 CodeMirror-orig/src/input/movement.js CodeMirror-edit/src/input/move
    return null
 +  */
  }
-diff -NarU2 CodeMirror-orig/src/line/line_data.js CodeMirror-edit/src/line/line_data.js
---- CodeMirror-orig/src/line/line_data.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/line/line_data.js	2020-05-02 03:17:02.785065000 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/line/line_data.js codemirror-5.59.3/src/line/line_data.js
+--- codemirror-5.59.3-orig/src/line/line_data.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/line/line_data.js	2021-02-21 20:45:36.472549599 +0000
 @@ -79,6 +79,6 @@
      // Optionally wire in some hacks into the token-rendering
      // algorithm, to deal with browser quirks.
@@ -158,9 +158,9 @@ diff -NarU2 CodeMirror-orig/src/line/line_data.js CodeMirror-edit/src/line/line_
 +    //  builder.addToken = buildTokenBadBidi(builder.addToken, order)
      builder.map = []
      let allowFrontierUpdate = lineView != cm.display.externalMeasured && lineNo(line)
-diff -NarU2 CodeMirror-orig/src/measurement/position_measurement.js CodeMirror-edit/src/measurement/position_measurement.js
---- CodeMirror-orig/src/measurement/position_measurement.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/measurement/position_measurement.js	2020-05-02 03:35:20.674159600 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/measurement/position_measurement.js codemirror-5.59.3/src/measurement/position_measurement.js
+--- codemirror-5.59.3-orig/src/measurement/position_measurement.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/measurement/position_measurement.js	2021-02-21 20:50:52.372945293 +0000
 @@ -380,5 +380,6 @@
      sticky = "after"
    }
@@ -199,9 +199,9 @@ diff -NarU2 CodeMirror-orig/src/measurement/position_measurement.js CodeMirror-e
 +*/
  
  let measureText
-diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
---- CodeMirror-orig/src/util/bidi.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/util/bidi.js	2020-05-02 03:12:44.418649800 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/util/bidi.js codemirror-5.59.3/src/util/bidi.js
+--- codemirror-5.59.3-orig/src/util/bidi.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/util/bidi.js	2021-02-21 20:52:18.168092225 +0000
 @@ -4,5 +4,5 @@
  
  export function iterateBidiSections(order, from, to, f) {
@@ -239,20 +239,19 @@ diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
 +  var fun = function(str, direction) {
      let outerType = direction == "ltr" ? "L" : "R"
  
-@@ -204,12 +210,16 @@
+@@ -204,5 +210,11 @@
      return direction == "rtl" ? order.reverse() : order
    }
--})()
- 
 +  return function(str, direction) {
 +    var ret = fun(str, direction);
 +    console.log("bidiOrdering inner ([%s], %s) => [%s]", str, direction, ret);
 +    return ret;
 +  }
-+})()
+ })()
 +*/
+ 
  // Get the bidi ordering for the given line (and cache it). Returns
- // false for lines that are fully left-to-right, and an array of
+@@ -210,6 +222,4 @@
  // BidiSpan objects otherwise.
  export function getOrder(line, direction) {
 -  let order = line.order
@@ -260,9 +259,9 @@ diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
 -  return order
 +  return false;
  }
-diff -NarU2 CodeMirror-orig/src/util/feature_detection.js CodeMirror-edit/src/util/feature_detection.js
---- CodeMirror-orig/src/util/feature_detection.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/util/feature_detection.js	2020-05-02 03:16:21.085621400 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/util/feature_detection.js codemirror-5.59.3/src/util/feature_detection.js
+--- codemirror-5.59.3-orig/src/util/feature_detection.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/util/feature_detection.js	2021-02-21 20:49:22.191269270 +0000
 @@ -25,4 +25,5 @@
  }
  

scripts/deps-docker/easymde.patch

@@ -1,33 +1,57 @@
-diff -NarU2 easymde-orig/gulpfile.js easymde-mod1/gulpfile.js
---- easymde-orig/gulpfile.js	2020-04-06 14:09:36.000000000 +0200
-+++ easymde-mod1/gulpfile.js	2020-05-01 14:33:52.260175200 +0200
+diff -NarU2 easy-markdown-editor-2.14.0-orig/gulpfile.js easy-markdown-editor-2.14.0/gulpfile.js
+--- easy-markdown-editor-2.14.0-orig/gulpfile.js	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/gulpfile.js	2021-02-21 20:55:37.134701007 +0000
 @@ -25,5 +25,4 @@
      './node_modules/codemirror/lib/codemirror.css',
      './src/css/*.css',
 -    './node_modules/codemirror-spell-checker/src/css/spell-checker.css',
  ];
  
-diff -NarU2 easymde-orig/package.json easymde-mod1/package.json
---- easymde-orig/package.json	2020-04-06 14:09:36.000000000 +0200
-+++ easymde-mod1/package.json	2020-05-01 14:33:57.189975800 +0200
+diff -NarU2 easy-markdown-editor-2.14.0-orig/package.json easy-markdown-editor-2.14.0/package.json
+--- easy-markdown-editor-2.14.0-orig/package.json	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/package.json	2021-02-21 20:55:47.761190082 +0000
 @@ -21,5 +21,4 @@
      "dependencies": {
-         "codemirror": "^5.52.2",
+         "codemirror": "^5.59.2",
 -        "codemirror-spell-checker": "1.1.2",
-         "marked": "^0.8.2"
+         "marked": "^2.0.0"
      },
-diff -NarU2 easymde-orig/src/js/easymde.js easymde-mod1/src/js/easymde.js
---- easymde-orig/src/js/easymde.js	2020-04-06 14:09:36.000000000 +0200
-+++ easymde-mod1/src/js/easymde.js	2020-05-01 14:34:19.878774400 +0200
-@@ -11,5 +11,4 @@
+diff -NarU2 easy-markdown-editor-2.14.0-orig/src/js/easymde.js easy-markdown-editor-2.14.0/src/js/easymde.js
+--- easy-markdown-editor-2.14.0-orig/src/js/easymde.js	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/src/js/easymde.js	2021-02-21 20:57:09.143171536 +0000
+@@ -12,5 +12,4 @@
  require('codemirror/mode/gfm/gfm.js');
  require('codemirror/mode/xml/xml.js');
 -var CodeMirrorSpellChecker = require('codemirror-spell-checker');
  var marked = require('marked/lib/marked');
  
-@@ -1889,18 +1888,7 @@
+@@ -1762,9 +1761,4 @@
+         options.autosave.uniqueId = options.autosave.unique_id;
  
+-    // If overlay mode is specified and combine is not provided, default it to true
+-    if (options.overlayMode && options.overlayMode.combine === undefined) {
+-      options.overlayMode.combine = true;
+-    }
+-
+     // Update this options
+     this.options = options;
+@@ -2003,28 +1997,7 @@
      var mode, backdrop;
+ 
+-    // CodeMirror overlay mode
+-    if (options.overlayMode) {
+-      CodeMirror.defineMode('overlay-mode', function(config) {
+-        return CodeMirror.overlayMode(CodeMirror.getMode(config, options.spellChecker !== false ? 'spell-checker' : 'gfm'), options.overlayMode.mode, options.overlayMode.combine);
+-      });
+-
+-      mode = 'overlay-mode';
+-      backdrop = options.parsingConfig;
+-      backdrop.gitHubSpice = false;
+-    } else {
+         mode = options.parsingConfig;
+         mode.name = 'gfm';
+         mode.gitHubSpice = false;
+-    }
 -    if (options.spellChecker !== false) {
 -        mode = 'spell-checker';
 -        backdrop = options.parsingConfig;
@@ -37,16 +61,28 @@ diff -NarU2 easymde-orig/src/js/easymde.js easymde-mod1/src/js/easymde.js
 -        CodeMirrorSpellChecker({
 -            codeMirrorInstance: CodeMirror,
 -        });
--    } else {
-         mode = options.parsingConfig;
-         mode.name = 'gfm';
-         mode.gitHubSpice = false;
 -    }
  
      // eslint-disable-next-line no-unused-vars
-@@ -1927,5 +1915,4 @@
-         configureMouse: configureMouse,
-         inputStyle: (options.inputStyle != undefined) ? options.inputStyle : isMobile() ? 'contenteditable' : 'textarea',
--        spellcheck: (options.nativeSpellcheck != undefined) ? options.nativeSpellcheck : true,
-     });
+diff -NarU2 easy-markdown-editor-2.14.0-orig/types/easymde.d.ts easy-markdown-editor-2.14.0/types/easymde.d.ts
+--- easy-markdown-editor-2.14.0-orig/types/easymde.d.ts	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/types/easymde.d.ts	2021-02-21 20:57:42.492620979 +0000
+@@ -160,9 +160,4 @@
+     }
  
+-    interface OverlayModeOptions {
+-      mode: CodeMirror.Mode<any>
+-      combine?: boolean
+-    }
+-
+     interface Options {
+         autoDownloadFontAwesome?: boolean;
+@@ -214,7 +209,5 @@
+ 
+         promptTexts?: PromptTexts;
+-        syncSideBySidePreviewScroll?: boolean;
+-
+-        overlayMode?: OverlayModeOptions
++        syncSideBySidePreviewScroll?: boolean
+     }
+ }

scripts/make-pypi-release.sh

@@ -86,6 +86,8 @@ function have() {
 	python -c "import $1; $1; $1.__version__"
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 . buildenv/bin/activate
 have setuptools
 have wheel

scripts/make-tgz-release.sh

@@ -35,6 +35,8 @@ ver="$1"
 	exit 1
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 mkdir -p dist
 zip_path="$(pwd)/dist/copyparty-$ver.zip"
 tgz_path="$(pwd)/dist/copyparty-$ver.tar.gz"