v0.9.8

fix tree trying to make surprise appearances
turns out whitespace compresses better than tabs
2025-10-24 00:24:04 +00:00 · 2021-03-15 01:13:46 +01:00 · 2021-03-13 02:29:13 +01:00 · 2021-03-13 00:16:07 +01:00 · 2021-03-13 00:15:53 +01:00 · 2021-03-13 00:14:22 +01:00
30 changed files with 1095 additions and 449 deletions
--- a/README.md
+++ b/README.md
@@ -69,7 +69,9 @@ summary: it works! you can use it! (but technically not even close to beta)

 # bugs

-* probably, pls let me know
+* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
+* Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* probably more, pls let me know


 # searching
@@ -210,6 +212,7 @@ pip install black bandit pylint flake8  # vscode tooling
 in the `scripts` folder:

 * run `make -C deps-docker` to build all dependencies
+* `git tag v1.2.3 && git push origin --tags`
 * create github release with `make-tgz-release.sh`
 * upload to pypi with `make-pypi-release.(sh|bat)`
 * create sfx with `make-sfx.sh`
--- a/bin/copyparty-fuse.py
+++ b/bin/copyparty-fuse.py
@@ -1008,6 +1008,12 @@ def main():
        log = null_log
        dbg = null_log

+    if ar.a and ar.a.startswith("$"):
+        fn = ar.a[1:]
+        log("reading password from file [{}]".format(fn))
+        with open(fn, "rb") as f:
+            ar.a = f.read().decode("utf-8").strip()
+
    if WINDOWS:
        os.system("rem")

--- a/contrib/systemd/copyparty.service
+++ b/contrib/systemd/copyparty.service
@@ -12,7 +12,7 @@
 Description=copyparty file server

 [Service]
-ExecStart=/usr/bin/python /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'

 [Install]
--- a/copyparty/main.py
+++ b/copyparty/main.py
@@ -18,7 +18,7 @@ import locale
 import argparse
 from textwrap import dedent

-from .__init__ import E, WINDOWS, VT100
+from .__init__ import E, WINDOWS, VT100, PY2
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
 from .util import py_desc, align_tab
@@ -53,6 +53,10 @@ class RiceFormatter(argparse.HelpFormatter):
        return "".join(indent + line + "\n" for line in text.splitlines())


+def warn(msg):
+    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+
+
 def ensure_locale():
    for x in [
        "en_US.UTF-8",
@@ -243,7 +247,8 @@ def main():
    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
    ap.add_argument("-nih", action="store_true", help="no info hostname")
    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
+    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
    ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")

@@ -255,6 +260,7 @@ def main():
    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
+    ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
    ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
    ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
@@ -298,7 +304,13 @@ def main():
        if al.ciphers:
            configure_ssl_ciphers(al)
    else:
-        print("\033[33m  ssl module does not exist; cannot enable https\033[0m\n")
+        warn("ssl module does not exist; cannot enable https")
+
+    if PY2 and WINDOWS and al.e2d:
+        warn(
+            "windows py2 cannot do unicode filenames with -e2d\n"
+            + "  (if you crash with codec errors then that is why)"
+        )

    SvcHub(al).run()

--- a/copyparty/version.py
+++ b/copyparty/version.py
@@ -1,8 +1,8 @@
 # coding: utf-8

-VERSION = (0, 9, 1)
+VERSION = (0, 9, 8)
 CODENAME = "the strongest music server"
-BUILD_DT = (2021, 3, 3)
+BUILD_DT = (2021, 3, 15)

 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
--- a/copyparty/authsrv.py
+++ b/copyparty/authsrv.py
@@ -6,7 +6,7 @@ import re
 import threading

 from .__init__ import PY2, WINDOWS
-from .util import undot, Pebkac, fsdec, fsenc
+from .util import undot, Pebkac, fsdec, fsenc, statdir, nuprint


 class VFS(object):
@@ -102,12 +102,11 @@ class VFS(object):

        return fsdec(os.path.realpath(fsenc(rp)))

-    def ls(self, rem, uname):
+    def ls(self, rem, uname, scandir, lstat=False):
        """return user-readable [fsdir,real,virt] items at vpath"""
        virt_vis = {}  # nodes readable by user
        abspath = self.canonical(rem)
-        items = os.listdir(fsenc(abspath))
-        real = [fsdec(x) for x in items]
+        real = list(statdir(nuprint, scandir, lstat, abspath))
        real.sort()
        if not rem:
            for name, vn2 in sorted(self.nodes.items()):
@@ -115,7 +114,7 @@ class VFS(object):
                    virt_vis[name] = vn2

            # no vfs nodes in the list of real inodes
-            real = [x for x in real if x not in self.nodes]
+            real = [x for x in real if x[0] not in self.nodes]

        return [abspath, real, virt_vis]

@@ -148,8 +147,8 @@ class AuthSrv(object):
        self.mutex = threading.Lock()
        self.reload()

-    def log(self, msg):
-        self.log_func("auth", msg)
+    def log(self, msg, c=0):
+        self.log_func("auth", msg, c)

    def invert(self, orig):
        if PY2:
@@ -305,9 +304,9 @@ class AuthSrv(object):

        if missing_users:
            self.log(
-                "\033[31myou must -a the following users: "
-                + ", ".join(k for k in sorted(missing_users))
-                + "\033[0m"
+                "you must -a the following users: "
+                + ", ".join(k for k in sorted(missing_users)),
+                c=1,
            )
            raise Exception("invalid config")

@@ -315,7 +314,7 @@ class AuthSrv(object):
            if (self.args.e2ds and vol.uwrite) or self.args.e2dsa:
                vol.flags["e2ds"] = True

-            if self.args.e2d:
+            if self.args.e2d or "e2ds" in vol.flags:
                vol.flags["e2d"] = True

            for k in ["e2t", "e2ts", "e2tsr"]:
@@ -330,8 +329,8 @@ class AuthSrv(object):
            v, _ = vfs.get("/", "*", False, True)
            if self.warn_anonwrite and os.getcwd() == v.realpath:
                self.warn_anonwrite = False
-                msg = "\033[31manyone can read/write the current directory: {}\033[0m"
-                self.log(msg.format(v.realpath))
+                msg = "anyone can read/write the current directory: {}"
+                self.log(msg.format(v.realpath), c=1)
        except Pebkac:
            self.warn_anonwrite = True

--- a/copyparty/broker_mpw.py
+++ b/copyparty/broker_mpw.py
@@ -49,11 +49,11 @@ class MpWorker(object):
        # print('k')
        pass

-    def log(self, src, msg):
-        self.q_yield.put([0, "log", [src, msg]])
+    def log(self, src, msg, c=0):
+        self.q_yield.put([0, "log", [src, msg, c]])

-    def logw(self, msg):
-        self.log("mp{}".format(self.n), msg)
+    def logw(self, msg, c=0):
+        self.log("mp{}".format(self.n), msg, c)

    def httpdrop(self, addr):
        self.q_yield.put([0, "httpdrop", [addr]])
@@ -73,7 +73,7 @@ class MpWorker(object):
                if PY2:
                    sck = pickle.loads(sck)  # nosec

-                self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
+                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
                self.httpsrv.accept(sck, addr)

                with self.mutex:
--- a/copyparty/broker_thr.py
+++ b/copyparty/broker_thr.py
@@ -28,7 +28,7 @@ class BrokerThr(object):
    def put(self, want_retval, dest, *args):
        if dest == "httpconn":
            sck, addr = args
-            self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
+            self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
            self.httpsrv.accept(sck, addr)

        else:
--- a/copyparty/httpcli.py
+++ b/copyparty/httpcli.py
@@ -41,8 +41,8 @@ class HttpCli(object):
        self.absolute_urls = False
        self.out_headers = {"Access-Control-Allow-Origin": "*"}

-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)

    def _check_nonfatal(self, ex):
        return ex.code < 400 or ex.code == 404
@@ -63,7 +63,7 @@ class HttpCli(object):

            if not headerlines[0]:
                # seen after login with IE6.0.2900.5512.xpsp.080413-2111 (xp-sp3)
-                self.log("\033[1;31mBUG: trailing newline from previous request\033[0m")
+                self.log("BUG: trailing newline from previous request", c="1;31")
                headerlines.pop(0)

            try:
@@ -74,7 +74,7 @@ class HttpCli(object):
        except Pebkac as ex:
            # self.log("pebkac at httpcli.run #1: " + repr(ex))
            self.keepalive = self._check_nonfatal(ex)
-            self.loud_reply(str(ex), status=ex.code)
+            self.loud_reply(unicode(ex), status=ex.code)
            return self.keepalive

        # time.sleep(0.4)
@@ -163,7 +163,7 @@ class HttpCli(object):
        response = ["HTTP/1.1 {} {}".format(status, HTTPCODE[status])]

        if length is not None:
-            response.append("Content-Length: " + str(length))
+            response.append("Content-Length: " + unicode(length))

        # close if unknown length, otherwise take client's preference
        response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
@@ -345,6 +345,10 @@ class HttpCli(object):
        with open(path, "wb", 512 * 1024) as f:
            post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)

+        self.conn.hsrv.broker.put(
+            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fn
+        )
+
        return post_sz, sha_b64, remains, path

    def handle_stash(self):
@@ -517,7 +521,7 @@ class HttpCli(object):
            if len(cstart) > 1 and path != os.devnull:
                self.log(
                    "clone {} to {}".format(
-                        cstart[0], " & ".join(str(x) for x in cstart[1:])
+                        cstart[0], " & ".join(unicode(x) for x in cstart[1:])
                    )
                )
                ofs = 0
@@ -675,6 +679,9 @@ class HttpCli(object):
                            raise Pebkac(400, "empty files in post")

                        files.append([sz, sha512_hex])
+                        self.conn.hsrv.broker.put(
+                            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
+                        )
                        self.conn.nbyte += sz

                except Pebkac:
@@ -690,7 +697,7 @@ class HttpCli(object):
                    raise

        except Pebkac as ex:
-            errmsg = str(ex)
+            errmsg = unicode(ex)

        td = max(0.1, time.time() - t0)
        sz_total = sum(x[0] for x in files)
@@ -999,7 +1006,7 @@ class HttpCli(object):
            mime=guess_mime(req_path)[0] or "application/octet-stream",
        )

-        logmsg += str(status) + logtail
+        logmsg += unicode(status) + logtail

        if self.mode == "HEAD" or not do_send:
            self.log(logmsg)
@@ -1013,7 +1020,7 @@ class HttpCli(object):
                remains = sendfile_py(lower, upper, f, self.s)

        if remains > 0:
-            logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
+            logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"

        spd = self._spd((upper - lower) - remains)
        self.log("{},  {}".format(logmsg, spd))
@@ -1060,7 +1067,7 @@ class HttpCli(object):
        sz_html = len(template.render(**targs).encode("utf-8"))
        self.send_headers(sz_html + sz_md, status)

-        logmsg += str(status)
+        logmsg += unicode(status)
        if self.mode == "HEAD" or not do_send:
            self.log(logmsg)
            return True
@@ -1074,7 +1081,7 @@ class HttpCli(object):
            self.log(logmsg + " \033[31md/c\033[0m")
            return False

-        self.log(logmsg + " " + str(len(html)))
+        self.log(logmsg + " " + unicode(len(html)))
        return True

    def tx_mounts(self):
@@ -1108,11 +1115,12 @@ class HttpCli(object):
        excl = None
        if target:
            excl, target = (target.split("/", 1) + [""])[:2]
-            ret["k" + excl] = self.gen_tree("/".join([top, excl]).strip("/"), target)
+            sub = self.gen_tree("/".join([top, excl]).strip("/"), target)
+            ret["k" + quotep(excl)] = sub

        try:
            vn, rem = self.auth.vfs.get(top, self.uname, True, False)
-            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)
+            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
        except:
            vfs_ls = []
            vfs_virt = {}
@@ -1123,13 +1131,13 @@ class HttpCli(object):

        dirs = []

+        vfs_ls = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
        if not self.args.ed or "dots" not in self.uparam:
            vfs_ls = exclude_dotfiles(vfs_ls)

        for fn in [x for x in vfs_ls if x != excl]:
-            abspath = os.path.join(fsroot, fn)
-            if os.path.isdir(abspath):
-                dirs.append(fn)
+            dirs.append(quotep(fn))

        for x in vfs_virt.keys():
            if x != excl:
@@ -1168,7 +1176,9 @@ class HttpCli(object):

            return self.tx_file(abspath)

-        fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)
+        fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
+        stats = {k: v for k, v in vfs_ls}
+        vfs_ls = [x[0] for x in vfs_ls]
        vfs_ls.extend(vfs_virt.keys())

        # check for old versions of files,
@@ -1219,7 +1229,7 @@ class HttpCli(object):
                fspath = fsroot + "/" + fn

            try:
-                inf = os.stat(fsenc(fspath))
+                inf = stats.get(fn) or os.stat(fsenc(fspath))
            except:
                self.log("broken symlink: {}".format(repr(fspath)))
                continue
@@ -1251,7 +1261,7 @@ class HttpCli(object):
                "sz": sz,
                "ext": ext,
                "dt": dt,
-                "ts": inf.st_mtime,
+                "ts": int(inf.st_mtime),
            }
            if is_dir:
                dirs.append(item)
@@ -1266,19 +1276,24 @@ class HttpCli(object):
            del f["rd"]
            if icur:
                q = "select w from up where rd = ? and fn = ?"
-                r = icur.execute(q, (rd, fn)).fetchone()
+                try:
+                    r = icur.execute(q, (rd, fn)).fetchone()
+                except:
+                    args = s3enc(idx.mem_cur, rd, fn)
+                    r = icur.execute(q, args).fetchone()
+
+                tags = {}
+                f["tags"] = tags
+                
                if not r:
                    continue

                w = r[0][:16]
-                tags = {}
                q = "select k, v from mt where w = ? and k != 'x'"
                for k, v in icur.execute(q, (w,)):
                    taglist[k] = True
                    tags[k] = v

-                f["tags"] = tags
-
        if icur:
            taglist = [k for k in self.args.mte.split(",") if k in taglist]
            for f in dirs:
@@ -1288,7 +1303,7 @@ class HttpCli(object):

        try:
            if not self.args.nih:
-                srv_info.append(str(socket.gethostname()).split(".")[0])
+                srv_info.append(unicode(socket.gethostname()).split(".")[0])
        except:
            self.log("#wow #whoa")
            pass
--- a/copyparty/httpconn.py
+++ b/copyparty/httpconn.py
@@ -81,8 +81,8 @@ class HttpConn(object):
    def respath(self, res_name):
        return os.path.join(E.mod, "web", res_name)

-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)

    def get_u2idx(self):
        if not self.u2idx:
@@ -129,7 +129,7 @@ class HttpConn(object):

        if is_https:
            if self.sr:
-                self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
+                self.log("TODO: cannot do https in jython", c="1;31")
                return

            self.log_src = self.log_src.replace("[36m", "[35m")
@@ -180,7 +180,7 @@ class HttpConn(object):
                    pass

                else:
-                    self.log("\033[35mhandshake\033[0m " + em)
+                    self.log("handshake\033[0m " + em, c=5)

                return

--- a/copyparty/httpsrv.py
+++ b/copyparty/httpsrv.py
@@ -38,7 +38,7 @@ class HttpSrv(object):

    def accept(self, sck, addr):
        """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log("%s %s" % addr, "\033[1;30m|%sC-cthr\033[0m" % ("-" * 5,))
+        self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
        thr = threading.Thread(target=self.thr_client, args=(sck, addr))
        thr.daemon = True
        thr.start()
@@ -66,11 +66,11 @@ class HttpSrv(object):
                thr.start()

        try:
-            self.log("%s %s" % addr, "\033[1;30m|%sC-crun\033[0m" % ("-" * 6,))
+            self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
            cli.run()

        finally:
-            self.log("%s %s" % addr, "\033[1;30m|%sC-cdone\033[0m" % ("-" * 7,))
+            self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
            try:
                sck.shutdown(socket.SHUT_RDWR)
                sck.close()
@@ -78,7 +78,8 @@ class HttpSrv(object):
                if not MACOS:
                    self.log(
                        "%s %s" % addr,
-                        "\033[1;30mshut({}): {}\033[0m".format(sck.fileno(), ex),
+                        "shut({}): {}".format(sck.fileno(), ex),
+                        c="1;30",
                    )
                if ex.errno not in [10038, 10054, 107, 57, 9]:
                    # 10038 No longer considered a socket
--- a/copyparty/mtag.py
+++ b/copyparty/mtag.py
@@ -1,6 +1,5 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
-from math import fabs

 import re
 import os
@@ -11,27 +10,33 @@ import subprocess as sp
 from .__init__ import PY2, WINDOWS
 from .util import fsenc, fsdec

+if not PY2:
+    unicode = str
+

 class MTag(object):
    def __init__(self, log_func, args):
        self.log_func = log_func
        self.usable = True
+        self.prefer_mt = False
        mappings = args.mtm
-        backend = "ffprobe" if args.no_mutagen else "mutagen"
+        self.backend = "ffprobe" if args.no_mutagen else "mutagen"
+        or_ffprobe = " or ffprobe"

-        if backend == "mutagen":
+        if self.backend == "mutagen":
            self.get = self.get_mutagen
            try:
                import mutagen
            except:
-                self.log("\033[33mcould not load mutagen, trying ffprobe instead")
-                backend = "ffprobe"
+                self.log("could not load mutagen, trying ffprobe instead", c=3)
+                self.backend = "ffprobe"

-        if backend == "ffprobe":
+        if self.backend == "ffprobe":
            self.get = self.get_ffprobe
+            self.prefer_mt = True
            # about 20x slower
            if PY2:
-                cmd = ["ffprobe", "-version"]
+                cmd = [b"ffprobe", b"-version"]
                try:
                    sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
                except:
@@ -40,9 +45,15 @@ class MTag(object):
                if not shutil.which("ffprobe"):
                    self.usable = False

+            if self.usable and WINDOWS and sys.version_info < (3, 8):
+                self.usable = False
+                or_ffprobe = " or python >= 3.8"
+                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                self.log(msg, c=1)
+
        if not self.usable:
-            msg = "\033[31mneed mutagen or ffprobe to read media tags so please run this:\n  {} -m pip install --user mutagen \033[0m"
-            self.log(msg.format(os.path.basename(sys.executable)))
+            msg = "need mutagen{} to read media tags so please run this:\n  {} -m pip install --user mutagen"
+            self.log(msg.format(or_ffprobe, os.path.basename(sys.executable)), c=1)
            return

        # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -114,8 +125,8 @@ class MTag(object):
        }
        # self.get = self.compare

-    def log(self, msg):
-        self.log_func("mtag", msg)
+    def log(self, msg, c=0):
+        self.log_func("mtag", msg, c)

    def normalize_tags(self, ret, md):
        for k, v in dict(md).items():
@@ -132,7 +143,7 @@ class MTag(object):
                ret[mk] = [pref, v[0]]

        # take first value
-        ret = {k: str(v[1]).strip() for k, v in ret.items()}
+        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}

        # track 3/7 => track 3
        for k, v in ret.items():
@@ -205,7 +216,7 @@ class MTag(object):
        return self.normalize_tags(ret, md)

    def get_ffprobe(self, abspath):
-        cmd = ["ffprobe", "-hide_banner", "--", fsenc(abspath)]
+        cmd = [b"ffprobe", b"-hide_banner", b"--", fsenc(abspath)]
        p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
        r = p.communicate()
        txt = r[1].decode("utf-8", "replace")
@@ -284,9 +295,7 @@ class MTag(object):
                            sec *= 60
                            sec += int(f)
                    except:
-                        self.log(
-                            "\033[33minvalid timestr from ffmpeg: [{}]".format(tstr)
-                        )
+                        self.log("invalid timestr from ffmpeg: [{}]".format(tstr), c=3)

                ret[".dur"] = sec
                m = ptn_br1.search(ln)
--- a/copyparty/svchub.py
+++ b/copyparty/svchub.py
@@ -9,7 +9,6 @@ from datetime import datetime, timedelta
 import calendar

 from .__init__ import PY2, WINDOWS, MACOS, VT100
-from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
 from .util import mp
@@ -66,10 +65,10 @@ class SvcHub(object):
            self.broker.shutdown()
            print("nailed it")

-    def _log_disabled(self, src, msg):
+    def _log_disabled(self, src, msg, c=0):
        pass

-    def _log_enabled(self, src, msg):
+    def _log_enabled(self, src, msg, c=0):
        """handles logging from all components"""
        with self.log_mutex:
            now = time.time()
@@ -92,6 +91,13 @@ class SvcHub(object):
                    msg = self.ansi_re.sub("", msg)
                if "\033" in src:
                    src = self.ansi_re.sub("", src)
+            elif c:
+                if isinstance(c, int):
+                    msg = "\033[3{}m{}".format(c, msg)
+                elif "\033" not in c:
+                    msg = "\033[{}m{}\033[0m".format(c, msg)
+                else:
+                    msg = "{}{}\033[0m".format(c, msg)

            ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
            msg = fmt.format(ts, src, msg)
--- a/copyparty/tcpsrv.py
+++ b/copyparty/tcpsrv.py
@@ -68,21 +68,22 @@ class TcpSrv(object):
            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))

        while True:
-            self.log("tcpsrv", "\033[1;30m|%sC-ncli\033[0m" % ("-" * 1,))
+            self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
            if self.num_clients.v >= self.args.nc:
                time.sleep(0.1)
                continue

-            self.log("tcpsrv", "\033[1;30m|%sC-acc1\033[0m" % ("-" * 2,))
+            self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
            ready, _, _ = select.select(self.srv, [], [])
            for srv in ready:
                sck, addr = srv.accept()
                sip, sport = srv.getsockname()
                self.log(
                    "%s %s" % addr,
-                    "\033[1;30m|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                    "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
                        "-" * 3, sip, sport % 8, sport
                    ),
+                    c="1;30",
                )
                self.num_clients.add()
                self.hub.broker.put(False, "httpconn", sck, addr)
--- a/copyparty/u2idx.py
+++ b/copyparty/u2idx.py
@@ -25,9 +25,11 @@ class U2idx(object):
            return

        self.cur = {}
+        self.mem_cur = sqlite3.connect(":memory:")
+        self.mem_cur.execute(r"create table a (b text)")

-    def log(self, msg):
-        self.log_func("u2idx", msg)
+    def log(self, msg, c=0):
+        self.log_func("u2idx", msg, c)

    def fsearch(self, vols, body):
        """search by up2k hashlist"""
@@ -37,7 +39,11 @@ class U2idx(object):
        fsize = body["size"]
        fhash = body["hash"]
        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
-        return self.run_query(vols, "w = ?", [wark], "", [])[0]
+
+        uq = "substr(w,1,16) = ? and w = ?"
+        uv = [wark[:16], wark]
+
+        return self.run_query(vols, uq, uv, "", [])[0]

    def get_cur(self, ptop):
        cur = self.cur.get(ptop)
@@ -108,6 +114,9 @@ class U2idx(object):
                if lim <= 0:
                    break

+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
                rp = os.path.join(vtop, rd, fn).replace("\\", "/")
                sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})

--- a/copyparty/up2k.py
+++ b/copyparty/up2k.py
@@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals

 import re
 import os
-import sys
 import time
 import math
 import json
@@ -26,8 +25,9 @@ from .util import (
    sanitize_fn,
    ren_open,
    atomic_move,
-    w8b64enc,
-    w8b64dec,
+    s3enc,
+    s3dec,
+    statdir,
 )
 from .mtag import MTag
 from .authsrv import AuthSrv
@@ -51,23 +51,31 @@ class Up2k(object):
        self.broker = broker
        self.args = broker.args
        self.log_func = broker.log
-        self.persist = self.args.e2d

        # config
        self.salt = broker.args.salt

        # state
        self.mutex = threading.Lock()
+        self.hashq = Queue()
+        self.tagq = Queue()
        self.registry = {}
        self.entags = {}
        self.flags = {}
        self.cur = {}
+        self.mtag = None
+        self.n_mtag_tags_added = -1

        self.mem_cur = None
+        self.sqlite_ver = None
+        self.no_expr_idx = False
        if HAVE_SQLITE3:
            # mojibake detector
            self.mem_cur = self._orz(":memory:")
            self.mem_cur.execute(r"create table a (b text)")
+            self.sqlite_ver = tuple([int(x) for x in sqlite3.sqlite_version.split(".")])
+            if self.sqlite_ver < (3, 9):
+                self.no_expr_idx = True

        if WINDOWS:
            # usually fails to set lastmod too quickly
@@ -76,50 +84,31 @@ class Up2k(object):
            thr.daemon = True
            thr.start()

-        self.mtag = MTag(self.log_func, self.args)
-        if not self.mtag.usable:
-            self.mtag = None
-
        # static
        self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$")

-        if self.persist and not HAVE_SQLITE3:
+        if not HAVE_SQLITE3:
            self.log("could not initialize sqlite3, will use in-memory registry only")

        # this is kinda jank
-        auth = AuthSrv(self.args, self.log, False)
-        self.init_indexes(auth)
+        auth = AuthSrv(self.args, self.log_func, False)
+        have_e2d = self.init_indexes(auth)

-        if self.persist:
+        if have_e2d:
            thr = threading.Thread(target=self._snapshot)
            thr.daemon = True
            thr.start()

-    def log(self, msg):
-        self.log_func("up2k", msg + "\033[K")
+            thr = threading.Thread(target=self._tagger)
+            thr.daemon = True
+            thr.start()

-    def w8enc(self, rd, fn):
-        ret = []
-        for v in [rd, fn]:
-            try:
-                self.mem_cur.execute("select * from a where b = ?", (v,))
-                ret.append(v)
-            except:
-                ret.append("//" + w8b64enc(v))
-                # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+            thr = threading.Thread(target=self._hasher)
+            thr.daemon = True
+            thr.start()

-        return tuple(ret)
-
-    def w8dec(self, rd, fn):
-        ret = []
-        for k, v in [["d", rd], ["f", fn]]:
-            if v.startswith("//"):
-                ret.append(w8b64dec(v[2:]))
-                # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
-            else:
-                ret.append(v)
-
-        return tuple(ret)
+    def log(self, msg, c=0):
+        self.log_func("up2k", msg + "\033[K", c)

    def _vis_job_progress(self, job):
        perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
@@ -133,10 +122,33 @@ class Up2k(object):

        return ret

+    def _expr_idx_filter(self, flags):
+        if not self.no_expr_idx:
+            return False, flags
+
+        ret = {k: v for k, v in flags.items() if not k.startswith("e2t")}
+        if ret.keys() == flags.keys():
+            return False, flags
+
+        return True, ret
+
    def init_indexes(self, auth):
        self.pp = ProgressPrinter()
        vols = auth.vfs.all_vols.values()
        t0 = time.time()
+        have_e2d = False
+
+        if self.no_expr_idx:
+            modified = False
+            for vol in vols:
+                m, f = self._expr_idx_filter(vol.flags)
+                if m:
+                    vol.flags = f
+                    modified = True
+
+            if modified:
+                msg = "disabling -e2t because your sqlite belongs in a museum"
+                self.log(msg, c=3)

        live_vols = []
        for vol in vols:
@@ -144,10 +156,20 @@ class Up2k(object):
                os.listdir(vol.realpath)
                live_vols.append(vol)
            except:
-                self.log("\033[31mcannot access " + vol.realpath)
+                self.log("cannot access " + vol.realpath, c=1)

        vols = live_vols

+        need_mtag = False
+        for vol in vols:
+            if "e2t" in vol.flags:
+                need_mtag = True
+
+        if need_mtag:
+            self.mtag = MTag(self.log_func, self.args)
+            if not self.mtag.usable:
+                self.mtag = None
+
        # e2ds(a) volumes first,
        # also covers tags where e2ts is set
        for vol in vols:
@@ -157,6 +179,9 @@ class Up2k(object):

            self.entags[vol.realpath] = en

+            if "e2d" in vol.flags:
+                have_e2d = True
+
            if "e2ds" in vol.flags:
                r = self._build_file_index(vol, vols)
                if not r:
@@ -182,17 +207,21 @@ class Up2k(object):
        self.log(msg.format(len(vols), time.time() - t0))

        if needed_mutagen:
-            msg = "\033[31mcould not read tags because no backends are available (mutagen or ffprobe)\033[0m"
-            self.log(msg)
+            msg = "could not read tags because no backends are available (mutagen or ffprobe)"
+            self.log(msg, c=1)
+
+        return have_e2d

    def register_vpath(self, ptop, flags):
        with self.mutex:
            if ptop in self.registry:
                return None

+            _, flags = self._expr_idx_filter(flags)
+
            reg = {}
            path = os.path.join(ptop, ".hist", "up2k.snap")
-            if self.persist and os.path.exists(path):
+            if "e2d" in flags and os.path.exists(path):
                with gzip.GzipFile(path, "rb") as f:
                    j = f.read().decode("utf-8")

@@ -206,7 +235,7 @@ class Up2k(object):

            self.flags[ptop] = flags
            self.registry[ptop] = reg
-            if not self.persist or not HAVE_SQLITE3 or "d2d" in flags:
+            if not HAVE_SQLITE3 or "e2d" not in flags or "d2d" in flags:
                return None

            try:
@@ -269,23 +298,12 @@ class Up2k(object):
        self.log(msg)

    def _build_dir(self, dbw, top, excl, cdir):
-        try:
-            inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
-        except Exception as ex:
-            self.log("listdir: {} @ [{}]".format(repr(ex), cdir))
-            return 0
-
        self.pp.msg = "a{} {}".format(self.pp.n, cdir)
        histdir = os.path.join(top, ".hist")
        ret = 0
-        for inode in inodes:
-            abspath = os.path.join(cdir, inode)
-            try:
-                inf = os.stat(fsenc(abspath))
-            except Exception as ex:
-                self.log("stat: {} @ [{}]".format(repr(ex), abspath))
-                continue
-
+        for iname, inf in statdir(self.log, not self.args.no_scandir, False, cdir):
+            abspath = os.path.join(cdir, iname)
+            lmod = int(inf.st_mtime)
            if stat.S_ISDIR(inf.st_mode):
                if abspath in excl or abspath == histdir:
                    continue
@@ -299,7 +317,7 @@ class Up2k(object):
                try:
                    c = dbw[0].execute(sql, (rd, fn))
                except:
-                    c = dbw[0].execute(sql, self.w8enc(rd, fn))
+                    c = dbw[0].execute(sql, s3enc(self.mem_cur, rd, fn))

                in_db = list(c.fetchall())
                if in_db:
@@ -311,11 +329,11 @@ class Up2k(object):
                        self.log(m.format(top, rp, len(in_db), rep_db))
                        dts = -1

-                    if dts == inf.st_mtime and dsz == inf.st_size:
+                    if dts == lmod and dsz == inf.st_size:
                        continue

                    m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format(
-                        top, rp, dts, inf.st_mtime, dsz, inf.st_size
+                        top, rp, dts, lmod, dsz, inf.st_size
                    )
                    self.log(m)
                    self.db_rm(dbw[0], rd, fn)
@@ -334,7 +352,7 @@ class Up2k(object):
                    continue

                wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
-                self.db_add(dbw[0], wark, rd, fn, inf.st_mtime, inf.st_size)
+                self.db_add(dbw[0], wark, rd, fn, lmod, inf.st_size)
                dbw[1] += 1
                ret += 1
                td = time.time() - dbw[2]
@@ -353,7 +371,7 @@ class Up2k(object):
        for dwark, dts, dsz, drd, dfn in c:
            nchecked += 1
            if drd.startswith("//") or dfn.startswith("//"):
-                drd, dfn = self.w8dec(drd, dfn)
+                drd, dfn = s3dec(drd, dfn)

            abspath = os.path.join(top, drd, dfn)
            # almost zero overhead dw
@@ -415,7 +433,23 @@ class Up2k(object):
            if not self.mtag:
                return n_add, n_rm, False

+            mpool = False
+            if self.mtag.prefer_mt and not self.args.no_mtag_mt:
+                # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
+                # both do crazy runahead so lets reinvent another wheel
+                nw = os.cpu_count() if hasattr(os, "cpu_count") else 4
+                if self.n_mtag_tags_added == -1:
+                    self.log("using {}x {}".format(nw, self.mtag.backend))
+                    self.n_mtag_tags_added = 0
+
+                mpool = Queue(nw)
+                for _ in range(nw):
+                    thr = threading.Thread(target=self._tag_thr, args=(mpool,))
+                    thr.daemon = True
+                    thr.start()
+
            c2 = cur.connection.cursor()
+            c3 = cur.connection.cursor()
            n_left = cur.execute("select count(w) from up").fetchone()[0]
            for w, rd, fn in cur.execute("select w, rd, fn from up"):
                n_left -= 1
@@ -423,19 +457,22 @@ class Up2k(object):
                if c2.execute(q, (w[:16],)).fetchone():
                    continue

+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
                abspath = os.path.join(ptop, rd, fn)
                self.pp.msg = "c{} {}".format(n_left, abspath)
-                tags = self.mtag.get(abspath)
-                tags = {k: v for k, v in tags.items() if k in entags}
-                if not tags:
-                    # indicate scanned without tags
-                    tags = {"x": 0}
+                args = c3, entags, w, abspath
+                if not mpool:
+                    n_tags = self._tag_file(*args)
+                else:
+                    mpool.put(args)
+                    with self.mutex:
+                        n_tags = self.n_mtag_tags_added
+                        self.n_mtag_tags_added = 0

-                for k, v in tags.items():
-                    q = "insert into mt values (?,?,?)"
-                    c2.execute(q, (w[:16], k, v))
-                    n_add += 1
-                    n_buf += 1
+                n_add += n_tags
+                n_buf += n_tags

                td = time.time() - last_write
                if n_buf >= 4096 or td >= 60:
@@ -444,10 +481,52 @@ class Up2k(object):
                    last_write = time.time()
                    n_buf = 0

+            if mpool:
+                for _ in range(mpool.maxsize):
+                    mpool.put(None)
+
+                mpool.join()
+
+            c3.close()
            c2.close()

        return n_add, n_rm, True

+    def _tag_thr(self, q):
+        while True:
+            task = q.get()
+            if not task:
+                q.task_done()
+                return
+
+            try:
+                write_cur, entags, wark, abspath = task
+                tags = self.mtag.get(abspath)
+                with self.mutex:
+                    n = self._tag_file(write_cur, entags, wark, abspath, tags)
+                    self.n_mtag_tags_added += n
+            except:
+                ex = traceback.format_exc()
+                msg = "{} failed to read tags from {}:\n{}"
+                self.log(msg.format(self.mtag.backend, abspath, ex), c=3)
+
+            q.task_done()
+
+    def _tag_file(self, write_cur, entags, wark, abspath, tags=None):
+        tags = tags or self.mtag.get(abspath)
+        tags = {k: v for k, v in tags.items() if k in entags}
+        if not tags:
+            # indicate scanned without tags
+            tags = {"x": 0}
+
+        ret = 0
+        for k, v in tags.items():
+            q = "insert into mt values (?,?,?)"
+            write_cur.execute(q, (wark[:16], k, v))
+            ret += 1
+
+        return ret
+
    def _orz(self, db_path):
        return sqlite3.connect(db_path, check_same_thread=False).cursor()

@@ -545,8 +624,12 @@ class Up2k(object):
                except:
                    pass

+        idx = r"create index up_w on up(substr(w,1,16))"
+        if self.no_expr_idx:
+            idx = r"create index up_w on up(w)"
+
        for cmd in [
-            r"create index up_w on up(substr(w,1,16))",
+            idx,
            r"create table mt (w text, k text, v int)",
            r"create index mt_w on mt(w)",
            r"create index mt_k on mt(k)",
@@ -591,12 +674,17 @@ class Up2k(object):
            cur = self.cur.get(cj["ptop"], None)
            reg = self.registry[cj["ptop"]]
            if cur:
-                q = r"select * from up where substr(w,1,16) = ? and w = ?"
-                argv = (wark[:16], wark)
+                if self.no_expr_idx:
+                    q = r"select * from up where w = ?"
+                    argv = (wark,)
+                else:
+                    q = r"select * from up where substr(w,1,16) = ? and w = ?"
+                    argv = (wark[:16], wark)
+
                cur = cur.execute(q, argv)
                for _, dtime, dsize, dp_dir, dp_fn in cur:
                    if dp_dir.startswith("//") or dp_fn.startswith("//"):
-                        dp_dir, dp_fn = self.w8dec(dp_dir, dp_fn)
+                        dp_dir, dp_fn = s3dec(dp_dir, dp_fn)

                    dp_abs = os.path.join(cj["ptop"], dp_dir, dp_fn).replace("\\", "/")
                    # relying on path.exists to return false on broken symlinks
@@ -718,8 +806,13 @@ class Up2k(object):
                raise OSError()
            elif fs1 == fs2:
                # same fs; make symlink as relative as possible
-                nsrc = src.replace("\\", "/").split("/")
-                ndst = dst.replace("\\", "/").split("/")
+                v = []
+                for p in [src, dst]:
+                    if WINDOWS:
+                        p = p.replace("\\", "/")
+                    v.append(p.split("/"))
+
+                nsrc, ndst = v
                nc = 0
                for a, b in zip(nsrc, ndst):
                    if a != b:
@@ -727,7 +820,8 @@ class Up2k(object):
                    nc += 1
                if nc > 1:
                    lsrc = nsrc[nc:]
-                    lsrc = "../" * (len(lsrc) - 1) + "/".join(lsrc)
+                    hops = len(ndst[nc:]) - 1
+                    lsrc = "../" * hops + "/".join(lsrc)
            os.symlink(fsenc(lsrc), fsenc(ldst))
        except (AttributeError, OSError) as ex:
            self.log("cannot symlink; creating copy: " + repr(ex))
@@ -779,24 +873,40 @@ class Up2k(object):
            if WINDOWS:
                self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))])

-            cur = self.cur.get(job["ptop"], None)
-            if cur:
-                j = job
-                self.db_rm(cur, j["prel"], j["name"])
-                self.db_add(cur, j["wark"], j["prel"], j["name"], j["lmod"], j["size"])
-                cur.connection.commit()
-
+            # legit api sware 2 me mum
+            if self.idx_wark(
+                job["ptop"],
+                job["wark"],
+                job["prel"],
+                job["name"],
+                job["lmod"],
+                job["size"],
+            ):
                del self.registry[ptop][wark]
                # in-memory registry is reserved for unfinished uploads

-            return ret, dst
+        return ret, dst
+
+    def idx_wark(self, ptop, wark, rd, fn, lmod, sz):
+        cur = self.cur.get(ptop, None)
+        if not cur:
+            return False
+
+        self.db_rm(cur, rd, fn)
+        self.db_add(cur, wark, rd, fn, int(lmod), sz)
+        cur.connection.commit()
+
+        if "e2t" in self.flags[ptop]:
+            self.tagq.put([ptop, wark, rd, fn])
+
+        return True

    def db_rm(self, db, rd, fn):
        sql = "delete from up where rd = ? and fn = ?"
        try:
            db.execute(sql, (rd, fn))
        except:
-            db.execute(sql, self.w8enc(rd, fn))
+            db.execute(sql, s3enc(self.mem_cur, rd, fn))

    def db_add(self, db, wark, rd, fn, ts, sz):
        sql = "insert into up values (?,?,?,?,?)"
@@ -804,7 +914,7 @@ class Up2k(object):
        try:
            db.execute(sql, v)
        except:
-            rd, fn = self.w8enc(rd, fn)
+            rd, fn = s3enc(self.mem_cur, rd, fn)
            v = (wark, ts, sz, rd, fn)
            db.execute(sql, v)

@@ -833,7 +943,7 @@ class Up2k(object):
        ret = []
        with open(path, "rb", 512 * 1024) as f:
            while fsz > 0:
-                self.pp.msg = "{} MB".format(int(fsz / 1024 / 1024))
+                self.pp.msg = "{} MB, {}".format(int(fsz / 1024 / 1024), path)
                hashobj = hashlib.sha512()
                rem = min(csz, fsz)
                fsz -= rem
@@ -940,6 +1050,45 @@ class Up2k(object):
        self.log("snap: {} |{}|".format(path, len(reg.keys())))
        prev[k] = etag

+    def _tagger(self):
+        while True:
+            ptop, wark, rd, fn = self.tagq.get()
+            abspath = os.path.join(ptop, rd, fn)
+            self.log("tagging " + abspath)
+            with self.mutex:
+                cur = self.cur[ptop]
+                if not cur:
+                    self.log("no cursor to write tags with??", c=1)
+                    continue
+
+                entags = self.entags[ptop]
+                if not entags:
+                    self.log("no entags okay.jpg", c=3)
+                    continue
+
+                if "e2t" in self.flags[ptop]:
+                    self._tag_file(cur, entags, wark, abspath)
+
+                cur.connection.commit()
+
+    def _hasher(self):
+        while True:
+            ptop, rd, fn = self.hashq.get()
+            if "e2d" not in self.flags[ptop]:
+                continue
+
+            abspath = os.path.join(ptop, rd, fn)
+            self.log("hashing " + abspath)
+            inf = os.stat(fsenc(abspath))
+            hashes = self._hashlist_from_file(abspath)
+            wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
+            with self.mutex:
+                self.idx_wark(ptop, wark, rd, fn, inf.st_mtime, inf.st_size)
+
+    def hash_file(self, ptop, flags, rd, fn):
+        self.register_vpath(ptop, flags)
+        self.hashq.put([ptop, rd, fn])
+

 def up2k_chunksize(filesize):
    chunksize = 1024 * 1024
--- a/copyparty/util.py
+++ b/copyparty/util.py
@@ -119,19 +119,26 @@ class ProgressPrinter(threading.Thread):
                continue

            msg = self.msg
-            m = " {}\033[K\r".format(msg)
-            try:
-                print(m, end="")
-            except UnicodeEncodeError:
-                try:
-                    print(m.encode("utf-8", "replace").decode(), end="")
-                except:
-                    print(m.encode("ascii", "replace").decode(), end="")
+            uprint(" {}\033[K\r".format(msg))

        print("\033[K", end="")
        sys.stdout.flush()  # necessary on win10 even w/ stderr btw


+def uprint(msg):
+    try:
+        print(msg, end="")
+    except UnicodeEncodeError:
+        try:
+            print(msg.encode("utf-8", "replace").decode(), end="")
+        except:
+            print(msg.encode("ascii", "replace").decode(), end="")
+
+
+def nuprint(msg):
+    uprint("{}\n".format(msg))
+
+
@contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
    fdir = kwargs.pop("fdir", None)
@@ -521,9 +528,7 @@ def u8safe(txt):


 def exclude_dotfiles(filepaths):
-    for fpath in filepaths:
-        if not fpath.split("/")[-1].startswith("."):
-            yield fpath
+    return [x for x in filepaths if not x.split("/")[-1].startswith(".")]


 def html_escape(s, quote=False):
@@ -599,6 +604,31 @@ else:
    fsdec = w8dec


+def s3enc(mem_cur, rd, fn):
+    ret = []
+    for v in [rd, fn]:
+        try:
+            mem_cur.execute("select * from a where b = ?", (v,))
+            ret.append(v)
+        except:
+            ret.append("//" + w8b64enc(v))
+            # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+    return tuple(ret)
+
+
+def s3dec(rd, fn):
+    ret = []
+    for k, v in [["d", rd], ["f", fn]]:
+        if v.startswith("//"):
+            ret.append(w8b64dec(v[2:]))
+            # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+        else:
+            ret.append(v)
+
+    return tuple(ret)
+
+
 def atomic_move(src, dst):
    if not PY2:
        os.replace(src, dst)
@@ -726,6 +756,33 @@ def sendfile_kern(lower, upper, f, s):
    return 0


+def statdir(logger, scandir, lstat, top):
+    try:
+        btop = fsenc(top)
+        if scandir and hasattr(os, "scandir"):
+            src = "scandir"
+            with os.scandir(btop) as dh:
+                for fh in dh:
+                    try:
+                        yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
+                    except Exception as ex:
+                        msg = "scan-stat: \033[36m{} @ {}"
+                        logger(msg.format(repr(ex), fsdec(fh.path)))
+        else:
+            src = "listdir"
+            fun = os.lstat if lstat else os.stat
+            for name in os.listdir(btop):
+                abspath = os.path.join(btop, name)
+                try:
+                    yield [fsdec(name), fun(abspath)]
+                except Exception as ex:
+                    msg = "list-stat: \033[36m{} @ {}"
+                    logger(msg.format(repr(ex), fsdec(abspath)))
+
+    except Exception as ex:
+        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
+
+
 def unescape_cookie(orig):
    # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
    ret = ""
@@ -780,7 +837,11 @@ def chkcmd(*argv):
 def gzip_orig_sz(fn):
    with open(fsenc(fn), "rb") as f:
        f.seek(-4, 2)
-        return struct.unpack(b"I", f.read(4))[0]
+        rv = f.read(4)
+        try:
+            return struct.unpack(b"I", rv)[0]
+        except:
+            return struct.unpack("I", rv)[0]


 def py_desc():
@@ -790,7 +851,11 @@ def py_desc():
    if ofs > 0:
        py_ver = py_ver[:ofs]

-    bitness = struct.calcsize(b"P") * 8
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
    host_os = platform.system()
    compiler = platform.python_compiler()

--- a/copyparty/web/browser.css
+++ b/copyparty/web/browser.css
@@ -42,12 +42,8 @@ body {
 #path #entree {
 	margin-left: -.7em;
 }
-#treetab {
-	display: none;
-}
 #files {
 	border-spacing: 0;
-	margin-top: 2em;
 	z-index: 1;
 	position: relative;
 }
@@ -55,11 +51,10 @@ body {
 	display: block;
 	padding: .3em 0;
 }
-#files[ts] tbody div a {
+#files tbody div a {
 	color: #f5a;
 }
-a,
-#files[ts] tbody div a:last-child {
+a, #files tbody div a:last-child {
 	color: #fc5;
 	padding: .2em;
 	text-decoration: none;
@@ -67,16 +62,18 @@ a,
 #files a:hover {
 	color: #fff;
 	background: #161616;
+	text-decoration: underline;
 }
 #files thead a {
 	color: #999;
 	font-weight: normal;
 }
-#files tr:hover {
+#files tr+tr:hover {
 	background: #1c1c1c;
 }
 #files thead th {
 	padding: .5em 1.3em .3em 1.3em;
+	cursor: pointer;
 }
 #files thead th:last-child {
 	background: #444;
@@ -156,6 +153,15 @@ a,
 .logue {
 	padding: .2em 1.5em;
 }
+.logue:empty {
+	display: none;
+}
+#pro.logue {
+	margin-bottom: .8em;
+}
+#epi.logue {
+	margin: .8em 0;
+}
 #srv_info {
 	opacity: .5;
 	font-size: .8em;
@@ -305,11 +311,11 @@ a,
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
-@media (min-width: 100em) {
+@media (min-width: 90em) {
 	#barpos,
 	#barbuf {
 		width: calc(100% - 24em);
-		left: 10em;
+		left: 9.8em;
 		top: .7em;
 		height: 1.6em;
 		bottom: auto;
@@ -441,25 +447,62 @@ input[type="checkbox"]:checked+label {
 #files td div a:last-child {
 	width: 100%;
 }
-#tree,
-#treefiles {
-	vertical-align: top;
+#wrap {
+	margin-top: 2em;
 }
 #tree {
-	padding-top: 2em;
+	display: none;
+	position: fixed;
+	left: 0;
+	bottom: 0;
+	top: 7em;
+	padding-top: .2em;
+	overflow-y: auto;
+	-ms-scroll-chaining: none;
+	overscroll-behavior-y: none;
+	scrollbar-color: #eb0 #333;
+}
+#thx_ff {
+	padding: 5em 0;
+}
+#tree::-webkit-scrollbar-track {
+	background: #333;
+}
+#tree::-webkit-scrollbar {
+	background: #333;
+}
+#tree::-webkit-scrollbar-thumb {
+	background: #eb0;
+}
+#tree:hover {
+	z-index: 2;
+}
+#treeul {
+	position: relative;
+	left: -1.7em;
+	width: calc(100% + 1.3em);
+}
+#tree>a+a {
+	padding: .2em .4em;
+	font-size: 1.2em;
+	background: #2a2a2a;
+	box-shadow: 0 .1em .2em #222 inset;
+	border-radius: .3em;
+	margin: .2em;
+	position: relative;
+	top: -.2em;
+}
+#tree>a+a:hover {
+	background: #805;
+}
+#tree>a+a.on {
+	background: #fc4;
+	color: #400;
+	text-shadow: none;
 }
 #detree {
 	padding: .3em .5em;
 	font-size: 1.5em;
-	display: inline-block;
-	min-width: 12em;
-	width: 100%;
-}
-#treefiles #files tbody {
-	border-radius: 0 .7em 0 .7em;
-}
-#treefiles #files thead th:nth-child(1) {
-	border-radius: .7em 0 0 0;
 }
 #tree ul,
 #tree li {
@@ -467,43 +510,35 @@ input[type="checkbox"]:checked+label {
 	margin: 0;
 }
 #tree ul {
-	border-left: .2em solid #444;
+	border-left: .2em solid #555;
 }
 #tree li {
 	margin-left: 1em;
 	list-style: none;
-	white-space: nowrap;
+	border-top: 1px solid #4c4c4c;
+	border-bottom: 1px solid #222;
 }
-#tree a.hl {
+#tree li:last-child {
+	border-bottom: none;
+}
+#treeul a.hl {
 	color: #400;
 	background: #fc4;
 	border-radius: .3em;
 	text-shadow: none;
 }
-#tree a {
+#treeul a {
 	display: inline-block;
 }
-#tree a+a {
+#treeul a+a {
 	width: calc(100% - 2em);
 	background: #333;
+	line-height: 1em;
 }
-#tree a+a:hover {
+#treeul a+a:hover {
 	background: #222;
 	color: #fff;
 }
-#treeul {
-	position: relative;
-	overflow: hidden;
-	left: -1.7em;
-}
-#treeul:hover {
-	z-index: 2;
-	overflow: visible;
-}
-#treeul:hover a+a {
-	width: auto;
-	min-width: calc(100% - 2em);
-}
 #treeul a:first-child {
 	font-family: monospace, monospace;
 }
@@ -535,7 +570,7 @@ input[type="checkbox"]:checked+label {
 #files>thead>tr>th.min span {
 	position: absolute;
 	transform: rotate(270deg);
-	background: linear-gradient(90deg, #222, #444);
+	background: linear-gradient(90deg, rgba(68,68,68,0), rgba(68,68,68,0.5) 70%, #444);
 	margin-left: -4.6em;
 	padding: .4em;
 	top: 5.4em;
@@ -554,4 +589,46 @@ input[type="checkbox"]:checked+label {
 	border-color: transparent;
 	color: #400;
 	text-shadow: none;
+}
+#files tr.play a {
+	color: inherit;
+}
+#files tr.play a:hover {
+	color: #300;
+	background: #fea;
+}
+#op_cfg {
+	max-width: none;
+	margin-right: 1.5em;
+}
+#key_notation>span {
+	display: inline-block;
+	padding: .2em .4em;
+}
+#op_cfg h3 {
+	margin: .8em 0 0 .6em;
+	padding: 0;
+	border-bottom: 1px solid #555;
+}
+#opdesc {
+	display: none;
+}
+#ops:hover #opdesc {
+	display: block;
+	background: linear-gradient(0deg,#555, #4c4c4c 80%, #444);
+	box-shadow: 0 .3em 1em #222;
+	padding: 1em;
+	border-radius: .3em;
+	position: absolute;
+	z-index: 3;
+	top: 6em;
+	right: 1.5em;
+}
+#opdesc code {
+	background: #3c3c3c;
+	padding: .2em .3em;
+	border-top: 1px solid #777;
+	border-radius: .3em;
+	font-family: monospace, monospace;
+	line-height: 2em;
 }
--- a/copyparty/web/browser.html
+++ b/copyparty/web/browser.html
@@ -12,17 +12,19 @@

 <body>
    <div id="ops">
-        <a href="#" data-dest="">---</a>
-        <a href="#" data-perm="read" data-dest="search">🔎</a>
+        <a href="#" data-dest="" data-desc="close submenu">---</a>
+        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
        {%- if have_up2k_idx %}
-        <a href="#" data-dest="up2k">🚀</a>
+        <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
        {%- else %}
-        <a href="#" data-perm="write" data-dest="up2k">🚀</a>
+        <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
        {%- endif %}
-        <a href="#" data-perm="write" data-dest="bup">🎈</a>
-        <a href="#" data-perm="write" data-dest="mkdir">📂</a>
-        <a href="#" data-perm="write" data-dest="new_md">📝</a>
-        <a href="#" data-perm="write" data-dest="msg">📟</a>
+        <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
+        <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
+        <a href="#" data-perm="write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
+        <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
+        <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
+        <div id="opdesc"></div>
    </div>

    <div id="op_search" class="opview">
@@ -33,7 +35,13 @@
        {%- endif %}
        <div id="srch_q"></div>
    </div>
+
    {%- include 'upload.html' %}
+
+    <div id="op_cfg" class="opview opbox">
+        <h3>key notation</h3>
+        <div id="key_notation"></div>
+    </div>
    
    <h1 id="path">
        <a href="#" id="entree">🌲</a>
@@ -42,17 +50,18 @@
        {%- endfor %}
    </h1>
    
-    <div id="pro" class="logue">{{ logues[0] }}</div>
+    <div id="tree">
+        <a href="#" id="detree">🍞...</a>
+        <a href="#" step="2" id="twobytwo">+</a>
+        <a href="#" step="-2" id="twig">&ndash;</a>
+        <a href="#" id="dyntree">a</a>
+        <ul id="treeul"></ul>
+        <div id="thx_ff">&nbsp;</div>
+    </div>

-    <table id="treetab">
-        <tr>
-            <td id="tree">
-                <a href="#" id="detree">🍞...</a>
-                <ul id="treeul"></ul>
-            </td>
-            <td id="treefiles"></td>
-        </tr>
-    </table>
+<div id="wrap">
+
+    <div id="pro" class="logue">{{ logues[0] }}</div>

    <table id="files">
        <thead>
@@ -90,6 +99,8 @@

    <h2><a href="?h">control-panel</a></h2>

+</div>
+
    {%- if srv_info %}
    <div id="srv_info"><span>{{ srv_info }}</span></div>
    {%- endif %}
--- a/copyparty/web/browser.js
+++ b/copyparty/web/browser.js
@@ -6,7 +6,7 @@ function dbg(msg) {
 	ebi('path').innerHTML = msg;
 }

-makeSortable(ebi('files'));
+makeSortable(ebi('files'), reload_mp);


 // extract songs + add play column
@@ -138,6 +138,9 @@ var pbar = (function () {
 	var grad = null;

 	r.drawbuf = function () {
+		if (!mp.au)
+			return;
+
 		var cs = getComputedStyle(r.bcan);
 		var sw = parseInt(cs['width']);
 		var sh = parseInt(cs['height']);
@@ -164,6 +167,9 @@ var pbar = (function () {
 		}
 	};
 	r.drawpos = function () {
+		if (!mp.au)
+			return;
+
 		var cs = getComputedStyle(r.bcan);
 		var sw = parseInt(cs['width']);
 		var sh = parseInt(cs['height']);
@@ -461,8 +467,7 @@ function play(tid, call_depth) {
 		var o = ebi(oid);
 		o.setAttribute('id', 'thx_js');
 		if (window.history && history.replaceState) {
-			var nurl = (document.location + '').split('#')[0] + '#' + oid;
-			history.replaceState(ebi('files').innerHTML, nurl, nurl);
+			hist_replace(document.location.pathname + '#' + oid);
 		}
 		else {
 			document.location.hash = oid;
@@ -505,7 +510,7 @@ function evau_error(e) {
 	if (eplaya.error.message)
 		err += '\n\n' + eplaya.error.message;

-	err += '\n\nFile: «' + decodeURIComponent(eplaya.src.split('/').slice(-1)[0]) + '»';
+	err += '\n\nFile: «' + uricom_dec(eplaya.src.split('/').slice(-1)[0])[0] + '»';

 	alert(err);
 }
@@ -540,7 +545,7 @@ function autoplay_blocked() {
 	var na = ebi('blk_na');

 	var fn = mp.tracks[mp.au.tid].split(/\//).pop();
-	fn = decodeURIComponent(fn.replace(/\+/g, ' '));
+	fn = uricom_dec(fn.replace(/\+/g, ' '))[0];

 	go.textContent = 'Play "' + fn + '"';
 	go.onclick = function (e) {
@@ -581,10 +586,11 @@ function autoplay_blocked() {
 			["name", "name", "name contains &nbsp; (negate with -nope)", "46"]
 		]
 	];
+	var oldcfg = [];

 	if (document.querySelector('#srch_form.tags'))
 		sconf.push(["tags",
-			["tags", "tags", "tags contains", "46"]
+			["tags", "tags", "tags contains &nbsp; (^=start, end=$)", "46"]
 		]);

 	var html = [];
@@ -648,7 +654,7 @@ function autoplay_blocked() {
 			return;

 		if (this.status !== 200) {
-			alert('ah fug\n' + this.status + ": " + this.responseText);
+			alert("http " + this.status + ": " + this.responseText);
 			return;
 		}

@@ -659,8 +665,16 @@ function autoplay_blocked() {
 		if (ofiles.getAttribute('ts') > this.ts)
 			return;

-		ebi('path').style.display = 'none';
-		ebi('tree').style.display = 'none';
+		if (!oldcfg.length) {
+			oldcfg = [
+				ebi('path').style.display,
+				ebi('tree').style.display,
+				ebi('wrap').style.marginLeft
+			];
+			ebi('path').style.display = 'none';
+			ebi('tree').style.display = 'none';
+			ebi('wrap').style.marginLeft = '0';
+		}

 		var html = mk_files_header(tagord);
 		html.push('<tbody>');
@@ -709,8 +723,10 @@ function autoplay_blocked() {

 	function unsearch(e) {
 		ev(e);
-		ebi('path').style.display = 'inline-block';
-		ebi('tree').style.display = 'block';
+		ebi('path').style.display = oldcfg[0];
+		ebi('tree').style.display = oldcfg[1];
+		ebi('wrap').style.marginLeft = oldcfg[2];
+		oldcfg = [];
 		ebi('files').innerHTML = orig_html;
 		orig_html = null;
 		reload_browser();
@@ -718,31 +734,77 @@ function autoplay_blocked() {
 })();


-// tree
-(function () {
-	var treedata = null;
+var treectl = (function () {
+	var dyn = bcfg_get('dyntree', true);
+	var treesz = icfg_get('treesz', 16);
+	treesz = Math.min(Math.max(treesz, 4), 50);
+	console.log('treesz [' + treesz + ']');
+	var entreed = false;

 	function entree(e) {
 		ev(e);
+		entreed = true;
 		ebi('path').style.display = 'none';

-		var treetab = ebi('treetab');
-		var treefiles = ebi('treefiles');
-
-		treetab.style.display = 'table';
-
-		treefiles.appendChild(ebi('pro'));
-		treefiles.appendChild(ebi('files'));
-		treefiles.appendChild(ebi('epi'));
+		var tree = ebi('tree');
+		tree.style.display = 'block';

 		swrite('entreed', 'tree');
-		get_tree("", get_vpath());
+		get_tree("", get_evpath(), true);
+		window.addEventListener('scroll', onscroll);
+		window.addEventListener('resize', onresize);
+		onresize();
 	}

-	function get_tree(top, dst) {
+	function detree(e) {
+		ev(e);
+		entreed = false;
+		ebi('tree').style.display = 'none';
+		ebi('path').style.display = 'inline-block';
+		ebi('wrap').style.marginLeft = '0';
+		swrite('entreed', 'na');
+		window.removeEventListener('resize', onresize);
+		window.removeEventListener('scroll', onscroll);
+	}
+
+	function onscroll() {
+		if (!entreed)
+			return;
+
+		var top = ebi('wrap').getBoundingClientRect().top;
+		ebi('tree').style.top = Math.max(0, parseInt(top)) + 'px';
+	}
+
+	function periodic() {
+		onscroll();
+		setTimeout(periodic, document.visibilityState ? 200 : 5000);
+	}
+	periodic();
+
+	function onresize(e) {
+		if (!entreed)
+			return;
+
+		var q = '#tree';
+		var nq = 0;
+		while (dyn) {
+			nq++;
+			q += '>ul>li';
+			if (!document.querySelector(q))
+				break;
+		}
+		var w = treesz + nq;
+		ebi('tree').style.width = w + 'em';
+		ebi('wrap').style.marginLeft = w + 'em';
+		onscroll();
+	}
+
+	function get_tree(top, dst, rst) {
 		var xhr = new XMLHttpRequest();
 		xhr.top = top;
 		xhr.dst = dst;
+		xhr.rst = rst;
+		xhr.ts = new Date().getTime();
 		xhr.open('GET', dst + '?tree=' + top, true);
 		xhr.onreadystatechange = recvtree;
 		xhr.send();
@@ -754,12 +816,19 @@ function autoplay_blocked() {
 			return;

 		if (this.status !== 200) {
-			alert('ah fug\n' + this.status + ": " + this.responseText);
+			alert("http " + this.status + ": " + this.responseText);
 			return;
 		}

+		var cur = ebi('treeul').getAttribute('ts');
+		if (cur && parseInt(cur) > this.ts) {
+			console.log("reject tree");
+			return;
+		}
+		ebi('treeul').setAttribute('ts', this.ts);
+
 		var top = this.top == '.' ? this.dst : this.top,
-			name = top.split('/').slice(-2)[0],
+			name = uricom_dec(top.split('/').slice(-2)[0])[0],
 			rtop = top.replace(/^\/+/, "");

 		try {
@@ -771,7 +840,7 @@ function autoplay_blocked() {
 		var html = parsetree(res, rtop);
 		if (!this.top) {
 			html = '<li><a href="#">-</a><a href="/">[root]</a>\n<ul>' + html;
-			if (!ebi('treeul').getElementsByTagName('li').length)
+			if (this.rst || !ebi('treeul').getElementsByTagName('li').length)
 				ebi('treeul').innerHTML = html + '</ul></li>';
 		}
 		else {
@@ -779,7 +848,7 @@ function autoplay_blocked() {
 				esc(top) + '">' + esc(name) +
 				"</a>\n<ul>\n" + html + "</ul>";

-			var links = document.querySelectorAll('#tree a+a');
+			var links = document.querySelectorAll('#treeul a+a');
 			for (var a = 0, aa = links.length; a < aa; a++) {
 				if (links[a].getAttribute('href') == top) {
 					var o = links[a].parentNode;
@@ -793,27 +862,18 @@ function autoplay_blocked() {
 		document.querySelector('#treeul>li>a+a').textContent = '[root]';
 		despin('#tree');
 		reload_tree();
-
-		var q = '#tree';
-		var nq = 0;
-		while (true) {
-			nq++;
-			q += '>ul>li';
-			if (!document.querySelector(q))
-				break;
-		}
-		ebi('treeul').style.width = (24 + nq) + 'em';
+		onresize();
 	}

 	function reload_tree() {
-		var cdir = get_vpath();
-		var links = document.querySelectorAll('#tree a+a');
+		var cdir = get_evpath();
+		var links = document.querySelectorAll('#treeul a+a');
 		for (var a = 0, aa = links.length; a < aa; a++) {
 			var href = links[a].getAttribute('href');
 			links[a].setAttribute('class', href == cdir ? 'hl' : '');
 			links[a].onclick = treego;
 		}
-		links = document.querySelectorAll('#tree li>a:first-child');
+		links = document.querySelectorAll('#treeul li>a:first-child');
 		for (var a = 0, aa = links.length; a < aa; a++) {
 			links[a].setAttribute('dst', links[a].nextSibling.getAttribute('href'));
 			links[a].onclick = treegrow;
@@ -827,12 +887,20 @@ function autoplay_blocked() {
 			treegrow.call(this.previousSibling, e);
 			return;
 		}
+		reqls(this.getAttribute('href'), true);
+	}
+
+	function reqls(url, hpush) {
 		var xhr = new XMLHttpRequest();
-		xhr.top = this.getAttribute('href');
+		xhr.top = url;
+		xhr.hpush = hpush;
+		xhr.ts = new Date().getTime();
 		xhr.open('GET', xhr.top + '?ls', true);
 		xhr.onreadystatechange = recvls;
 		xhr.send();
-		get_tree('.', xhr.top);
+		if (hpush)
+			get_tree('.', xhr.top);
+
 		enspin('#files');
 	}

@@ -844,6 +912,7 @@ function autoplay_blocked() {
 				rm.parentNode.removeChild(rm);
 			}
 			this.textContent = '+';
+			onresize();
 			return;
 		}
 		var dst = this.getAttribute('dst');
@@ -855,10 +924,17 @@ function autoplay_blocked() {
 			return;

 		if (this.status !== 200) {
-			alert('ah fug\n' + this.status + ": " + this.responseText);
+			alert("http " + this.status + ": " + this.responseText);
 			return;
 		}

+		var cur = ebi('files').getAttribute('ts');
+		if (cur && parseInt(cur) > this.ts) {
+			console.log("reject ls");
+			return;
+		}
+		ebi('files').setAttribute('ts', this.ts);
+
 		try {
 			var res = JSON.parse(this.responseText);
 		}
@@ -875,7 +951,7 @@ function autoplay_blocked() {
 		for (var a = 0; a < nodes.length; a++) {
 			var r = nodes[a],
 				ln = ['<tr><td>' + r.lead + '</td><td><a href="' +
-					top + r.href + '">' + esc(decodeURIComponent(r.href)) + '</a>', r.sz];
+					top + r.href + '">' + esc(uricom_dec(r.href)[0]) + '</a>', r.sz];

 			for (var b = 0; b < res.taglist.length; b++) {
 				var k = res.taglist[b],
@@ -898,7 +974,9 @@ function autoplay_blocked() {
 		html = html.join('\n');
 		ebi('files').innerHTML = html;

-		history.pushState(html, this.top, this.top);
+		if (this.hpush)
+			hist_push(this.top);
+
 		apply_perms(res.perms);
 		despin('#files');

@@ -906,6 +984,7 @@ function autoplay_blocked() {
 		ebi('epi').innerHTML = res.logues ? res.logues[1] || "" : "";

 		filecols.set_style();
+		mukey.render();
 		reload_tree();
 		reload_browser();
 	}
@@ -921,12 +1000,14 @@ function autoplay_blocked() {
 		keys.sort();
 		for (var a = 0; a < keys.length; a++) {
 			var kk = keys[a],
-				k = kk.slice(1),
-				url = '/' + (top ? top + k : k) + '/',
-				ek = esc(k),
+				ks = kk.slice(1),
+				k = uricom_dec(ks),
+				hek = esc(k[0]),
+				uek = k[1] ? uricom_enc(k[0], true) : k[0],
+				url = '/' + (top ? top + uek : uek) + '/',
 				sym = res[kk] ? '-' : '+',
 				link = '<a href="#">' + sym + '</a><a href="' +
-					esc(url) + '">' + ek + '</a>';
+					url + '">' + hek + '</a>';

 			if (res[kk]) {
 				var subtree = parsetree(res[kk], url.slice(1));
@@ -939,37 +1020,48 @@ function autoplay_blocked() {
 		return ret;
 	}

-	function detree(e) {
+	function dyntree(e) {
 		ev(e);
-		var treetab = ebi('treetab');
+		dyn = !dyn;
+		bcfg_set('dyntree', dyn);
+		onresize();
+	}

-		treetab.parentNode.insertBefore(ebi('pro'), treetab);
-		treetab.parentNode.insertBefore(ebi('files'), treetab.nextSibling);
-		treetab.parentNode.insertBefore(ebi('epi'), ebi('files').nextSibling);
+	function scaletree(e) {
+		ev(e);
+		treesz += parseInt(this.getAttribute("step"));
+		if (isNaN(treesz))
+			treesz = 16;

-		ebi('path').style.display = 'inline-block';
-		treetab.style.display = 'none';
-
-		swrite('entreed', 'na');
+		swrite('treesz', treesz);
+		onresize();
 	}

 	ebi('entree').onclick = entree;
 	ebi('detree').onclick = detree;
+	ebi('dyntree').onclick = dyntree;
+	ebi('twig').onclick = scaletree;
+	ebi('twobytwo').onclick = scaletree;
 	if (sread('entreed') == 'tree')
 		entree();

 	window.onpopstate = function (e) {
-		console.log(e.url + ' ,, ' + ((e.state + '').slice(0, 64)));
-		if (e.state) {
-			ebi('files').innerHTML = e.state;
-			reload_tree();
-			reload_browser();
-		}
+		console.log("h-pop " + e.state);
+		if (!e.state)
+			return;
+
+		var url = new URL(e.state, "https://" + document.location.host);
+		url = url.pathname;
+		get_tree("", url, true);
+		reqls(url);
 	};

 	if (window.history && history.pushState) {
-		var u = get_vpath() + window.location.hash;
-		history.replaceState(ebi('files').innerHTML, u, u);
+		hist_replace(get_evpath() + window.location.hash);
+	}
+
+	return {
+		"onscroll": onscroll
 	}
 })();

@@ -1131,17 +1223,158 @@ var filecols = (function () {
 })();


+var mukey = (function () {
+	var maps = {
+		"rekobo_alnum": [
+			"1B ", "2B ", "3B ", "4B ", "5B ", "6B ", "7B ", "8B ", "9B ", "10B", "11B", "12B",
+			"1A ", "2A ", "3A ", "4A ", "5A ", "6A ", "7A ", "8A ", "9A ", "10A", "11A", "12A"
+		],
+		"rekobo_classic": [
+			"B  ", "F# ", "Db ", "Ab ", "Eb ", "Bb ", "F  ", "C  ", "G  ", "D  ", "A  ", "E  ",
+			"Abm", "Ebm", "Bbm", "Fm ", "Cm ", "Gm ", "Dm ", "Am ", "Em ", "Bm ", "F#m", "Dbm"
+		],
+		"traktor_musical": [
+			"B  ", "Gb ", "Db ", "Ab ", "Eb ", "Bb ", "F  ", "C  ", "G  ", "D  ", "A  ", "E  ",
+			"Abm", "Ebm", "Bbm", "Fm ", "Cm ", "Gm ", "Dm ", "Am ", "Em ", "Bm ", "Gbm", "Dbm"
+		],
+		"traktor_sharps": [
+			"B  ", "F# ", "C# ", "G# ", "D# ", "A# ", "F  ", "C  ", "G  ", "D  ", "A  ", "E  ",
+			"G#m", "D#m", "A#m", "Fm ", "Cm ", "Gm ", "Dm ", "Am ", "Em ", "Bm ", "F#m", "C#m"
+		],
+		"traktor_open": [
+			"6d ", "7d ", "8d ", "9d ", "10d", "11d", "12d", "1d ", "2d ", "3d ", "4d ", "5d ",
+			"6m ", "7m ", "8m ", "9m ", "10m", "11m", "12m", "1m ", "2m ", "3m ", "4m ", "5m "
+		]
+	};
+	var map = {};
+
+	var html = [];
+	for (var k in maps) {
+		if (!maps.hasOwnProperty(k))
+			continue;
+
+		html.push(
+			'<span><input type="radio" name="keytype" value="' + k + '" id="key_' + k + '">' +
+			'<label for="key_' + k + '">' + k + '</label></span>');
+
+		for (var a = 0; a < 24; a++)
+			maps[k][a] = maps[k][a].trim();
+	}
+	ebi('key_notation').innerHTML = html.join('\n');
+
+	function set_key_notation(e) {
+		ev(e);
+		var notation = this.getAttribute('value');
+		load_notation(notation);
+		render();
+	}
+
+	function load_notation(notation) {
+		swrite("key_notation", notation);
+		map = {};
+		var dst = maps[notation];
+		for (var k in maps)
+			if (k != notation && maps.hasOwnProperty(k))
+				for (var a = 0; a < 24; a++)
+					if (maps[k][a] != dst[a])
+						map[maps[k][a]] = dst[a];
+	}
+
+	function render() {
+		var tds = ebi('files').tHead.getElementsByTagName('th');
+		var i = -1;
+		var min = false;
+		for (var a = 0; a < tds.length; a++) {
+			var spans = tds[a].getElementsByTagName('span');
+			if (spans.length && spans[0].textContent == 'Key') {
+				min = tds[a].getAttribute('class').indexOf('min') !== -1;
+				i = a;
+				break;
+			}
+		}
+
+		if (i == -1)
+			return;
+
+		var rows = ebi('files').tBodies[0].rows;
+
+		if (min)
+			for (var a = 0, aa = rows.length; a < aa; a++) {
+				var c = rows[a].cells[i];
+				if (!c)
+					continue;
+
+				var v = c.getAttribute('html');
+				c.setAttribute('html', map[v] || v);
+			}
+		else
+			for (var a = 0, aa = rows.length; a < aa; a++) {
+				var c = rows[a].cells[i];
+				if (!c)
+					continue;
+
+				var v = c.textContent;
+				c.textContent = map[v] || v;
+			}
+	}
+
+	function try_render() {
+		try {
+			render();
+		}
+		catch (ex) {
+			console.log("key notation failed: " + ex);
+		}
+	}
+
+	var notation = sread("key_notation") || "rekobo_alnum";
+	ebi('key_' + notation).checked = true;
+	load_notation(notation);
+
+	var o = document.querySelectorAll('#key_notation input');
+	for (var a = 0; a < o.length; a++) {
+		o[a].onchange = set_key_notation;
+	}
+
+	return {
+		"render": try_render
+	};
+})();
+
+
+(function () {
+	function set_tooltip(e) {
+		ev(e);
+		ebi('opdesc').innerHTML = this.getAttribute('data-desc');
+	}
+	var btns = document.querySelectorAll('#ops, #ops>a');
+	for (var a = 0; a < btns.length; a++) {
+		btns[a].onmouseenter = set_tooltip;
+	}
+})();
+
+
 function ev_row_tgl(e) {
 	ev(e);
 	filecols.toggle(this.parentElement.parentElement.getElementsByTagName('span')[0].textContent);
 }


+function reload_mp() {
+	if (mp && mp.au) {
+		mp.au.pause();
+		mp.au = null;
+	}
+	widget.close();
+	mp = init_mp();
+}
+
+
 function reload_browser(not_mp) {
 	filecols.set_style();
-	makeSortable(ebi('files'));
+	makeSortable(ebi('files'), reload_mp);

-	var parts = get_vpath().split('/');
+	var parts = get_evpath().split('/');
 	var rm = document.querySelectorAll('#path>a+a+a');
 	for (a = rm.length - 1; a >= 0; a--)
 		rm[a].parentNode.removeChild(rm[a]);
@@ -1163,16 +1396,11 @@ function reload_browser(not_mp) {
 		oo[a].textContent = hsz;
 	}

-	if (!not_mp) {
-		if (mp && mp.au) {
-			mp.au.pause();
-			mp.au = null;
-		}
-		widget.close();
-		mp = init_mp();
-	}
+	if (!not_mp)
+		reload_mp();

 	if (window['up2k'])
 		up2k.set_fsearch();
 }
 reload_browser(true);
+mukey.render();
--- a/copyparty/web/md.js
+++ b/copyparty/web/md.js
@@ -65,7 +65,7 @@ function statify(obj) {
        if (a > 0)
            loc.push(n[a]);

-        var dec = hesc(decodeURIComponent(n[a]));
+        var dec = hesc(uricom_dec(n[a])[0]);

        nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
    }
--- a/copyparty/web/mde.js
+++ b/copyparty/web/mde.js
@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
        if (a > 0)
            loc.push(n[a]);

-        var dec = decodeURIComponent(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+        var dec = uricom_dec(n[a])[0].replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");

        nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
    }
--- a/copyparty/web/up2k.js
+++ b/copyparty/web/up2k.js
@@ -46,9 +46,9 @@ function up2k_flagbus() {
    var dbg = function (who, msg) {
        console.log('flagbus(' + flag.id + '): [' + who + '] ' + msg);
    };
-    flag.ch.onmessage = function (ev) {
-        var who = ev.data[0],
-            what = ev.data[1];
+    flag.ch.onmessage = function (e) {
+        var who = e.data[0],
+            what = e.data[1];

        if (who == flag.id) {
            dbg(who, 'hi me (??)');
@@ -83,7 +83,7 @@ function up2k_flagbus() {
            flag.ch.postMessage([flag.id, "hey"]);
        }
        else {
-            dbg('?', ev.data);
+            dbg('?', e.data);
        }
    };
    var tx = function (now, msg) {
@@ -194,7 +194,7 @@ function up2k_init(have_crypto) {

    // handle user intent to use the basic uploader instead
    ebi('u2nope').onclick = function (e) {
-        e.preventDefault();
+        ev(e);
        setmsg();
        goto('bup');
    };
@@ -209,41 +209,7 @@ function up2k_init(have_crypto) {
        };
    }

-    function cfg_get(name) {
-        var val = sread(name);
-        if (val === null)
-            return parseInt(ebi(name).value);
-
-        ebi(name).value = val;
-        return val;
-    }
-
-    function bcfg_get(name, defval) {
-        var o = ebi(name);
-        if (!o)
-            return defval;
-
-        var val = sread(name);
-        if (val === null)
-            val = defval;
-        else
-            val = (val == '1');
-
-        o.checked = val;
-        return val;
-    }
-
-    function bcfg_set(name, val) {
-        swrite(name, val ? '1' : '0');
-
-        var o = ebi(name);
-        if (o)
-            o.checked = val;
-
-        return val;
-    }
-
-    var parallel_uploads = cfg_get('nthread');
+    var parallel_uploads = icfg_get('nthread');
    var multitask = bcfg_get('multitask', true);
    var ask_up = bcfg_get('ask_up', true);
    var flag_en = bcfg_get('flag_en', false);
@@ -288,29 +254,29 @@ function up2k_init(have_crypto) {
    }
    ebi('u2btn').addEventListener('click', nav, false);

-    function ondrag(ev) {
-        ev.stopPropagation();
-        ev.preventDefault();
-        ev.dataTransfer.dropEffect = 'copy';
-        ev.dataTransfer.effectAllowed = 'copy';
+    function ondrag(e) {
+        e.stopPropagation();
+        e.preventDefault();
+        e.dataTransfer.dropEffect = 'copy';
+        e.dataTransfer.effectAllowed = 'copy';
    }
    ebi('u2btn').addEventListener('dragover', ondrag, false);
    ebi('u2btn').addEventListener('dragenter', ondrag, false);

-    function gotfile(ev) {
-        ev.stopPropagation();
-        ev.preventDefault();
+    function gotfile(e) {
+        e.stopPropagation();
+        e.preventDefault();

        var files;
        var is_itemlist = false;
-        if (ev.dataTransfer) {
-            if (ev.dataTransfer.items) {
-                files = ev.dataTransfer.items; // DataTransferItemList
+        if (e.dataTransfer) {
+            if (e.dataTransfer.items) {
+                files = e.dataTransfer.items; // DataTransferItemList
                is_itemlist = true;
            }
-            else files = ev.dataTransfer.files; // FileList
+            else files = e.dataTransfer.files; // FileList
        }
-        else files = ev.target.files;
+        else files = e.target.files;

        if (files.length == 0)
            return alert('no files selected??');
@@ -366,7 +332,7 @@ function up2k_init(have_crypto) {
                "name": fobj.name,
                "size": fobj.size,
                "lmod": lmod / 1000,
-                "purl": get_vpath(),
+                "purl": get_evpath(),
                "done": false,
                "hash": []
            };
@@ -689,8 +655,8 @@ function up2k_init(have_crypto) {
            prog(t.n, nchunk, col_hashing);
        };

-        var segm_load = function (ev) {
-            cache_buf = ev.target.result;
+        var segm_load = function (e) {
+            cache_buf = e.target.result;
            cache_ofs = 0;
            hash_calc();
        };
@@ -764,7 +730,7 @@ function up2k_init(have_crypto) {
        st.busy.handshake.push(t);

        var xhr = new XMLHttpRequest();
-        xhr.onload = function (ev) {
+        xhr.onload = function (e) {
            if (xhr.status == 200) {
                var response = JSON.parse(xhr.responseText);

@@ -915,7 +881,7 @@ function up2k_init(have_crypto) {
            alert('y o u   b r o k e    i t\n\n(was that a folder? just files please)');
        };

-        reader.onload = function (ev) {
+        reader.onload = function (e) {
            var xhr = new XMLHttpRequest();
            xhr.upload.onprogress = function (xev) {
                var perc = xev.loaded / (cdr - car) * 100;
@@ -949,7 +915,7 @@ function up2k_init(have_crypto) {
            xhr.setRequestHeader('Content-Type', 'application/octet-stream');
            xhr.overrideMimeType('Content-Type', 'application/octet-stream');
            xhr.responseType = 'text';
-            xhr.send(ev.target.result);
+            xhr.send(e.target.result);
        };

        reader.readAsArrayBuffer(bobslice.call(t.fobj, car, cdr));
@@ -978,7 +944,7 @@ function up2k_init(have_crypto) {
    ///   config ui
    //

-    function onresize(ev) {
+    function onresize(e) {
        var bar = ebi('ops'),
            wpx = innerWidth,
            fpx = parseInt(getComputedStyle(bar)['font-size']),
@@ -993,17 +959,17 @@ function up2k_init(have_crypto) {
            ebi('u2conf').setAttribute('class', wide ? 'has_btn' : '');
        }
    }
-    window.onresize = onresize;
+    window.addEventListener('resize', onresize);
    onresize();

-    function desc_show(ev) {
+    function desc_show(e) {
        var msg = this.getAttribute('alt');
        msg = msg.replace(/\$N/g, "<br />");
        var cdesc = ebi('u2cdesc');
        cdesc.innerHTML = msg;
        cdesc.setAttribute('class', 'show');
    }
-    function desc_hide(ev) {
+    function desc_hide(e) {
        ebi('u2cdesc').setAttribute('class', '');
    }
    var o = document.querySelectorAll('#u2conf *[alt]');
@@ -1118,17 +1084,17 @@ function up2k_init(have_crypto) {
        }
    }

-    function nop(ev) {
-        ev.preventDefault();
+    function nop(e) {
+        ev(e);
        this.click();
    }

-    ebi('nthread_add').onclick = function (ev) {
-        ev.preventDefault();
+    ebi('nthread_add').onclick = function (e) {
+        ev(e);
        bumpthread(1);
    };
-    ebi('nthread_sub').onclick = function (ev) {
-        ev.preventDefault();
+    ebi('nthread_sub').onclick = function (e) {
+        ev(e);
        bumpthread(-1);
    };

--- a/copyparty/web/upload.css
+++ b/copyparty/web/upload.css
@@ -62,7 +62,7 @@
 	width: calc(100% - 2em);
 	max-width: 100em;
 }
-#u2form.srch #u2tab {
+#op_up2k.srch #u2tab {
 	max-width: none;
 }
 #u2tab td {
@@ -76,7 +76,7 @@
 #u2tab td:nth-child(3) {
 	width: 40%;
 }
-#u2form.srch #u2tab td:nth-child(3) {
+#op_up2k.srch #u2tab td:nth-child(3) {
 	font-family: sans-serif;
 	width: auto;
 }
--- a/copyparty/web/util.js
+++ b/copyparty/web/util.js
@@ -23,6 +23,7 @@ function esc(txt) {
 }
 function vis_exh(msg, url, lineNo, columnNo, error) {
    window.onerror = undefined;
+    window['vis_exh'] = null;
    var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
        esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];

@@ -127,7 +128,7 @@ function sortTable(table, col) {
    });
    for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
 }
-function makeSortable(table) {
+function makeSortable(table, cb) {
    var th = table.tHead, i;
    th && (th = th.rows[0]) && (th = th.cells);
    if (th) i = th.length;
@@ -136,6 +137,8 @@ function makeSortable(table) {
        th[i].onclick = function (e) {
            ev(e);
            sortTable(table, i);
+            if (cb)
+                cb();
        };
    }(i));
 }
@@ -156,7 +159,7 @@ function opclick(e) {
    var dest = this.getAttribute('data-dest');
    goto(dest);

-    swrite('opmode', dest || undefined);
+    swrite('opmode', dest || null);

    var input = document.querySelector('.opview.act input:not([type="hidden"])')
    if (input)
@@ -173,10 +176,6 @@ function goto(dest) {
    for (var a = obj.length - 1; a >= 0; a--)
        obj[a].classList.remove('act');

-    var others = ['path', 'files', 'widget'];
-    for (var a = 0; a < others.length; a++)
-        ebi(others[a]).classList.remove('hidden');
-
    if (dest) {
        var ui = ebi('op_' + dest);
        ui.classList.add('act');
@@ -186,6 +185,9 @@ function goto(dest) {
        if (fn)
            fn();
    }
+
+    if (window['treectl'])
+        treectl.onscroll();
 }


@@ -224,6 +226,31 @@ function linksplit(rp) {
 }


+function uricom_enc(txt, do_fb_enc) {
+    try {
+        return encodeURIComponent(txt);
+    }
+    catch (ex) {
+        console.log("uce-err [" + txt + "]");
+        if (do_fb_enc)
+            return esc(txt);
+
+        return txt;
+    }
+}
+
+
+function uricom_dec(txt) {
+    try {
+        return [decodeURIComponent(txt), true];
+    }
+    catch (ex) {
+        console.log("ucd-err [" + txt + "]");
+        return [txt, false];
+    }
+}
+
+
 function get_evpath() {
    var ret = document.location.pathname;

@@ -238,7 +265,7 @@ function get_evpath() {


 function get_vpath() {
-    return decodeURIComponent(get_evpath());
+    return uricom_dec(get_evpath())[0];
 }


@@ -266,12 +293,12 @@ function sread(key) {
    if (window.localStorage)
        return localStorage.getItem(key);

-    return '';
+    return null;
 }

 function swrite(key, val) {
    if (window.localStorage) {
-        if (val === undefined)
+        if (val === undefined || val === null)
            localStorage.removeItem(key);
        else
            localStorage.setItem(key, val);
@@ -292,3 +319,59 @@ function jwrite(key, val) {
    else
        swrite(key, JSON.stringify(val));
 }
+
+function icfg_get(name, defval) {
+    var o = ebi(name);
+
+    var val = parseInt(sread(name));
+    if (isNaN(val))
+        return parseInt(o ? o.value : defval);
+
+    if (o)
+        o.value = val;
+
+    return val;
+}
+
+function bcfg_get(name, defval) {
+    var o = ebi(name);
+    if (!o)
+        return defval;
+
+    var val = sread(name);
+    if (val === null)
+        val = defval;
+    else
+        val = (val == '1');
+
+    bcfg_upd_ui(name, val);
+    return val;
+}
+
+function bcfg_set(name, val) {
+    swrite(name, val ? '1' : '0');
+    bcfg_upd_ui(name, val);
+    return val;
+}
+
+function bcfg_upd_ui(name, val) {
+    var o = ebi(name);
+    if (!o)
+        return;
+
+    if (o.getAttribute('type') == 'checkbox')
+        o.checked = val;
+    else if (o)
+        o.setAttribute('class', val ? 'on' : '');
+}
+
+
+function hist_push(url) {
+    console.log("h-push " + url);
+    history.pushState(url, url, url);
+}
+
+function hist_replace(url) {
+    console.log("h-repl " + url);
+    history.replaceState(url, url, url);
+}
--- a/docs/notes.sh
+++ b/docs/notes.sh
@@ -11,6 +11,13 @@ gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f --
 gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done


+##
+## detect partial uploads based on file contents
+##  (in case of context loss or old copyparties)
+
+echo; find -type f | while IFS= read -r x; do printf '\033[A\033[36m%s\033[K\033[0m\n' "$x"; tail -c$((1024*1024)) <"$x" | xxd -a | awk 'NR==1&&/^[0: ]+.{16}$/{next} NR==2&&/^\*$/{next} NR==3&&/^[0f]+: [0 ]+65 +.{16}$/{next} {e=1} END {exit e}' || continue; printf '\033[A\033[31msus:\033[33m %s \033[0m\n\n' "$x"; done
+
+
 ##
 ## create a test payload

--- a/scripts/copyparty-repack.sh
+++ b/scripts/copyparty-repack.sh
@@ -20,6 +20,7 @@ set -e
 # -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py


+command -v gnutar && tar() { gnutar "$@"; }
 command -v gtar && tar() { gtar "$@"; }
 command -v gsed && sed() { gsed "$@"; }
 td="$(mktemp -d)"
@@ -29,11 +30,11 @@ pwd


 dl_text() {
-	command -v curl && exec curl "$@"
+	command -v curl >/dev/null && exec curl "$@"
 	exec wget -O- "$@"
 }
 dl_files() {
-	command -v curl && exec curl -L --remote-name-all "$@"
+	command -v curl >/dev/null && exec curl -L --remote-name-all "$@"
 	exec wget "$@"
 }
 export -f dl_files
--- a/scripts/make-sfx.sh
+++ b/scripts/make-sfx.sh
@@ -122,7 +122,7 @@ git describe --tags >/dev/null 2>/dev/null && {
 		exit 1
 	}

-	dt="$(git log -1 --format=%cd --date=format:'%Y,%m,%d' | sed -E 's/,0?/, /g')"
+	dt="$(git log -1 --format=%cd --date=short | sed -E 's/-0?/, /g')"
 	printf 'git %3s: \033[36m%s\033[0m\n' ver "$ver" dt "$dt"
 	sed -ri '
 		s/^(VERSION =)(.*)/#\1\2\n\1 ('"$t_ver"')/;
@@ -180,7 +180,7 @@ tmv "$f"

 # up2k goes from 28k to 22k laff
 echo entabbening
-find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
+find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
--- a/scripts/sfx.py
+++ b/scripts/sfx.py
@@ -1,9 +1,8 @@
 #!/usr/bin/env python
-# coding: utf-8
+# coding: latin-1
 from __future__ import print_function, unicode_literals

-import os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
-import subprocess as sp
+import os, sys, time, shutil, runpy, tarfile, hashlib, platform, tempfile, traceback

 """
 run me with any version of python, i will unpack and run copyparty
@@ -344,20 +343,24 @@ def get_payload():
                break


-def confirm():
+def confirm(rv):
    msg()
+    msg(traceback.format_exc())
    msg("*** hit enter to exit ***")
    try:
        raw_input() if PY2 else input()
    except:
        pass

+    sys.exit(rv)
+

 def run(tmp, j2ver):
    global cpp

    msg("jinja2:", j2ver or "bundled")
    msg("sfxdir:", tmp)
+    msg()

    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
    try:
@@ -373,30 +376,16 @@ def run(tmp, j2ver):
    if j2ver:
        del ld[-1]

-    cmd = (
-        "import sys, runpy; "
-        + "".join(['sys.path.insert(0, r"' + x + '"); ' for x in ld])
-        + 'runpy.run_module("copyparty", run_name="__main__")'
-    )
-    cmd = [sys.executable, "-c", cmd] + list(sys.argv[1:])
+    for x in ld:
+        sys.path.insert(0, x)

-    cmd = [str(x) for x in cmd]
-    msg("\n", cmd, "\n")
-    cpp = sp.Popen(cmd)
    try:
-        cpp.wait()
+        runpy.run_module(str("copyparty"), run_name=str("__main__"))
+    except SystemExit as ex:
+        if ex.code:
+            confirm(ex.code)
    except:
-        cpp.wait()
-
-    if cpp.returncode != 0:
-        confirm()
-
-    sys.exit(cpp.returncode)
-
-
-def bye(sig, frame):
-    if cpp is not None:
-        cpp.terminate()
+        confirm(1)


 def main():
@@ -430,8 +419,6 @@ def main():

    # skip 0

-    signal.signal(signal.SIGTERM, bye)
-
    tmp = unpack()

    try:
@@ -439,7 +426,7 @@ def main():
    except:
        j2ver = None

-    return run(tmp, j2ver)
+    run(tmp, j2ver)


 if __name__ == "__main__":
--- a/tests/test_vfs.py
+++ b/tests/test_vfs.py
@@ -16,6 +16,12 @@ from copyparty.authsrv import AuthSrv
 from copyparty import util


+class Cfg(Namespace):
+    def __init__(self, a=[], v=[], c=None):
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
+
+
 class TestVFS(unittest.TestCase):
    def dump(self, vfs):
        print(json.dumps(vfs, indent=4, sort_keys=True, default=lambda o: o.__dict__))
@@ -35,7 +41,13 @@ class TestVFS(unittest.TestCase):
    def ls(self, vfs, vpath, uname):
        """helper for resolving and listing a folder"""
        vn, rem = vfs.get(vpath, uname, True, False)
-        return vn.ls(rem, uname)
+        r1 = vn.ls(rem, uname, False)
+        r2 = vn.ls(rem, uname, False)
+        self.assertEqual(r1, r2)
+
+        fsdir, real, virt = r1
+        real = [x[0] for x in real]
+        return fsdir, real, virt

    def runcmd(self, *argv):
        p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
@@ -78,7 +90,7 @@ class TestVFS(unittest.TestCase):
        finally:
            return ret

-    def log(self, src, msg):
+    def log(self, src, msg, c=0):
        pass

    def test(self):
@@ -102,7 +114,7 @@ class TestVFS(unittest.TestCase):
                            f.write(fn)

        # defaults
-        vfs = AuthSrv(Namespace(c=None, a=[], v=[]), self.log).vfs
+        vfs = AuthSrv(Cfg(), self.log).vfs
        self.assertEqual(vfs.nodes, {})
        self.assertEqual(vfs.vpath, "")
        self.assertEqual(vfs.realpath, td)
@@ -110,7 +122,7 @@ class TestVFS(unittest.TestCase):
        self.assertEqual(vfs.uwrite, ["*"])

        # single read-only rootfs (relative path)
-        vfs = AuthSrv(Namespace(c=None, a=[], v=["a/ab/::r"]), self.log).vfs
+        vfs = AuthSrv(Cfg(v=["a/ab/::r"]), self.log).vfs
        self.assertEqual(vfs.nodes, {})
        self.assertEqual(vfs.vpath, "")
        self.assertEqual(vfs.realpath, os.path.join(td, "a", "ab"))
@@ -118,9 +130,7 @@ class TestVFS(unittest.TestCase):
        self.assertEqual(vfs.uwrite, [])

        # single read-only rootfs (absolute path)
-        vfs = AuthSrv(
-            Namespace(c=None, a=[], v=[td + "//a/ac/../aa//::r"]), self.log
-        ).vfs
+        vfs = AuthSrv(Cfg(v=[td + "//a/ac/../aa//::r"]), self.log).vfs
        self.assertEqual(vfs.nodes, {})
        self.assertEqual(vfs.vpath, "")
        self.assertEqual(vfs.realpath, os.path.join(td, "a", "aa"))
@@ -129,7 +139,7 @@ class TestVFS(unittest.TestCase):

        # read-only rootfs with write-only subdirectory (read-write for k)
        vfs = AuthSrv(
-            Namespace(c=None, a=["k:k"], v=[".::r:ak", "a/ac/acb:a/ac/acb:w:ak"]),
+            Cfg(a=["k:k"], v=[".::r:ak", "a/ac/acb:a/ac/acb:w:ak"]),
            self.log,
        ).vfs
        self.assertEqual(len(vfs.nodes), 1)
@@ -192,7 +202,10 @@ class TestVFS(unittest.TestCase):
        self.assertEqual(list(virt), [])

        # admin-only rootfs with all-read-only subfolder
-        vfs = AuthSrv(Namespace(c=None, a=["k:k"], v=[".::ak", "a:a:r"]), self.log,).vfs
+        vfs = AuthSrv(
+            Cfg(a=["k:k"], v=[".::ak", "a:a:r"]),
+            self.log,
+        ).vfs
        self.assertEqual(len(vfs.nodes), 1)
        self.assertEqual(vfs.vpath, "")
        self.assertEqual(vfs.realpath, td)
@@ -211,9 +224,7 @@ class TestVFS(unittest.TestCase):

        # breadth-first construction
        vfs = AuthSrv(
-            Namespace(
-                c=None,
-                a=[],
+            Cfg(
                v=[
                    "a/ac/acb:a/ac/acb:w",
                    "a:a:w",
@@ -234,7 +245,7 @@ class TestVFS(unittest.TestCase):
        self.undot(vfs, "./.././foo/..", "")

        # shadowing
-        vfs = AuthSrv(Namespace(c=None, a=[], v=[".::r", "b:a/ac:r"]), self.log).vfs
+        vfs = AuthSrv(Cfg(v=[".::r", "b:a/ac:r"]), self.log).vfs

        fsp, r1, v1 = self.ls(vfs, "", "*")
        self.assertEqual(fsp, td)
@@ -271,7 +282,7 @@ class TestVFS(unittest.TestCase):
                ).encode("utf-8")
            )

-        au = AuthSrv(Namespace(c=[cfg_path], a=[], v=[]), self.log)
+        au = AuthSrv(Cfg(c=[cfg_path]), self.log)
        self.assertEqual(au.user["a"], "123")
        self.assertEqual(au.user["asd"], "fgh:jkl")
        n = au.vfs
Author	SHA1	Message	Date
ed	e3f1d19756	v0.9.8	2021-03-15 01:13:46 +01:00
ed	93c2bd6ef6	fix tree trying to make surprise appearances	2021-03-13 02:29:13 +01:00
ed	4d0e5ff6db	turns out whitespace compresses better than tabs	2021-03-13 00:16:07 +01:00
ed	0893f06919	browser: reload music player on column-sort so tracks play in the right order	2021-03-13 00:15:53 +01:00
ed	46b6abde3f	fuse-client: password from file	2021-03-13 00:14:22 +01:00
ed	0696610dee	give up, just try both and see what sticks	2021-03-13 00:14:07 +01:00
ed	edf0d3684c	sfx: improvements from r0c	2021-03-13 00:13:10 +01:00
ed	7af159f5f6	heh	2021-03-09 21:36:14 +01:00
ed	7f2cb6764a	v0.9.7	2021-03-08 03:51:26 +01:00
ed	96495a9bf1	v0.9.6	2021-03-07 21:44:25 +01:00
ed	b2fafec5fc	handle key-normalization errors	2021-03-07 21:41:36 +01:00
ed	0850b8ae2b	v0.9.5	2021-03-07 19:25:24 +01:00
ed	8a68a96c57	css tweaks	2021-03-07 19:15:19 +01:00
ed	d3aae8ed6a	more mojibake fixes	2021-03-07 18:58:26 +01:00
ed	c62ebadda8	separate tree scrollbar	2021-03-07 18:26:57 +01:00
ed	ffcee6d390	add tooltips and more mojibake compat	2021-03-07 04:14:55 +01:00
ed	de32838346	key notation normalization (why tho)	2021-03-07 02:46:17 +01:00
ed	b9a4e47ea2	mojibake support for the spa stuff	2021-03-06 22:48:49 +01:00
ed	57d994422d	logging cleanup	2021-03-06 17:38:56 +01:00
ed	6ecd745323	so much for sessionStorage	2021-03-06 16:34:55 +01:00
ed	bd769f5bdb	fix py2 + encourage py3	2021-03-06 02:42:17 +01:00
ed	2381692aba	js cfg	2021-03-06 02:30:36 +01:00
ed	24fdada0a0	did you know rhel 7 has an sqlite3 from 2015	2021-03-06 02:28:49 +01:00
ed	bb5169710a	warn people when they're gonna have a bad time	2021-03-06 00:30:05 +01:00
ed	9cde2352f3	v0.9.4	2021-03-05 02:06:18 +01:00
ed	482dd7a938	v0.9.3	2021-03-05 00:00:22 +01:00
ed	bddcc69438	v0.9.2	2021-03-04 22:58:22 +01:00
ed	19d4540630	good	2021-03-04 22:38:12 +01:00
ed	4f5f6c81f5	add buttons to adjust tree width	2021-03-04 22:34:09 +01:00
ed	7e4c1238ba	oh	2021-03-04 21:12:54 +01:00
ed	f7196ac773	dodge pushstate size limit	2021-03-04 21:06:59 +01:00
ed	7a7c832000	sfx-builder: support ancient git versions	2021-03-04 20:30:28 +01:00
ed	2b4ccdbebb	multithread the slow mtag backends	2021-03-04 20:28:03 +01:00
ed	0d16b49489	broke this too	2021-03-04 01:35:09 +01:00
ed	768405b691	tree broke	2021-03-04 01:32:44 +01:00
ed	da01413b7b	remove speedbumps	2021-03-04 01:21:04 +01:00
ed	914e22c53e	async tagging of incoming files	2021-03-03 18:36:05 +01:00