v0.11.18

README.md

@@ -62,6 +62,14 @@ download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/do
 
 running the sfx without arguments (for example doubleclicking it on Windows) will give everyone full access to the current folder; see `-h` for help if you want accounts and volumes etc
 
+some recommended options:
+* `-e2dsa` enables general file indexing, see [search configuration](#search-configuration)
+* `-e2ts` enables audio metadata indexing (needs either FFprobe or mutagen), see [optional dependencies](#optional-dependencies)
+* `-v /mnt/music:/music:r:afoo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, with user `foo` as `a`dmin (read/write), password `bar`
+  * replace `:r:afoo` with `:rfoo` to only make the folder readable by `foo` and nobody else
+  * in addition to `r`ead and `a`dmin, `w`rite makes a folder write-only, so cannot list/access files in it
+* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access
+
 you may also want these, especially on servers:
 * [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
 * [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
@@ -101,7 +109,7 @@ summary: all planned features work! now please enjoy the bloatening
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view
-  * ☑ media player
+  * ☑ audio player
   * ☑ thumbnails
     * ☑ images using Pillow
     * ☑ videos using FFmpeg
@@ -163,7 +171,7 @@ the browser has the following hotkeys
   * `0..9` jump to 10%..90%
   * `U/O` skip 10sec back/forward
   * `J/L` prev/next song
-    * `J` also starts playing the folder
+  * `M` play/pause (also starts playing the folder)
 * in the grid view:
   * `S` toggle multiselect
   * `A/D` zoom
@@ -301,7 +309,7 @@ the same arguments can be set as volume flags, in addition to `d2d` and `d2t` fo
 * `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 
 note:
-* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those
 * the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
 
 you can choose to only index filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash` or the volume-flag `cdhash`, this has the following consequences:

contrib/nginx/copyparty.conf

@@ -1,3 +1,8 @@
+# when running copyparty behind a reverse-proxy,
+# make sure that copyparty allows at least as many clients as the proxy does,
+# so run copyparty with -nc 512 if your nginx has the default limits
+# (worker_processes 1, worker_connections 512)
+
 upstream cpp {
 	server 127.0.0.1:3923;
 	keepalive 120;

copyparty/__main__.py

@@ -23,7 +23,7 @@ from textwrap import dedent
 from .__init__ import E, WINDOWS, VT100, PY2
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab, IMPLICATIONS
+from .util import py_desc, align_tab, IMPLICATIONS, alltrace
 
 HAVE_SSL = True
 try:
@@ -182,6 +182,16 @@ def sighandler(sig=None, frame=None):
     print("\n".join(msg))
 
 
+def stackmon(fp, ival):
+    ctr = 0
+    while True:
+        ctr += 1
+        time.sleep(ival)
+        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
+        with open(fp, "wb") as f:
+            f.write(st.encode("utf-8", "replace"))
+
+
 def run_argparse(argv, formatter):
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
@@ -222,10 +232,6 @@ def run_argparse(argv, formatter):
               "print,get" prints the data in the log and returns GET
               (leave out the ",get" to return an error instead)
 
-            --ciphers help = available ssl/tls ciphers,
-            --ssl-ver help = available ssl/tls versions,
-              default is what python considers safe, usually >= TLS1
-            
             values for --ls:
               "USR" is a user to browse as; * is anonymous, ** is all users
               "VOL" is a single volume to scan, default is * (all vols)
@@ -244,27 +250,45 @@ def run_argparse(argv, formatter):
     )
     # fmt: off
     ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
-    ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
     ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
     ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
-    ap.add_argument("-q", action="store_true", help="quiet")
+    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
+    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
-    ap.add_argument("-nih", action="store_true", help="no info hostname")
-    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
-    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
-    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
-    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
 
-    ap2 = ap.add_argument_group('appearance options')
-    ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
+    ap2 = ap.add_argument_group('network options')
+    ap2.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
+    
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ssl/tls ciphers; [help] shows available ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+
+    ap2 = ap.add_argument_group('opt-outs')
+    ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
+    ap2.add_argument("-nih", action="store_true", help="no info hostname")
+    ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+
+    ap2 = ap.add_argument_group('safety options')
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
+    ap2.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+
+    ap2 = ap.add_argument_group('logging options')
+    ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
 
     ap2 = ap.add_argument_group('admin panel options')
     ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
@@ -299,22 +323,14 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
 
-    ap2 = ap.add_argument_group('SSL/TLS options')
-    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
-    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="ssl/tls versions to allow")
-    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
-    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
-    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+    ap2 = ap.add_argument_group('appearance options')
+    ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
 
     ap2 = ap.add_argument_group('debug options')
-    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes")
-    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
     ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
-    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
-    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
+    ap2.add_argument("--stackmon", metavar="P,S", help="write stacktrace to Path every S second")
     
     return ap.parse_args(args=argv[1:])
     # fmt: on
@@ -354,6 +370,16 @@ def main(argv=None):
     except AssertionError:
         al = run_argparse(argv, Dodge11874)
 
+    if al.stackmon:
+        fp, f = al.stackmon.rsplit(",", 1)
+        f = int(f)
+        t = threading.Thread(
+            target=stackmon,
+            args=(fp, f),
+        )
+        t.daemon = True
+        t.start()
+
     # propagate implications
     for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 13)
+VERSION = (0, 11, 18)
 CODENAME = "the grid"
-BUILD_DT = (2021, 6, 12)
+BUILD_DT = (2021, 6, 18)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/broker_mpw.py

@@ -68,6 +68,7 @@ class MpWorker(object):
 
             # self.logw("work: [{}]".format(d[0]))
             if dest == "shutdown":
+                self.httpsrv.shutdown()
                 self.logw("ok bye")
                 sys.exit(0)
                 return

copyparty/broker_thr.py

@@ -25,6 +25,7 @@ class BrokerThr(object):
 
     def shutdown(self):
         # self.log("broker", "shutting down")
+        self.httpsrv.shutdown()
         pass
 
     def put(self, want_retval, dest, *args):

copyparty/httpcli.py

@@ -104,10 +104,21 @@ class HttpCli(object):
         v = self.headers.get("connection", "").lower()
         self.keepalive = not v.startswith("close") and self.http_ver != "HTTP/1.0"
 
-        v = self.headers.get("x-forwarded-for", None)
-        if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
-            self.ip = v.split(",")[0]
-            self.log_src = self.conn.set_rproxy(self.ip)
+        n = self.args.rproxy
+        if n:
+            v = self.headers.get("x-forwarded-for")
+            if v and self.conn.addr[0] in ["127.0.0.1", "::1"]:
+                if n > 0:
+                    n -= 1
+
+                vs = v.split(",")
+                try:
+                    self.ip = vs[n].strip()
+                except:
+                    self.ip = vs[-1].strip()
+                    self.log("rproxy={} oob x-fwd {}".format(self.args.rproxy, v), c=3)
+
+                self.log_src = self.conn.set_rproxy(self.ip)
 
         if self.args.ihead:
             keys = self.args.ihead
@@ -245,10 +256,11 @@ class HttpCli(object):
         if self.is_rclone:
             return ""
 
+        cmap = {"pw": "cppwd"}
         kv = {
             k: v
             for k, v in self.uparam.items()
-            if k not in rm and self.cookies.get(k) != v
+            if k not in rm and self.cookies.get(cmap.get(k, k)) != v
         }
         kv.update(add)
         if not kv:
@@ -570,10 +582,17 @@ class HttpCli(object):
             try:
                 dst = os.path.join(vfs.realpath, rem)
                 os.makedirs(fsenc(dst))
-            except:
-                if not os.path.isdir(fsenc(dst)):
+            except OSError as ex:
+                if ex.errno == 13:
+                    raise Pebkac(500, "the server OS denied write-access")
+
+                if ex.errno == 17:
                     raise Pebkac(400, "some file got your folder name")
 
+                raise Pebkac(500, min_ex())
+            except:
+                raise Pebkac(500, min_ex())
+
         x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
         ret = x.get()
         if sub:
@@ -758,8 +777,13 @@ class HttpCli(object):
 
             try:
                 os.mkdir(fsenc(fn))
+            except OSError as ex:
+                if ex.errno == 13:
+                    raise Pebkac(500, "the server OS denied write-access")
+
+                raise Pebkac(500, "mkdir failed:\n" + min_ex())
             except:
-                raise Pebkac(500, "mkdir failed, check the logs")
+                raise Pebkac(500, min_ex())
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         self.redirect(vpath)
@@ -1176,7 +1200,7 @@ class HttpCli(object):
         #
         # send reply
 
-        if not is_compressed:
+        if not is_compressed and "cache" not in self.uparam:
             self.out_headers.update(NO_CACHE)
 
         self.out_headers["Accept-Ranges"] = "bytes"
@@ -1423,33 +1447,8 @@ class HttpCli(object):
         if self.args.no_stack:
             raise Pebkac(403, "disabled by argv")
 
-        threads = {}
-        names = dict([(t.ident, t.name) for t in threading.enumerate()])
-        for tid, stack in sys._current_frames().items():
-            name = "{} ({:x})".format(names.get(tid), tid)
-            threads[name] = stack
-
-        rret = []
-        bret = []
-        for name, stack in sorted(threads.items()):
-            ret = ["\n\n# {}".format(name)]
-            pad = None
-            for fn, lno, name, line in traceback.extract_stack(stack):
-                fn = os.sep.join(fn.split(os.sep)[-3:])
-                ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
-                if line:
-                    ret.append("  " + str(line.strip()))
-                    if "self.not_empty.wait()" in line:
-                        pad = " " * 4
-
-            if pad:
-                bret += [ret[0]] + [pad + x for x in ret[1:]]
-            else:
-                rret += ret
-
-        ret = rret + bret
-        ret = ("<pre>" + "\n".join(ret)).encode("utf-8")
-        self.reply(ret)
+        ret = "<pre>{}\n{}".format(time.time(), alltrace())
+        self.reply(ret.encode("utf-8"))
 
     def tx_tree(self):
         top = self.uparam["tree"] or ""

copyparty/httpconn.py

@@ -43,6 +43,7 @@ class HttpConn(object):
         self.ico = Ico(self.args)
 
         self.t0 = time.time()
+        self.stopping = False
         self.nbyte = 0
         self.workload = 0
         self.u2idx = None
@@ -50,6 +51,14 @@ class HttpConn(object):
         self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
         self.set_rproxy()
 
+    def shutdown(self):
+        self.stopping = True
+        try:
+            self.s.shutdown(socket.SHUT_RDWR)
+            self.s.close()
+        except:
+            pass
+
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -174,7 +183,7 @@ class HttpConn(object):
         if not self.sr:
             self.sr = Unrecv(self.s)
 
-        while True:
+        while not self.stopping:
             if self.is_mp:
                 self.workload += 50
                 if self.workload >= 2 ** 31:

copyparty/httpsrv.py

@@ -80,7 +80,14 @@ class HttpSrv(object):
             return len(self.clients)
 
     def shutdown(self):
-        self.log("ok bye")
+        clients = list(self.clients.keys())
+        for cli in clients:
+            try:
+                cli.shutdown()
+            except:
+                pass
+
+        self.log("httpsrv-n", "ok bye")
 
     def thr_client(self, sck, addr):
         """thread managing one tcp client"""
@@ -100,25 +107,35 @@ class HttpSrv(object):
                     thr.daemon = True
                     thr.start()
 
+        fno = sck.fileno()
         try:
             if self.args.log_conn:
                 self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
 
             cli.run()
 
+        except (OSError, socket.error) as ex:
+            if ex.errno not in [10038, 10054, 107, 57, 9]:
+                self.log(
+                    "%s %s" % addr,
+                    "run({}): {}".format(fno, ex),
+                    c=6,
+                )
+
         finally:
             sck = cli.s
             if self.args.log_conn:
                 self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
 
             try:
+                fno = sck.fileno()
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
             except (OSError, socket.error) as ex:
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "shut({}): {}".format(sck.fileno(), ex),
+                        "shut({}): {}".format(fno, ex),
                         c="1;30",
                     )
                 if ex.errno not in [10038, 10054, 107, 57, 9]:

copyparty/mtag.py

@@ -16,6 +16,7 @@ if not PY2:
 
 def have_ff(cmd):
     if PY2:
+        print("# checking {}".format(cmd))
         cmd = (cmd + " -version").encode("ascii").split(b" ")
         try:
             sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()

copyparty/tcpsrv.py

@@ -21,6 +21,7 @@ class TcpSrv(object):
         self.log = hub.log
 
         self.num_clients = Counter()
+        self.stopping = False
 
         ip = "127.0.0.1"
         eps = {ip: "local only"}
@@ -67,7 +68,7 @@ class TcpSrv(object):
             ip, port = srv.getsockname()
             self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
 
-        while True:
+        while not self.stopping:
             if self.args.log_conn:
                 self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
 
@@ -80,6 +81,9 @@ class TcpSrv(object):
 
             ready, _, _ = select.select(self.srv, [], [])
             for srv in ready:
+                if self.stopping:
+                    break
+
                 sck, addr = srv.accept()
                 sip, sport = srv.getsockname()
                 if self.args.log_conn:
@@ -95,6 +99,13 @@ class TcpSrv(object):
                 self.hub.broker.put(False, "httpconn", sck, addr)
 
     def shutdown(self):
+        self.stopping = True
+        try:
+            for srv in self.srv:
+                srv.close()
+        except:
+            pass
+
         self.log("tcpsrv", "ok bye")
 
     def detect_interfaces(self, listen_ips):

copyparty/th_srv.py

@@ -154,7 +154,8 @@ class ThumbSrv(object):
         histpath = self.asrv.vfs.histtab[ptop]
         tpath = thumb_path(histpath, rem, mtime, fmt)
         abspath = os.path.join(ptop, rem)
-        cond = threading.Condition()
+        cond = threading.Condition(self.mutex)
+        do_conv = False
         with self.mutex:
             try:
                 self.busy[tpath].append(cond)
@@ -172,8 +173,11 @@ class ThumbSrv(object):
                         f.write(fsenc(os.path.dirname(abspath)))
 
                 self.busy[tpath] = [cond]
-                self.q.put([abspath, tpath])
-                self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
+                do_conv = True
+
+        if do_conv:
+            self.q.put([abspath, tpath])
+            self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
 
         while not self.stopping:
             with self.mutex:
@@ -181,7 +185,7 @@ class ThumbSrv(object):
                     break
 
             with cond:
-                cond.wait()
+                cond.wait(3)
 
         try:
             st = os.stat(tpath)
@@ -335,29 +339,32 @@ class ThumbSrv(object):
         interval = self.args.th_clean
         while True:
             time.sleep(interval)
+            ndirs = 0
             for vol, histpath in self.asrv.vfs.histtab.items():
                 if histpath.startswith(vol):
                     self.log("\033[Jcln {}/\033[A".format(histpath))
                 else:
                     self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
 
-                self.clean(histpath)
+                ndirs += self.clean(histpath)
 
-            self.log("\033[Jcln ok")
+            self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
 
     def clean(self, histpath):
-        # self.log("cln {}".format(histpath))
+        thumbpath = os.path.join(histpath, "th")
+        # self.log("cln {}".format(thumbpath))
         maxage = self.args.th_maxage
         now = time.time()
         prev_b64 = None
         prev_fp = None
         try:
-            ents = os.listdir(histpath)
+            ents = os.listdir(thumbpath)
         except:
-            return
+            return 0
 
+        ndirs = 0
         for f in sorted(ents):
-            fp = os.path.join(histpath, f)
+            fp = os.path.join(thumbpath, f)
             cmp = fp.lower().replace("\\", "/")
 
             # "top" or b64 prefix/full (a folder)
@@ -372,10 +379,11 @@ class ThumbSrv(object):
                                 break
 
                         if safe:
+                            ndirs += 1
                             self.log("rm -rf [{}]".format(fp))
                             shutil.rmtree(fp, ignore_errors=True)
                 else:
-                    self.clean(fp)
+                    ndirs += self.clean(fp)
                 continue
 
             # thumb file
@@ -397,3 +405,5 @@ class ThumbSrv(object):
 
             prev_b64 = b64
             prev_fp = fp
+
+        return ndirs

copyparty/up2k.py

@@ -16,7 +16,7 @@ import traceback
 import subprocess as sp
 from copy import deepcopy
 
-from .__init__ import WINDOWS, ANYWIN
+from .__init__ import WINDOWS, ANYWIN, PY2
 from .util import (
     Pebkac,
     Queue,
@@ -134,7 +134,7 @@ class Up2k(object):
     def get_state(self):
         mtpq = 0
         q = "select count(w) from mt where k = 't:mtp'"
-        got_lock = self.mutex.acquire(timeout=0.5)
+        got_lock = False if PY2 else self.mutex.acquire(timeout=0.5)
         if got_lock:
             for cur in self.cur.values():
                 try:
@@ -1022,7 +1022,7 @@ class Up2k(object):
         now = time.time()
         job = None
         with self.mutex:
-            cur = self.cur.get(cj["ptop"], None)
+            cur = self.cur.get(cj["ptop"])
             reg = self.registry[cj["ptop"]]
             if cur:
                 if self.no_expr_idx:
@@ -1180,7 +1180,7 @@ class Up2k(object):
 
     def handle_chunk(self, ptop, wark, chash):
         with self.mutex:
-            job = self.registry[ptop].get(wark, None)
+            job = self.registry[ptop].get(wark)
             if not job:
                 known = " ".join([x for x in self.registry[ptop].keys()])
                 self.log("unknown wark [{}], known: {}".format(wark, known))
@@ -1245,7 +1245,7 @@ class Up2k(object):
         return ret, dst
 
     def idx_wark(self, ptop, wark, rd, fn, lmod, sz):
-        cur = self.cur.get(ptop, None)
+        cur = self.cur.get(ptop)
         if not cur:
             return False
 
@@ -1414,7 +1414,7 @@ class Up2k(object):
 
         newest = max(x["poke"] for _, x in reg.items()) if reg else 0
         etag = [len(reg), newest]
-        if etag == prev.get(ptop, None):
+        if etag == prev.get(ptop):
             return
 
         try:

copyparty/util.py

@@ -254,6 +254,34 @@ def trace(*args, **kwargs):
     nuprint(msg)
 
 
+def alltrace():
+    threads = {}
+    names = dict([(t.ident, t.name) for t in threading.enumerate()])
+    for tid, stack in sys._current_frames().items():
+        name = "{} ({:x})".format(names.get(tid), tid)
+        threads[name] = stack
+
+    rret = []
+    bret = []
+    for name, stack in sorted(threads.items()):
+        ret = ["\n\n# {}".format(name)]
+        pad = None
+        for fn, lno, name, line in traceback.extract_stack(stack):
+            fn = os.sep.join(fn.split(os.sep)[-3:])
+            ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
+            if line:
+                ret.append("  " + str(line.strip()))
+                if "self.not_empty.wait()" in line:
+                    pad = " " * 4
+
+        if pad:
+            bret += [ret[0]] + [pad + x for x in ret[1:]]
+        else:
+            rret += ret
+
+    return "\n".join(rret + bret)
+
+
 def min_ex():
     et, ev, tb = sys.exc_info()
     tb = traceback.extract_tb(tb, 2)

copyparty/web/browser.css

@@ -25,6 +25,35 @@ html, body {
 body {
 	padding-bottom: 5em;
 }
+#tt {
+	position: fixed;
+	max-width: 34em;
+	background: #222;
+	border: 0 solid #555;
+	overflow: hidden;
+	margin-top: 1em;
+	padding: 0 1em;
+	height: 0;
+	opacity: .1;
+	transition: opacity 0.14s, height 0.14s, padding 0.14s;
+	box-shadow: 0 .2em .5em #222;
+	border-radius: .4em;
+	z-index: 9001;
+}
+#tt.show {
+	padding: 1em;
+	height: auto;
+	border-width: .2em 0;
+	opacity: 1;
+}
+#tt code {
+	background: #3c3c3c;
+	padding: .2em .3em;
+	border-top: 1px solid #777;
+	border-radius: .3em;
+	font-family: monospace, monospace;
+	line-height: 2em;
+}
 #path,
 #path * {
 	font-size: 1em;
@@ -53,6 +82,7 @@ body {
 #files tbody a {
 	display: block;
 	padding: .3em 0;
+	scroll-margin-top: 45vh;
 }
 #files tbody div a {
 	color: #f5a;
@@ -68,7 +98,6 @@ a, #files tbody div a:last-child {
 	text-decoration: underline;
 }
 #files thead {
-	background: #333;
 	position: sticky;
 	top: 0;
 }
@@ -76,29 +105,30 @@ a, #files tbody div a:last-child {
 	color: #999;
 	font-weight: normal;
 }
-#files tr:hover {
+#files tr:hover td {
 	background: #1c1c1c;
 }
 #files thead th {
-	padding: .5em 1.3em .3em 1.3em;
+	padding: .5em .3em .3em .3em;
+	border-right: 2px solid #3c3c3c;
+	border-bottom: 2px solid #444;
+	background: #333;
 	cursor: pointer;
 }
+#files thead th+th {
+	border-left: 2px solid #2a2a2a;
+}
 #files thead th:last-child {
-	background: #444;
-	border-radius: .7em .7em 0 0;
+	border-right: none;
 }
-#files thead th:first-child {
+#files tbody {
 	background: #222;
 }
-#files tbody,
-#files thead th:nth-child(2) {
-	background: #222;
-	border-radius: 0 .7em 0 0;
-}
 #files td {
 	margin: 0;
 	padding: 0 .5em;
 	border-bottom: 1px solid #111;
+	border-left: 1px solid #2c2c2c;
 }
 #files td+td+td {
 	max-width: 30em;
@@ -185,9 +215,17 @@ a, #files tbody div a:last-child {
 	margin: -.2em;
 }
 #files tbody a.play.act {
-	color: #840;
+	color: #720;
 	text-shadow: 0 0 .3em #b80;
 }
+#ggrid a.play,
+html.light #ggrid a.play {
+	color: #fff;
+	background: #750;
+	border-color: #c90;
+	border-top: 1px solid #da4;
+	box-shadow: 0 .1em 1.2em #b83;
+}
 #files tbody tr.sel td,
 #ggrid a.sel,
 html.light #ggrid a.sel {
@@ -209,11 +247,17 @@ html.light #ggrid a.sel {
 	box-shadow: 0 .1em 1.2em #b36;
 	transition: all 0.2s cubic-bezier(.2, 2.2, .5, 1); /* https://cubic-bezier.com/#.4,2,.7,1 */
 }
-#ggrid a.sel img {
+#ggrid a.sel img,
+#ggrid a.play img {
 	opacity: .7;
-	box-shadow: 0 0 1em #b36;
 	filter: contrast(130%) brightness(107%);
 }
+#ggrid a.sel img {
+	box-shadow: 0 0 1em #b36;
+}
+#ggrid a.play img {
+	box-shadow: 0 0 1em #b83;
+}
 #files tr.sel a {
 	color: #fff;
 }
@@ -483,20 +527,48 @@ html.light #ggrid a.sel {
 	margin: .5em;
 }
 .opview input[type=text] {
-	color: #fff;
 	background: #383838;
+	color: #fff;
 	border: none;
 	box-shadow: 0 0 .3em #222;
 	border-bottom: 1px solid #fc5;
 	border-radius: .2em;
 	padding: .2em .3em;
 }
+.opview input.err {
+	background: #a20;
+	border-color: #f00;
+	box-shadow: 0 0 .7em #f00;
+	text-shadow: 1px 1px 0 #500;
+	outline: none;
+}
 input[type="checkbox"]+label {
 	color: #f5a;
 }
 input[type="checkbox"]:checked+label {
 	color: #fc5;
 }
+input.eq_gain {
+	width: 3em;
+	text-align: center;
+	margin: 0 .6em;
+}
+#audio_eq table {
+	border-collapse: collapse;
+}
+#audio_eq td {
+	text-align: center;
+}
+#audio_eq a.eq_step {
+	font-size: 1.5em;
+	display: block;
+	padding: 0;
+}
+#au_eq {
+	display: block;
+	margin-top: .5em;
+	padding: 1.3em .3em;
+}
 
 
 
@@ -563,6 +635,7 @@ input[type="checkbox"]:checked+label {
 }
 #wrap {
 	margin-top: 2em;
+	min-height: 90vh;
 }
 #tree {
 	display: none;
@@ -575,6 +648,12 @@ input[type="checkbox"]:checked+label {
 	overscroll-behavior-y: none;
 	scrollbar-color: #eb0 #333;
 }
+#treeh {
+	background: #333;
+	position: sticky;
+	z-index: 1;
+	top: 0;
+}
 #thx_ff {
 	padding: 5em 0;
 }
@@ -600,6 +679,7 @@ input[type="checkbox"]:checked+label {
 	box-shadow: 0 .1em .2em #222 inset;
 	border-radius: .3em;
 	margin: .2em;
+	white-space: pre;
 	position: relative;
 	top: -.2em;
 }
@@ -667,34 +747,20 @@ input[type="checkbox"]:checked+label {
 	font-size: 2em;
 	white-space: nowrap;
 }
-#files th:hover .cfg,
-#files th.min .cfg {
+#files th:hover .cfg {
 	display: block;
 	width: 1em;
 	border-radius: .2em;
 	margin: -1.3em auto 0 auto;
 	background: #444;
 }
-#files th.min .cfg {
-	margin: -.6em;
-}
-#files>thead>tr>th.min span {
-	position: absolute;
-	transform: rotate(270deg);
-	background: linear-gradient(90deg, rgba(68,68,68,0), rgba(68,68,68,0.5) 70%, #444);
-	margin-left: -4.6em;
-	padding: .4em;
-	top: 5.4em;
-	width: 8em;
-	text-align: right;
-	letter-spacing: .04em;
+#files>thead>tr>th.min,
+#files td.min {
+		display: none;
 }
 #files td:nth-child(2n) {
 	color: #f5a;
 }
-#files td.min a {
-	display: none;
-}
 #files tr.play td,
 #files tr.play div a {
 	background: #fc4;
@@ -709,47 +775,32 @@ input[type="checkbox"]:checked+label {
 	color: #300;
 	background: #fea;
 }
-#op_cfg {
+.opwide {
 	max-width: none;
 	margin-right: 1.5em;
 }
-#op_cfg>div>a {
+.opwide>div {
+	display: inline-block;
+	vertical-align: top;
+	border-left: .2em solid #4c4c4c;
+	margin-left: .5em;
+	padding-left: .5em;
+}
+.opwide>div.fill {
+	display: block;
+}
+.opwide>div>div>a {
 	line-height: 2em;
 }
-#op_cfg>div>span {
+#op_cfg>div>div>span {
 	display: inline-block;
 	padding: .2em .4em;
 }
-#op_cfg h3 {
+.opbox h3 {
 	margin: .8em 0 0 .6em;
 	padding: 0;
 	border-bottom: 1px solid #555;
 }
-#opdesc {
-	display: none;
-}
-#ops:hover #opdesc {
-	display: block;
-	background: linear-gradient(0deg,#555, #4c4c4c 80%, #444);
-	box-shadow: 0 .3em 1em #222;
-	padding: 1em;
-	border-radius: .3em;
-	position: absolute;
-	z-index: 3;
-	top: 6em;
-	right: 1.5em;
-}
-#ops:hover #opdesc.off {
-	display: none;
-}
-#opdesc code {
-	background: #3c3c3c;
-	padding: .2em .3em;
-	border-top: 1px solid #777;
-	border-radius: .3em;
-	font-family: monospace, monospace;
-	line-height: 2em;
-}
 #thumbs {
 	opacity: .3;
 }
@@ -804,7 +855,6 @@ html.light #ghead {
 	content: '📂';
 	line-height: 0;
 	font-size: 2em;
-	display: inline-block;
 	margin: -.7em .1em -.5em -.3em;
 }
 #ggrid a:hover {
@@ -857,6 +907,15 @@ html.light {
 	background: #eee;
 	text-shadow: none;
 }
+html.light #tt {
+	background: #fff;
+	border-color: #888;
+	box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
+}
+html.light #tt code {
+	background: #060;
+	color: #fff;
+}
 html.light #ops,
 html.light .opbox,
 html.light #srch_form {
@@ -919,13 +978,14 @@ html.light #files {
 }
 html.light #files thead th {
 	background: #eee;
-	border-radius: 0;
+	border-right: 1px solid #ccc;
+	border-bottom: 1px solid #ccc;
 }
-html.light #files tr td {
-	border-top: 1px solid #ddd;
+html.light #files thead th {
+	border-left: 1px solid #f7f7f7;
 }
 html.light #files td {
-	border-bottom: 1px solid #f7f7f7;
+	border-color: #ddd #fff #fff #ddd;
 }
 html.light #files tbody tr:last-child td {
 	border-bottom: .2em solid #ccc;
@@ -933,25 +993,28 @@ html.light #files tbody tr:last-child td {
 html.light #files td:nth-child(2n) {
 	color: #d38;
 }
-html.light #files tr:hover td {
-	background: #fff;
+html.light #files tr.play td:nth-child(2n) {
+	color: #c16;
 }
 html.light #files tbody a.play {
 	color: #c0f;
 }
-html.light tr.play td {
+html.light #files tbody a.play.act {
+	color: #90c;
+}
+html.light #files tr.play td {
 	background: #fc5;
+	border-color: #eb1;
+}
+html.light #files tr:hover td {
+	background: #fff;
 }
 html.light tr.play a {
 	color: #406;
 }
-html.light #files th:hover .cfg,
-html.light #files th.min .cfg {
+html.light #files th:hover .cfg {
 	background: #ccc;
 }
-html.light #files > thead > tr > th.min span {
-	background: linear-gradient(90deg, rgba(204,204,204,0), rgba(204,204,204,0.5) 70%, #ccc);
-}
 html.light #blocked {
 	background: #eee;
 }
@@ -961,7 +1024,21 @@ html.light #blk_abrt a {
 	box-shadow: 0 .2em .4em #ddd;
 }
 html.light #widget a {
-	color: #fc5;
+	color: #06a;
+}
+html.light #wtoggle,
+html.light #widgeti {
+	background: #eee;
+}
+html.light #wtoggle {
+	box-shadow: 0 0 .5em #bbb;
+}
+html.light #widget.open {
+	border-top: .2em solid #f7f7f7;
+}
+html.light #wzip,
+html.light #wnp {
+	border-color: #ccc;
 }
 html.light #files tr.sel:hover td {
 	background: #c37;
@@ -978,20 +1055,15 @@ html.light #files tr.sel a.play.act {
 html.light input[type="checkbox"] + label {
 	color: #333;
 }
+html.light .opwide>div {
+	border-color: #ccc;
+}
 html.light .opview input[type="text"] {
 	background: #fff;
 	color: #333;
 	box-shadow: 0 0 2px #888;
 	border-color: #38d;
 }
-html.light #ops:hover #opdesc {
-	background: #fff;
-	box-shadow: 0 .3em 1em #ccc;
-}
-html.light #opdesc code {
-	background: #060;
-	color: #fff;
-}
 html.light #u2tab a>span,
 html.light #files td div span {
 	color: #000;
@@ -1001,9 +1073,6 @@ html.light #path {
 	text-shadow: none;
 	box-shadow: 0 0 .3em #bbb;
 }
-html.light #path a {
-	color: #333;
-}
 html.light #path a:not(:last-child)::after {
 	border-color: #ccc;
 	background: none;
@@ -1012,7 +1081,7 @@ html.light #path a:not(:last-child)::after {
 }
 html.light #path a:hover {
 	background: none;
-	color: #60a;
+	color: #90d;
 }
 html.light #files tbody div a {
 	color: #d38;
@@ -1022,6 +1091,9 @@ html.light #files tr.sel a:hover {
 	color: #000;
 	background: #fff;
 }
+html.light #treeh {
+	background: #eee;
+}
 html.light #tree {
 	scrollbar-color: #a70 #ddd;
 }

copyparty/web/browser.html

@@ -15,18 +15,19 @@
 
 <body>
     <div id="ops">
-        <a href="#" data-dest="" data-desc="close submenu">---</a>
+        <a href="#" data-dest="" tt="close submenu">---</a>
         {%- if have_up2k_idx %}
-        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
-        <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
+        <a href="#" data-perm="read" data-dest="search" tt="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
+        <a href="#" data-dest="up2k" tt="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
         {%- else %}
-        <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
+        <a href="#" data-perm="write" data-dest="up2k" tt="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
         {%- endif %}
-        <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
-        <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
-        <a href="#" data-perm="read write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
-        <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
-        <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
+        <a href="#" data-perm="write" data-dest="bup" tt="bup: basic uploader, even supports netscape 4.0">🎈</a>
+        <a href="#" data-perm="write" data-dest="mkdir" tt="mkdir: create a new directory">📂</a>
+        <a href="#" data-perm="read write" data-dest="new_md" tt="new-md: create a new markdown document">📝</a>
+        <a href="#" data-perm="write" data-dest="msg" tt="msg: send a message to the server log">📟</a>
+        <a href="#" data-dest="player" tt="media player options">🎺</a>
+        <a href="#" data-dest="cfg" tt="configuration options">⚙️</a>
         <div id="opdesc"></div>
     </div>
 
@@ -39,36 +40,41 @@
         <div id="srch_q"></div>
     </div>
 
+    <div id="op_player" class="opview opbox opwide"></div>
+
     {%- include 'upload.html' %}
 
-    <div id="op_cfg" class="opview opbox">
-        <h3>switches</h3>
+    <div id="op_cfg" class="opview opbox opwide">
         <div>
-            <a id="tooltips" class="tgl btn" href="#">tooltips</a>
-            <a id="lightmode" class="tgl btn" href="#">lightmode</a>
-            <a id="griden" class="tgl btn" href="#">the grid</a>
-            <a id="thumbs" class="tgl btn" href="#">thumbs</a>
+            <h3>switches</h3>
+            <div>
+                <a id="tooltips" class="tgl btn" href="#" tt="◔ ◡ ◔">ℹ️ tooltips</a>
+                <a id="lightmode" class="tgl btn" href="#">☀️ lightmode</a>
+                <a id="griden" class="tgl btn" href="#" tt="toggle icons or list-view$NHotkey: G">田 the grid</a>
+                <a id="thumbs" class="tgl btn" href="#" tt="in icon view, toggle icons or thumbnails$NHotkey: T">🖼️ thumbs</a>
+            </div>
         </div>
         {%- if have_zip %}
-        <h3>folder download</h3>
-        <div id="arc_fmt"></div>
+        <div><h3>folder download</h3><div id="arc_fmt"></div></div>
         {%- endif %}
-        <h3>key notation</h3>
-        <div id="key_notation"></div>
+        <div><h3>key notation</h3><div id="key_notation"></div></div>
+        <div class="fill"><h3>hidden columns</h3><div id="hcols"></div></div>
     </div>
     
     <h1 id="path">
-        <a href="#" id="entree">🌲</a>
+        <a href="#" id="entree" tt="show directory tree$NHotkey: B">🌲</a>
         {%- for n in vpnodes %}
         <a href="/{{ n[0] }}">{{ n[1] }}</a>
         {%- endfor %}
     </h1>
     
     <div id="tree">
-        <a href="#" id="detree">🍞...</a>
-        <a href="#" class="btn" step="2" id="twobytwo">+</a>
-        <a href="#" class="btn" step="-2" id="twig">&ndash;</a>
-        <a href="#" class="tgl btn" id="dyntree">a</a>
+        <div id="treeh">
+            <a href="#" id="detree" tt="show breadcrumbs$NHotkey: B">🍞...</a>
+            <a href="#" class="btn" step="2" id="twobytwo">+</a>
+            <a href="#" class="btn" step="-2" id="twig">&ndash;</a>
+            <a href="#" class="tgl btn" id="dyntree" tt="autogrow as tree expands">a</a>
+        </div>
         <ul id="treeul"></ul>
         <div id="thx_ff">&nbsp;</div>
     </div>

copyparty/web/dbg-audio.js

@@ -0,0 +1,61 @@
+var ofun = audio_eq.apply.bind(audio_eq);
+audio_eq.apply = function () {
+    var ac1 = mp.ac;
+    ofun();
+    var ac = mp.ac,
+        w = 2048,
+        h = 256;
+
+    if (!audio_eq.filters.length) {
+        audio_eq.ana = null;
+        return;
+    }
+
+    var can = ebi('fft_can');
+    if (!can) {
+        can = mknod('canvas');
+        can.setAttribute('id', 'fft_can');
+        can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
+        document.body.appendChild(can);
+        can.width = w;
+        can.height = h;
+    }
+    var cc = can.getContext('2d');
+    if (!ac)
+        return;
+
+    var ana = ac.createAnalyser();
+    ana.smoothingTimeConstant = 0;
+    ana.fftSize = 8192;
+
+    audio_eq.filters[0].connect(ana);
+    audio_eq.ana = ana;
+
+    var buf = new Uint8Array(ana.frequencyBinCount),
+        colw = can.width / buf.length;
+
+    cc.fillStyle = '#fc0';
+    function draw() {
+        if (ana == audio_eq.ana)
+            requestAnimationFrame(draw);
+
+        ana.getByteFrequencyData(buf);
+
+        cc.clearRect(0, 0, can.width, can.height);
+
+        /*var x = 0, w = 1;
+        for (var a = 0; a < buf.length; a++) {
+            cc.fillRect(x, h - buf[a], w, h);
+            x += w;
+        }*/
+        var mul = Math.pow(w, 4) / buf.length;
+        for (var x = 0; x < w; x++) {
+            var a = Math.floor(Math.pow(x, 4) / mul),
+                v = buf[a];
+
+            cc.fillRect(x, h - v, 1, v);
+        }
+    }
+    draw();
+};
+audio_eq.apply();

copyparty/web/splash.html

@@ -59,7 +59,7 @@
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
+            <form method="post" enctype="multipart/form-data" action="/">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />

copyparty/web/up2k.js

@@ -804,6 +804,14 @@ function up2k_init(subtle) {
 
                 var mou_ikkai = false;
 
+                if (st.busy.handshake.length > 0 &&
+                    st.busy.handshake[0].busied < Date.now() - 30 * 1000
+                ) {
+                    console.log("retrying stuck handshake");
+                    var t = st.busy.handshake.shift();
+                    st.todo.handshake.unshift(t);
+                }
+
                 if (st.todo.handshake.length > 0 &&
                     st.busy.handshake.length == 0 && (
                         st.todo.handshake[0].t4 || (
@@ -1019,11 +1027,27 @@ function up2k_init(subtle) {
     //
 
     function exec_handshake() {
-        var t = st.todo.handshake.shift();
+        var t = st.todo.handshake.shift(),
+            me = Date.now();
+
         st.busy.handshake.push(t);
+        t.busied = me;
 
         var xhr = new XMLHttpRequest();
+        xhr.onerror = function () {
+            if (t.busied != me) {
+                console.log('zombie handshake onerror,', t);
+                return;
+            }
+            console.log('handshake onerror, retrying');
+            st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+            st.todo.handshake.unshift(t);
+        };
         xhr.onload = function (e) {
+            if (t.busied != me) {
+                console.log('zombie handshake onload,', t);
+                return;
+            }
             if (xhr.status == 200) {
                 var response = JSON.parse(xhr.responseText);
 
@@ -1254,31 +1278,11 @@ function up2k_init(subtle) {
     window.addEventListener('resize', onresize);
     onresize();
 
-    function desc_show(e) {
-        var cfg = sread('tooltips');
-        if (cfg !== null && cfg != '1')
-            return;
-
-        var msg = this.getAttribute('alt'),
-            cdesc = ebi('u2cdesc');
-
-        cdesc.innerHTML = msg.replace(/\$N/g, "<br />");
-        cdesc.setAttribute('class', 'show');
-    }
-    function desc_hide(e) {
-        ebi('u2cdesc').setAttribute('class', '');
-    }
-    var o = QSA('#u2conf *[alt]');
+    var o = QSA('#u2conf *[tt]');
     for (var a = o.length - 1; a >= 0; a--) {
-        o[a].parentNode.getElementsByTagName('input')[0].setAttribute('alt', o[a].getAttribute('alt'));
-    }
-    var o = QSA('#u2conf *[alt]');
-    for (var a = 0; a < o.length; a++) {
-        o[a].onfocus = desc_show;
-        o[a].onblur = desc_hide;
-        o[a].onmouseenter = desc_show;
-        o[a].onmouseleave = desc_hide;
+        o[a].parentNode.getElementsByTagName('input')[0].setAttribute('tt', o[a].getAttribute('tt'));
     }
+    tt.init();
 
     function bumpthread(dir) {
         try {
@@ -1426,5 +1430,7 @@ function warn_uploader_busy(e) {
 }
 
 
+tt.init();
+
 if (QS('#op_up2k.act'))
     goto_up2k();

copyparty/web/upload.css

@@ -211,29 +211,6 @@
 	box-shadow: none;
 	opacity: .2;
 }
-#u2cdesc {
-	position: absolute;
-	width: 34em;
-	left: calc(50% - 15em);
-	background: #222;
-	border: 0 solid #555;
-	text-align: center;
-	overflow: hidden;
-	margin: 0 -2em;
-	padding: 0 1em;
-	height: 0;
-	opacity: .1;
-	transition: all 0.14s ease-in-out;
-	box-shadow: 0 .2em .5em #222;
-	border-radius: .4em;
-	z-index: 1;
-}
-#u2cdesc.show {
-	padding: 1em;
-	height: auto;
-	border-width: .2em 0;
-	opacity: 1;
-}
 #u2foot {
 	color: #fff;
 	font-style: italic;
@@ -286,10 +263,6 @@ html.light #u2conf .txtbox.err {
 	background: #f96;
 	color: #300;
 }
-html.light #u2cdesc {
-	background: #fff;
-	border: none;
-}
 html.light #op_up2k.srch #u2btn {
 	border-color: #a80;
 }

copyparty/web/upload.html

@@ -39,20 +39,20 @@
                     <td><br />parallel uploads:</td>
                     <td rowspan="2">
                         <input type="checkbox" id="multitask" />
-                        <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
+                        <label for="multitask" tt="continue hashing other files while uploading">🏃</label>
                     </td>
                     <td rowspan="2">
                         <input type="checkbox" id="ask_up" />
-                        <label for="ask_up" alt="ask for confirmation befofre upload starts">💭</label>
+                        <label for="ask_up" tt="ask for confirmation befofre upload starts">💭</label>
                     </td>
                     <td rowspan="2">
                         <input type="checkbox" id="flag_en" />
-                        <label for="flag_en" alt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
+                        <label for="flag_en" tt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
                     </td>
                 {%- if have_up2k_idx %}
                     <td data-perm="read" rowspan="2">
                         <input type="checkbox" id="fsearch" />
-                        <label for="fsearch" alt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
+                        <label for="fsearch" tt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
                     </td>
                 {%- endif %}
                     <td data-perm="read" rowspan="2" id="u2btn_cw"></td>
@@ -66,8 +66,6 @@
                 </tr>
             </table>
 
-            <div id="u2cdesc"></div>
-
             <div id="u2notbtn"></div>
 
             <div id="u2btn_ct">
@@ -80,11 +78,11 @@
             </div>
 
             <div id="u2cards">
-                <a href="#" act="ok">ok <span>0</span></a><a
-                href="#" act="ng">ng <span>0</span></a><a
-                href="#" act="done">done <span>0</span></a><a
-                href="#" act="bz" class="act">busy <span>0</span></a><a
-                href="#" act="q">que <span>0</span></a>
+                <a href="#" act="ok" tt="completed successfully">ok <span>0</span></a><a
+                href="#" act="ng" tt="failed / rejected / not-found">ng <span>0</span></a><a
+                href="#" act="done" tt="ok and ng combined">done <span>0</span></a><a
+                href="#" act="bz" tt="hashing or uploading" class="act">busy <span>0</span></a><a
+                href="#" act="q" tt="idle, pending">que <span>0</span></a>
             </div>
 
             <table id="u2tab">
@@ -92,7 +90,7 @@
                     <tr>
                         <td>filename</td>
                         <td>status</td>
-                        <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
+                        <td>progress<a href="#" id="u2cleanup" tt="remove completed uploads$N(makes it possible to upload a file after searching for it)">cleanup</a></td>
                     </tr>
                 </thead>
                 <tbody></tbody>

copyparty/web/util.js

@@ -528,3 +528,63 @@ function hist_replace(url) {
     console.log("h-repl " + url);
     history.replaceState(url, url, url);
 }
+
+
+var tt = (function () {
+    var r = {
+        "tt": mknod("div"),
+        "en": bcfg_get('tooltips', true),
+    };
+
+    r.tt.setAttribute('id', 'tt');
+    document.body.appendChild(r.tt);
+
+    function show() {
+        var cfg = sread('tooltips');
+        if (cfg !== null && cfg != '1')
+            return;
+
+        var msg = this.getAttribute('tt');
+        if (!msg)
+            return;
+
+        var pos = this.getBoundingClientRect(),
+            left = pos.left < window.innerWidth / 2,
+            top = pos.top < window.innerHeight / 2;
+
+        r.tt.style.top = top ? pos.bottom + 'px' : 'auto';
+        r.tt.style.bottom = top ? 'auto' : (window.innerHeight - pos.top) + 'px';
+        r.tt.style.left = left ? pos.left + 'px' : 'auto';
+        r.tt.style.right = left ? 'auto' : (window.innerWidth - pos.right) + 'px';
+
+        r.tt.innerHTML = msg.replace(/\$N/g, "<br />");
+        clmod(r.tt, 'show', 1);
+    }
+
+    function hide() {
+        clmod(r.tt, 'show');
+    }
+
+    r.init = function () {
+        var _show = r.en ? show : null,
+            _hide = r.en ? hide : null;
+
+        var o = QSA('*[tt]');
+        for (var a = o.length - 1; a >= 0; a--) {
+            o[a].onfocus = _show;
+            o[a].onblur = _hide;
+            o[a].onmouseenter = _show;
+            o[a].onmouseleave = _hide;
+        }
+        hide();
+    };
+
+    ebi('tooltips').onclick = function (e) {
+        ev(e);
+        r.en = !r.en;
+        bcfg_set('tooltips', r.en);
+        r.init();
+    };
+
+    return r;
+})();

docs/README.md

@@ -1,8 +1,19 @@
-## [`minimal-up2k.html`](minimal-up2k.html)
-* save as `.epilogue.html` inside a folder to [simplify the ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+# example `.epilogue.html`
+save one of these as `.epilogue.html` inside a folder to customize it:
 
-## [`browser.css`](browser.css)
-* example for `--css-browser`
+* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+
+
+# example browser-css
+point `--css-browser` to one of these by URL:
+
+* [`browser.css`](browser.css) changes the background
+* [`browser-icons.css`](browser-icons.css) adds filetype icons
+
+
+
+# other stuff
 
 ## [`rclone.md`](rclone.md)
 * notes on using rclone as a fuse client/server

docs/biquad.html

@@ -0,0 +1,95 @@
+<!DOCTYPE html><html><head></head><body><script>
+
+setTimeout(location.reload.bind(location), 700);
+document.documentElement.scrollLeft = 0;
+
+var can = document.createElement('canvas'),
+    cc = can.getContext('2d'),
+    w = 2048,
+    h = 1024;
+
+w = 2048;
+
+can.width = w;
+can.height = h;
+document.body.appendChild(can);
+can.style.cssText = 'width:' + w + 'px;height:' + h + 'px';
+
+cc.fillStyle = '#000';
+cc.fillRect(0, 0, w, h);
+
+var cfg = [ // hz, q, g
+    [31.25 * 0.88, 0, 1.4],  // shelf
+    [31.25 * 1.04, 0.7, 0.96],  // peak
+    [62.5, 0.7, 1],
+    [125, 0.8, 1],
+    [250, 0.9, 1.03],
+    [500, 0.9, 1.1],
+    [1000, 0.9, 1.1],
+    [2000, 0.9, 1.105],
+    [4000, 0.88, 1.05],
+    [8000 * 1.006, 0.73, 1.24],
+    //[16000 * 1.00, 0.5, 1.75],  // peak.v1
+    //[16000 * 1.19, 0, 1.8]  // shelf.v1
+    [16000 * 0.89, 0.7, 1.26],  // peak
+    [16000 * 1.13, 0.82, 1.09],  // peak
+    [16000 * 1.205, 0, 1.9]  // shelf
+];
+
+var freqs = new Float32Array(22000),
+    sum = new Float32Array(freqs.length),
+    ac = new AudioContext(),
+    step = w / freqs.length,
+    colors = [
+        'rgba(255, 0, 0, 0.7)',
+        'rgba(0, 224, 0, 0.7)',
+        'rgba(0, 64, 255, 0.7)'
+    ];
+
+var order = [];
+
+for (var a = 0; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var a = 1; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var ia = 0; ia < order.length; ia++) {
+    var a = order[ia],
+        fi = ac.createBiquadFilter(),
+        mag = new Float32Array(freqs.length),
+        phase = new Float32Array(freqs.length);
+
+    for (var b = 0; b < freqs.length; b++)
+        freqs[b] = b;
+
+    fi.type = a == 0 ? 'lowshelf' : a == cfg.length - 1 ? 'highshelf' : 'peaking';
+    fi.frequency.value = cfg[a][0];
+    fi.Q.value = cfg[a][1];
+    fi.gain.value = 1;
+
+    fi.getFrequencyResponse(freqs, mag, phase);
+    cc.fillStyle = colors[a % colors.length];
+    for (var b = 0; b < sum.length; b++) {
+        mag[b] -= 1;
+        sum[b] += mag[b] * cfg[a][2];
+        var y = h - (mag[b] * h * 3);
+        cc.fillRect(b * step, y, step, h - y);
+        cc.fillRect(b * step - 1, y - 1, 3, 3);
+    }
+}
+
+var min = 999999, max = 0;
+for (var a = 0; a < sum.length; a++) {
+    min = Math.min(min, sum[a]);
+    max = Math.max(max, sum[a]);
+}
+cc.fillStyle = 'rgba(255,255,255,1)';
+for (var a = 0; a < sum.length; a++) {
+    var v = (sum[a] - min) / (max - min);
+    cc.fillRect(a * step, 0, step, v * h / 2);
+}
+
+cc.fillRect(0, 460, w, 1);
+
+</script></body></html>

docs/browser-icons.css

@@ -0,0 +1,68 @@
+/* put filetype icons inline with text
+#ggrid>a>span:before,
+#ggrid>a>span.dir:before {
+	display: inline;
+	line-height: 0;
+	font-size: 1.7em;
+	margin: -.7em .1em -.5em -.6em;
+}
+*/
+
+
+/* move folder icons top-left */
+#ggrid>a>span.dir:before {
+	content: initial;
+}
+#ggrid>a[href$="/"]:before {
+	content: '📂';
+    display: block;
+    position: absolute;
+    margin: -.1em -.4em;
+    text-shadow: 0 0 .1em #000;
+    font-size: 2em;
+}
+
+
+/* put filetype icons top-left */
+#ggrid>a:before {
+    display: block;
+    position: absolute;
+    margin: -.1em -.4em;
+    text-shadow: 0 0 .1em #000;
+    font-size: 2em;
+}
+
+
+/* video */
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '📺';
+}
+
+
+/* audio */
+#ggrid>a:is(
+[href$=".mp3"i],
+[href$=".ogg"i],
+[href$=".opus"i],
+[href$=".flac"i],
+[href$=".m4a"i],
+[href$=".aac"i],
+):before {
+    content: '🎵';
+}
+
+
+/* image */
+#ggrid>a:is(
+[href$=".jpg"i],
+[href$=".jpeg"i],
+[href$=".png"i],
+[href$=".gif"i],
+[href$=".webp"i],
+):before {
+    content: '🎨';
+}

docs/browser.css

@@ -1,5 +1,5 @@
 html {
-    background: url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
+    background: #333 url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
 }
 #files th {
     background: rgba(32, 32, 32, 0.9) !important;
@@ -12,7 +12,7 @@ html {
 
 
 html.light {
-    background: url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
+    background: #eee url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
 }
 html.light #files th {
     background: rgba(255, 255, 255, 0.9) !important;

docs/notes.sh

@@ -86,6 +86,9 @@ var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.quer
 # get the size and video-id of all youtube vids in folder, assuming filename ends with -id.ext, and create a copyparty search query
 find -maxdepth 1 -printf '%s %p\n' | sort -n | awk '!/-([0-9a-zA-Z_-]{11})\.(mkv|mp4|webm)$/{next} {sub(/\.[^\.]+$/,"");n=length($0);v=substr($0,n-10);print $1, v}' | tee /dev/stderr | awk 'BEGIN {p="("} {printf("%s name like *-%s.* ",p,$2);p="or"} END {print ")\n"}' | cat >&2
 
+# unique stacks in a stackdump
+f=a; rm -rf stacks; mkdir stacks; grep -E '^#' $f | while IFS= read -r n; do awk -v n="$n" '!$0{o=0} o; $0==n{o=1}' <$f >stacks/f; h=$(sha1sum <stacks/f | cut -c-16); mv stacks/f stacks/$h-"$n"; done ; find stacks/ | sort | uniq -cw24
+
 
 ##
 ## sqlite3 stuff

docs/tcp-debug.sh

@@ -0,0 +1,32 @@
+(cd ~/dev/copyparty && strace -Tttyyvfs 256 -o strace.strace python3 -um copyparty -i 127.0.0.1 --http-only --stackmon /dev/shm/cpps,10 ) 2>&1 | tee /dev/stderr > ~/log-copyparty-$(date +%Y-%m%d-%H%M%S).txt
+
+14/Jun/2021:16:34:02 1623688447.212405 death
+14/Jun/2021:16:35:02 1623688502.420860 back
+
+tcpdump -nni lo -w /home/ed/lo.pcap
+
+# 16:35:25.324662 IP 127.0.0.1.48632 > 127.0.0.1.3920: Flags [F.], seq 849, ack 544, win 359, options [nop,nop,TS val 809396796 ecr 809396796], length 0
+
+tcpdump -nnr /home/ed/lo.pcap | awk '/ > 127.0.0.1.3920: /{sub(/ > .*/,"");sub(/.*\./,"");print}' | sort -n | uniq | while IFS= read -r port; do echo; tcpdump -nnr /home/ed/lo.pcap 2>/dev/null | grep -E "\.$port( > |: F)" | sed -r 's/ > .*, /, /'; done | grep -E '^16:35:0.*length [^0]' -C50
+
+16:34:02.441732 IP 127.0.0.1.48638, length 0
+16:34:02.441738 IP 127.0.0.1.3920, length 0
+16:34:02.441744 IP 127.0.0.1.48638, length 0
+16:34:02.441756 IP 127.0.0.1.48638, length 791
+16:34:02.441759 IP 127.0.0.1.3920, length 0
+16:35:02.445529 IP 127.0.0.1.48638, length 0
+16:35:02.489194 IP 127.0.0.1.3920, length 0
+16:35:02.515595 IP 127.0.0.1.3920, length 216
+16:35:02.515600 IP 127.0.0.1.48638, length 0
+
+grep 48638 "$(find ~ -maxdepth 1 -name log-copyparty-\*.txt | sort | tail -n 1)"
+
+1623688502.510380 48638 rh
+1623688502.511291 48638 Unrecv direct ...
+1623688502.511827 48638 rh = 791
+16:35:02.518 127.0.0.1 48638       shut(8): [Errno 107] Socket not connected
+Exception in thread httpsrv-0.1-48638:
+
+grep 48638 ~/dev/copyparty/strace.strace
+14561 16:35:02.506310 <... accept4 resumed> {sa_family=AF_INET, sin_port=htons(48638), sin_addr=inet_addr("127.0.0.1")}, [16], SOCK_CLOEXEC) = 8<TCP:[127.0.0.1:3920->127.0.0.1:48638]> <0.000012>
+15230 16:35:02.510725 write(1<pipe:[256639555]>, "1623688502.510380 48638 rh\n", 27 <unfinished ...>

scripts/copyparty-repack.sh

@@ -92,20 +92,34 @@ chmod 755 \
   copyparty-extras/copyparty-*/{scripts,bin}/*
 
 
-# extract and repack the sfx with less features enabled
+# extract the sfx
 ( cd copyparty-extras/sfx-full/
 ./copyparty-sfx.py -h
-cd ../copyparty-*/
-./scripts/make-sfx.sh re no-ogv no-cm
 )
 
 
-# put new sfx into copyparty-extras/sfx-lite/,
-# fuse client into copyparty-extras/,
+repack() {
+
+	# do the repack
+	(cd copyparty-extras/copyparty-*/
+	./scripts/make-sfx.sh $2
+	)
+
+	# put new sfx into copyparty-extras/$name/,
+	( cd copyparty-extras/
+	mv copyparty-*/dist/* $1/
+	)
+}
+
+repack sfx-full "re gz no-sh"
+repack sfx-lite "re no-ogv no-cm"
+repack sfx-lite "re no-ogv no-cm gz no-sh"
+
+
+# move fuse client into copyparty-extras/,
 # copy lite-sfx.py to ./copyparty,
 # delete extracted source code
 ( cd copyparty-extras/
-mv copyparty-*/dist/* sfx-lite/
 mv copyparty-*/bin/copyparty-fuse.py .
 cp -pv sfx-lite/copyparty-sfx.py ../copyparty
 rm -rf copyparty-{0..9}*.*.*{0..9}
@@ -119,6 +133,7 @@ true
 
 
 # create the bundle
+printf '\n\n'
 fn=copyparty-$(date +%Y-%m%d-%H%M%S).tgz
 tar -czvf "$od/$fn" *
 cd "$od"

scripts/make-sfx.sh

@@ -11,6 +11,10 @@ echo
 # `re` does a repack of an sfx which you already executed once
 #   (grabs files from the sfx-created tempdir), overrides `clean`
 #
+# `gz` creates a gzip-compressed python sfx instead of bzip2
+#
+# `no-sh` makes just the python sfx, skips the sh/unix sfx
+#
 # `no-ogv` saves ~500k by removing the opus/vorbis audio codecs
 #   (only affects apple devices; everything else has native support)
 #
@@ -167,7 +171,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.* copyparty/web/Makefile
+rm -f copyparty/web/deps/*.full.* copyparty/web/dbg-* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |

scripts/test/race.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import json
+import threading
+import http.client
+
+
+class Conn(object):
+    def __init__(self, ip, port):
+        self.s = http.client.HTTPConnection(ip, port, timeout=260)
+        self.st = []
+
+    def get(self, vpath):
+        self.st = [time.time()]
+
+        self.s.request("GET", vpath)
+        self.st.append(time.time())
+
+        ret = self.s.getresponse()
+        self.st.append(time.time())
+
+        if ret.status < 200 or ret.status >= 400:
+            raise Exception(ret.status)
+
+        ret = ret.read()
+        self.st.append(time.time())
+
+        return ret
+
+    def get_json(self, vpath):
+        ret = self.get(vpath)
+        return json.loads(ret)
+
+
+class CState(threading.Thread):
+    def __init__(self, cs):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.cs = cs
+        self.start()
+
+    def run(self):
+        colors = [5, 1, 3, 2, 7]
+        remotes = []
+        remotes_ok = False
+        while True:
+            time.sleep(0.001)
+            if not remotes_ok:
+                remotes = []
+                remotes_ok = True
+                for conn in self.cs:
+                    try:
+                        remotes.append(conn.s.sock.getsockname()[1])
+                    except:
+                        remotes.append("?")
+                        remotes_ok = False
+
+            m = []
+            for conn, remote in zip(self.cs, remotes):
+                stage = len(conn.st)
+                m.append(f"\033[3{colors[stage]}m{remote}")
+
+            m = " ".join(m)
+            print(f"{m}\033[0m\n\033[A", end="")
+
+
+def allget(cs, urls):
+    thrs = []
+    for c, url in zip(cs, urls):
+        t = threading.Thread(target=c.get, args=(url,))
+        t.start()
+        thrs.append(t)
+
+    for t in thrs:
+        t.join()
+
+
+def main():
+    os.system("")
+
+    ip, port = sys.argv[1].split(":")
+    port = int(port)
+
+    cs = []
+    for _ in range(64):
+        cs.append(Conn(ip, 3923))
+
+    CState(cs)
+
+    urlbase = "/doujin/c95"
+    j = cs[0].get_json(f"{urlbase}?ls")
+    urls = []
+    for d in j["dirs"]:
+        urls.append(f"{urlbase}/{d['href']}?th=w")
+
+    for n in range(100):
+        print(n)
+        allget(cs, urls)
+
+
+if __name__ == "__main__":
+    main()

tests/test_httpcli.py

@@ -28,6 +28,7 @@ class Cfg(Namespace):
             a=a,
             v=v,
             c=c,
+            rproxy=0,
             ed=False,
             no_zip=False,
             no_scandir=False,

tests/test_vfs.py

@@ -24,6 +24,7 @@ class Cfg(Namespace):
             "hist": None,
             "no_hash": False,
             "css_browser": None,
+            "rproxy": 0,
         }
         ex.update(ex2)
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)