v0.10.7

README.md

@@ -21,11 +21,13 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [status](#status)
 * [bugs](#bugs)
 * [usage](#usage)
+    * [zip downloads](#zip-downloads)
 * [searching](#searching)
     * [search configuration](#search-configuration)
     * [metadata from audio files](#metadata-from-audio-files)
     * [file parser plugins](#file-parser-plugins)
     * [complete examples](#complete-examples)
+* [browser support](#browser-support)
 * [client examples](#client-examples)
 * [dependencies](#dependencies)
     * [optional gpl stuff](#optional-gpl-stuff)
@@ -72,7 +74,7 @@ you may also want these, especially on servers:
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ folders as zip / tar files
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view
@@ -95,6 +97,8 @@ summary: it works! you can use it! (but technically not even close to beta)
 
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
 
@@ -108,6 +112,23 @@ the browser has the following hotkeys
 * `P` parent folder
 
 
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+
 # searching
 
 when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
@@ -176,6 +197,29 @@ copyparty can invoke external programs to collect additional metadata for files
   `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
 
 
+# browser support
+
+| feature         | ie6 | ie9 | ie10 | ie11 | ff 52+ | chr 49+ |
+| --------------- | --- | --- | ---- | ---- | ------ | ------- |
+| browse files    | yep | yep | yep  | yep  | yep    | yep     |
+| basic uploader  | yep | yep | yep  | yep  | yep    | yep     |
+| make directory  | yep | yep | yep  | yep  | yep    | yep     |
+| send message    | yep | yep | yep  | yep  | yep    | yep     |
+| set sort order  |  -  | yep | yep  | yep  | yep    | yep     |
+| zip selection   |  -  | yep | yep  | yep  | yep    | yep     |
+| directory tree  |  -  |  -  | `*1` | yep  | yep    | yep     |
+| up2k            |  -  |  -  | yep  | yep  | yep    | yep     |
+| icons work      |  -  |  -  | yep  | yep  | yep    | yep     |
+| markdown editor |  -  |  -  | yep  | yep  | yep    | yep     |
+| markdown viewer |  -  |  -  | yep  | yep  | yep    | yep     |
+| play mp3/mp4    |  -  | yep | yep  | yep  | yep    | yep     |
+| play ogg/opus   |  -  |  -  |  -   |  -   | yep    | yep     |
+
+* internet explorer 6 to 8 (and netscape 4.0) behave the same
+* firefox 52 and chrome 49 are the last winxp versions
+* `*1` only public folders (login session is dropped) and no history / back-button
+
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)
@@ -286,7 +330,6 @@ roughly sorted by priority
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* drop onto folders
 * `os.copy_file_range` for up2k cloning
 * up2k partials ui
 * support pillow-simd

copyparty/__main__.py

@@ -177,11 +177,14 @@ def sighandler(signal=None, frame=None):
     print("\n".join(msg))
 
 
-def main():
+def main(argv=None):
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
 
+    if argv is None:
+        argv = sys.argv
+
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
@@ -194,13 +197,13 @@ def main():
     deprecated = [["-e2s", "-e2ds"]]
     for dk, nk in deprecated:
         try:
-            idx = sys.argv.index(dk)
+            idx = argv.index(dk)
         except:
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
         print(msg.format(dk, nk))
-        sys.argv[idx] = nk
+        argv[idx] = nk
         time.sleep(2)
 
     ap = argparse.ArgumentParser(
@@ -261,6 +264,7 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
     ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
@@ -289,7 +293,7 @@ def main():
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     
-    al = ap.parse_args()
+    al = ap.parse_args(args=argv[1:])
     # fmt: on
 
     # propagate implications

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 9, 13)
-CODENAME = "the strongest music server"
-BUILD_DT = (2021, 3, 23)
+VERSION = (0, 10, 7)
+CODENAME = "zip it"
+BUILD_DT = (2021, 4, 3)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import os
 import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
@@ -53,6 +54,7 @@ class VFS(object):
                 self.uwrite,
                 self.flags,
             )
+            self._trk(vn)
             self.nodes[name] = vn
             return self._trk(vn.add(src, dst))
 
@@ -127,6 +129,78 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+
+                rm = [x for x in rd if x[0] not in flt]
+                [rd.remove(x) for x in rm]
+
+                rm = [x for x in vd.keys() if x not in flt]
+                [vd.pop(x) for x in rm]
+
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)

copyparty/broker_mp.py

@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()

copyparty/httpcli.py

@@ -7,6 +7,7 @@ import gzip
 import time
 import copy
 import json
+import string
 import socket
 import ctypes
 from datetime import datetime
@@ -14,6 +15,8 @@ import calendar
 
 from .__init__ import E, PY2, WINDOWS
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .szip import StreamZip
+from .star import StreamTar
 
 if not PY2:
     unicode = str
@@ -52,6 +55,10 @@ class HttpCli(object):
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def j2(self, name, **kwargs):
+        tpl = self.conn.hsrv.j2[name]
+        return tpl.render(**kwargs) if kwargs else tpl
+
     def run(self):
         """returns true if connection can be reused"""
         self.keepalive = False
@@ -154,7 +161,9 @@ class HttpCli(object):
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
                 self.keepalive = self._check_nonfatal(ex)
-                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
+                self.log("{}\033[0m, {}".format(str(ex), self.vpath), 3)
+                msg = "<pre>{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                self.reply(msg.encode("utf-8", "replace"), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
@@ -312,8 +321,19 @@ class HttpCli(object):
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
                 for buf in reader:
-                    buf = buf.decode("utf-8", "replace")
-                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
+                    orig = buf.decode("utf-8", "replace")
+                    m = "urlform_raw {} @ {}\n  {}\n"
+                    self.log(m.format(len(orig), self.vpath, orig))
+                    try:
+                        plain = unquote(buf.replace(b"+", b" "))
+                        plain = plain.decode("utf-8", "replace")
+                        if buf.startswith(b"msg="):
+                            plain = plain[4:]
+
+                        m = "urlform_dec {} @ {}\n  {}\n"
+                        self.log(m.format(len(plain), self.vpath, plain))
+                    except Exception as ex:
+                        self.log(repr(ex))
 
             if "get" in opt:
                 return self.handle_get()
@@ -388,8 +408,30 @@ class HttpCli(object):
         if act == "tput":
             return self.handle_text_upload()
 
+        if act == "zip":
+            return self.handle_zip_post()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
+    def handle_zip_post(self):
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                break
+
+        if v is None:
+            raise Pebkac(422, "need zip or tar keyword")
+
+        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        items = self.parser.require("files", 1024 * 1024)
+        if not items:
+            raise Pebkac(422, "need files list")
+
+        items = items.replace("\r", "").split("\n")
+        items = [unquotep(x) for x in items if items]
+
+        return self.tx_zip(k, v, vn, rem, items, self.args.ed)
+
     def handle_post_json(self):
         try:
             remains = int(self.headers["content-length"])
@@ -417,15 +459,18 @@ class HttpCli(object):
         if "srch" in self.uparam or "srch" in body:
             return self.handle_search(body)
 
-        # prefer this over undot; no reason to allow traversion
-        if "/" in body["name"]:
-            raise Pebkac(400, "folders verboten")
-
         # up2k-php compat
         for k in "chunkpit.php", "handshake.php":
             if self.vpath.endswith(k):
                 self.vpath = self.vpath[: -len(k)]
 
+        sub = None
+        name = undot(body["name"])
+        if "/" in name:
+            sub, name = name.rsplit("/", 1)
+            self.vpath = "/".join([self.vpath, sub]).strip("/")
+            body["name"] = name
+
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
 
         body["vtop"] = vfs.vpath
@@ -434,12 +479,22 @@ class HttpCli(object):
         body["addr"] = self.ip
         body["vcfg"] = vfs.flags
 
-        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
-        response = x.get()
-        response = json.dumps(response)
+        if sub:
+            try:
+                dst = os.path.join(vfs.realpath, rem)
+                os.makedirs(dst)
+            except:
+                if not os.path.isdir(dst):
+                    raise Pebkac(400, "some file got your folder name")
 
-        self.log(response)
-        self.reply(response.encode("utf-8"), mime="application/json")
+        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
+        ret = x.get()
+        if sub:
+            ret["name"] = "/".join([sub, ret["name"]])
+
+        ret = json.dumps(ret)
+        self.log(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
         return True
 
     def handle_search(self, body):
@@ -580,7 +635,7 @@ class HttpCli(object):
             pwd = "x"  # nosec
 
         h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
-        html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
+        html = self.j2("msg", h1=msg, h2='<a href="/">ack</a>', redir="/")
         self.reply(html.encode("utf-8"), headers=h)
         return True
 
@@ -611,7 +666,8 @@ class HttpCli(object):
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         esc_paths = [quotep(vpath), html_escape(vpath)]
-        html = self.conn.tpl_msg.render(
+        html = self.j2(
+            "msg",
             h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
             pre="aight",
             click=True,
@@ -643,7 +699,8 @@ class HttpCli(object):
                 f.write(b"`GRUNNUR`\n")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        html = self.conn.tpl_msg.render(
+        html = self.j2(
+            "msg",
             h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
                 quotep(vpath), html_escape(vpath)
             ),
@@ -749,7 +806,8 @@ class HttpCli(object):
                     ).encode("utf-8")
                 )
 
-        html = self.conn.tpl_msg.render(
+        html = self.j2(
+            "msg",
             h2='<a href="/{}">return to /{}</a>'.format(
                 quotep(self.vpath), html_escape(self.vpath)
             ),
@@ -1037,16 +1095,75 @@ class HttpCli(object):
         self.log("{},  {}".format(logmsg, spd))
         return ret
 
+    def tx_zip(self, fmt, uarg, vn, rem, items, dots):
+        if self.args.no_zip:
+            raise Pebkac(400, "not enabled")
+
+        logmsg = "{:4} {} ".format("", self.req)
+        self.keepalive = False
+
+        if not uarg:
+            uarg = ""
+
+        if fmt == "tar":
+            mime = "application/x-tar"
+            packer = StreamTar
+        else:
+            mime = "application/zip"
+            packer = StreamZip
+
+        fn = items[0] if items and items[0] else self.vpath
+        if fn:
+            fn = fn.rstrip("/").split("/")[-1]
+        else:
+            fn = self.headers.get("host", "hey")
+
+        afn = "".join(
+            [x if x in (string.ascii_letters + string.digits) else "_" for x in fn]
+        )
+
+        bascii = unicode(string.ascii_letters + string.digits).encode("utf-8")
+        ufn = fn.encode("utf-8", "xmlcharrefreplace")
+        if PY2:
+            ufn = [unicode(x) if x in bascii else "%{:02x}".format(ord(x)) for x in ufn]
+        else:
+            ufn = [
+                chr(x).encode("utf-8")
+                if x in bascii
+                else "%{:02x}".format(x).encode("ascii")
+                for x in ufn
+            ]
+        ufn = b"".join(ufn).decode("ascii")
+
+        cdis = "attachment; filename=\"{}.{}\"; filename*=UTF-8''{}.{}"
+        cdis = cdis.format(afn, fmt, ufn, fmt)
+        self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis})
+
+        fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
+        # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
+        bsent = 0
+        for buf in bgen.gen():
+            if not buf:
+                break
+
+            try:
+                self.s.sendall(buf)
+                bsent += len(buf)
+            except:
+                logmsg += " \033[31m" + unicode(bsent) + "\033[0m"
+                break
+
+        spd = self._spd(bsent)
+        self.log("{},  {}".format(logmsg, spd))
+        return True
+
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
-        if "edit2" in self.uparam:
-            html_path = "web/mde.html"
-            template = self.conn.tpl_mde
-        else:
-            html_path = "web/md.html"
-            template = self.conn.tpl_md
 
-        html_path = os.path.join(E.mod, html_path)
+        tpl = "mde" if "edit2" in self.uparam else "md"
+        html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
+        template = self.j2(tpl)
 
         st = os.stat(fsenc(fs_path))
         # sz_md = st.st_size
@@ -1098,7 +1215,7 @@ class HttpCli(object):
     def tx_mounts(self):
         rvol = [x + "/" if x else x for x in self.rvol]
         wvol = [x + "/" if x else x for x in self.wvol]
-        html = self.conn.tpl_mounts.render(this=self, rvol=rvol, wvol=wvol)
+        html = self.j2("splash", this=self, rvol=rvol, wvol=wvol)
         self.reply(html.encode("utf-8"))
         return True
 
@@ -1187,6 +1304,11 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                return self.tx_zip(k, v, vn, rem, [], self.args.ed)
+
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
@@ -1247,8 +1369,11 @@ class HttpCli(object):
 
             is_dir = stat.S_ISDIR(inf.st_mode)
             if is_dir:
-                margin = "DIR"
                 href += "/"
+                if self.args.no_zip:
+                    margin = "DIR"
+                else:
+                    margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
                     base, html_escape(hist[fn][2], quote=True), hist[fn][0]
@@ -1372,7 +1497,8 @@ class HttpCli(object):
 
         dirs.extend(files)
 
-        html = self.conn.tpl_browser.render(
+        html = self.j2(
+            "browser",
             vdir=quotep(self.vpath),
             vpnodes=vpnodes,
             files=dirs,
@@ -1384,6 +1510,7 @@ class HttpCli(object):
             ),
             have_up2k_idx=("e2d" in vn.flags),
             have_tags_idx=("e2t" in vn.flags),
+            have_zip=(not self.args.no_zip),
             logues=logues,
             title=html_escape(self.vpath),
             srv_info=srv_info,

copyparty/httpconn.py

@@ -12,23 +12,6 @@ try:
 except:
     HAVE_SSL = False
 
-try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * {} -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-""".format(
-            os.path.basename(sys.executable)
-        )
-    )
-    sys.exit(1)
-
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
@@ -57,14 +40,6 @@ class HttpConn(object):
         self.log_func = hsrv.log
         self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
-
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -112,7 +87,9 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
-                self.log(err)
+                if method:
+                    self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 

copyparty/httpsrv.py

@@ -2,10 +2,28 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
 from .__init__ import E, MACOS
 from .httpconn import HttpConn
 from .authsrv import AuthSrv
@@ -30,6 +48,13 @@ class HttpSrv(object):
         self.workload_thr_alive = False
         self.auth = AuthSrv(self.args, self.log)
 
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "msg", "md", "mde"]
+        }
+
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
             self.cert_path = cert_path

copyparty/star.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen)
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/sutil.py

@@ -0,0 +1,25 @@
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/szip.py

@@ -0,0 +1,271 @@
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, "/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/up2k.py

@@ -232,7 +232,8 @@ class Up2k(object):
                 (ft if v is True else ff if v is False else fv).format(k, str(v))
                 for k, v in flags.items()
             ]
-            self.log(" ".join(sorted(a)) + "\033[0m")
+            if a:
+                self.log(" ".join(sorted(a)) + "\033[0m")
 
             reg = {}
             path = os.path.join(ptop, ".hist", "up2k.snap")
@@ -1309,6 +1310,7 @@ class Up2k(object):
                     self.log("no cursor to write tags with??", c=1)
                     continue
 
+                # TODO is undef if vol 404 on startup
                 entags = self.entags[ptop]
                 if not entags:
                     self.log("no entags okay.jpg", c=3)

copyparty/util.py

@@ -576,11 +576,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
-    fn = fn.replace("\\", "/").split("/")[-1]
+def sanitize_fn(fn, ok=""):
+    if "/" not in ok:
+        fn = fn.replace("\\", "/").split("/")[-1]
 
     if WINDOWS:
-        for bad, good in [
+        for bad, good in [x for x in [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -590,7 +591,7 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
+        ] if x[0] not in ok]:
             fn = fn.replace(bad, good)
 
         bad = ["con", "prn", "aux", "nul"]
@@ -780,6 +781,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()

copyparty/web/browser.css

@@ -182,6 +182,11 @@ a, #files tbody div a:last-child {
 	color: #840;
 	text-shadow: 0 0 .3em #b80;
 }
+#files tbody tr.sel td {
+	background: #80b;
+	color: #fff;
+	border-color: #a3d;
+}
 #blocked {
 	position: fixed;
 	top: 0;
@@ -268,6 +273,25 @@ a, #files tbody div a:last-child {
 	padding: .2em 0 0 .07em;
 	color: #fff;
 }
+#wtoggle>span {
+	display: none;
+}
+#wtoggle.sel {
+	width: 4.27em;
+}
+#wtoggle.sel>span {
+	display: inline-block;
+	line-height: 0;
+}
+#wtoggle.sel>span a {
+	font-size: .4em;
+	margin: -.3em 0;
+	position: relative;
+	display: inline-block;
+}
+#wtoggle.sel>span #selzip {
+	top: -.6em;
+}
 #barpos,
 #barbuf {
 	position: absolute;
@@ -598,7 +622,8 @@ input[type="checkbox"]:checked+label {
 #files td.min a {
 	display: none;
 }
-#files tr.play td {
+#files tr.play td,
+#files tr.play div a {
 	background: #fc4;
 	border-color: transparent;
 	color: #400;

copyparty/web/browser.html

@@ -41,10 +41,12 @@
     <div id="op_cfg" class="opview opbox">
         <h3>key notation</h3>
         <div id="key_notation"></div>
+        {%- if have_zip %}
+        <h3>folder download</h3>
+        <div id="arc_fmt"></div>
+        {%- endif %}
         <h3>tooltips</h3>
-        <div>
-            <a id="tooltips" class="tglbtn" href="#">enable</a>
-        </div>
+        <div><a id="tooltips" class="tglbtn" href="#">enable</a></div>
     </div>
     
     <h1 id="path">
@@ -70,7 +72,7 @@
     <table id="files">
         <thead>
             <tr>
-                <th></th>
+                <th name="lead"><span>c</span></th>
                 <th name="href"><span>File Name</span></th>
                 <th name="sz" sort="int"><span>Size</span></th>
                 {%- for k in taglist %}
@@ -110,7 +112,14 @@
     {%- endif %}
 
     <div id="widget">
-        <div id="wtoggle">♫</div>
+        <div id="wtoggle">
+            <span>
+                <a href="#" id="selall">sel.<br />all</a>
+                <a href="#" id="selinv">sel.<br />inv.</a>
+                <a href="#" id="selzip">zip</a>
+            </span>
+            ♫
+        </div>
         <div id="widgeti">
             <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
             <canvas id="pvol" width="288" height="38"></canvas>

copyparty/web/browser.js

@@ -500,7 +500,7 @@ function play(tid, call_depth) {
 	setclass(oid, 'play act');
 	var trs = ebi('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr');
 	for (var a = 0, aa = trs.length; a < aa; a++) {
-		trs[a].className = trs[a].className.replace(/ *play */, "");
+		clmod(trs[a], 'play');
 	}
 	ebi(oid).parentElement.parentElement.className += ' play';
 
@@ -649,10 +649,10 @@ function tree_up() {
 
 
 document.onkeydown = function (e) {
-	if (document.activeElement != document.body && document.activeElement.nodeName.toLowerCase() != 'a')
+	if (!document.activeElement || document.activeElement != document.body && document.activeElement.nodeName.toLowerCase() != 'a')
 		return;
 
-	var k = e.code, pos = -1;
+	var k = (e.code + ''), pos = -1;
 	if (k.indexOf('Digit') === 0)
 		pos = parseInt(k.slice(-1)) * 0.1;
 
@@ -753,7 +753,7 @@ document.onkeydown = function (e) {
 		clearTimeout(search_timeout);
 		var now = new Date().getTime();
 		if (now - search_in_progress > 30 * 1000)
-			search_timeout = setTimeout(do_search, 100);
+			search_timeout = setTimeout(do_search, 200);
 	}
 
 	function do_search() {
@@ -772,6 +772,7 @@ document.onkeydown = function (e) {
 		// ebi('srch_q').textContent = JSON.stringify(params, null, 4);
 		var xhr = new XMLHttpRequest();
 		xhr.open('POST', '/?srch', true);
+		xhr.setRequestHeader('Content-Type', 'text/plain');
 		xhr.onreadystatechange = xhr_search_results;
 		xhr.ts = new Date().getTime();
 		xhr.send(JSON.stringify(params));
@@ -796,6 +797,8 @@ document.onkeydown = function (e) {
 		var res = JSON.parse(this.responseText),
 			tagord = res.tag_order;
 
+		sortfiles(res.hits);
+
 		var ofiles = ebi('files');
 		if (ofiles.getAttribute('ts') > this.ts)
 			return;
@@ -814,7 +817,7 @@ document.onkeydown = function (e) {
 
 		var html = mk_files_header(tagord);
 		html.push('<tbody>');
-		html.push('<tr><td>-</td><td colspan="42"><a href="#" id="unsearch">close search results</a></td></tr>');
+		html.push('<tr><td>-</td><td colspan="42"><a href="#" id="unsearch">! close search results</a></td></tr>');
 		for (var a = 0; a < res.hits.length; a++) {
 			var r = res.hits[a],
 				ts = parseInt(r.ts),
@@ -833,7 +836,7 @@ document.onkeydown = function (e) {
 					v = r.tags[k] || "";
 
 				if (k == ".dur") {
-					var sv = s2ms(v);
+					var sv = v ? s2ms(v) : "";
 					nodes[nodes.length - 1] += '</td><td sortv="' + v + '">' + sv;
 					continue;
 				}
@@ -867,6 +870,7 @@ document.onkeydown = function (e) {
 		oldcfg = [];
 		ebi('files').innerHTML = orig_html;
 		orig_html = null;
+		msel.render();
 		reload_browser();
 	}
 })();
@@ -995,8 +999,6 @@ var treectl = (function () {
 					var o = links[a].parentNode;
 					if (!o.getElementsByTagName('li').length)
 						o.innerHTML = html;
-					//else
-					//	links[a].previousSibling.textContent = '-';
 				}
 			}
 		}
@@ -1085,32 +1087,8 @@ var treectl = (function () {
 		}
 
 		ebi('srv_info').innerHTML = '<span>' + res.srvinf + '</span>';
-		var nodes = res.dirs.concat(res.files),
-			sopts = jread('fsort', []);
-
-		try {
-			for (var a = sopts.length - 1; a >= 0; a--) {
-				var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
-				if (name.indexOf('tags/') == -1) {
-					nodes.sort(function (v1, v2) {
-						if (!v1[name]) return -1 * rev;
-						if (!v2[name]) return 1 * rev;
-						return rev * (typ == 'int' ? (v1[name] - v2[name]) : (v1[name].localeCompare(v2[name])));
-					});
-				}
-				else {
-					name = name.slice(5);
-					nodes.sort(function (v1, v2) {
-						if (!v1.tags[name]) return -1 * rev;
-						if (!v2.tags[name]) return 1 * rev;
-						return rev * (typ == 'int' ? (v1.tags[name] - v2.tags[name]) : (v1.tags[name].localeCompare(v2.tags[name])));
-					});
-				}
-			}
-		}
-		catch (ex) {
-			console.log("failed to apply sort config: " + ex);
-		}
+		var nodes = res.dirs.concat(res.files);
+		nodes = sortfiles(nodes);
 
 		var top = this.top;
 		var html = mk_files_header(res.taglist);
@@ -1125,7 +1103,7 @@ var treectl = (function () {
 					v = (r.tags || {})[k] || "";
 
 				if (k == ".dur") {
-					var sv = s2ms(v);
+					var sv = v ? s2ms(v) : "";
 					ln[ln.length - 1] += '</td><td sortv="' + v + '">' + sv;
 					continue;
 				}
@@ -1149,6 +1127,7 @@ var treectl = (function () {
 
 		filecols.set_style();
 		mukey.render();
+		msel.render();
 		reload_tree();
 		reload_browser();
 	}
@@ -1303,8 +1282,8 @@ function find_file_col(txt) {
 
 function mk_files_header(taglist) {
 	var html = [
-		'<thead>',
-		'<th></th>',
+		'<thead><tr>',
+		'<th name="lead"><span>c</span></th>',
 		'<th name="href"><span>File Name</span></th>',
 		'<th name="sz" sort="int"><span>Size</span></th>'
 	];
@@ -1322,7 +1301,7 @@ function mk_files_header(taglist) {
 	html = html.concat([
 		'<th name="ext"><span>T</span></th>',
 		'<th name="ts"><span>Date</span></th>',
-		'</thead>',
+		'</tr></thead>',
 	]);
 	return html;
 }
@@ -1356,13 +1335,13 @@ var filecols = (function () {
 				continue;
 
 			var name = span[0].textContent,
-				cls = '';
+				cls = false;
 
 			if (has(hidden, name)) {
 				ohidden.push(a);
-				cls = ' min';
+				cls = true;
 			}
-			ths[a].className = ths[a].className.replace(/ *min */, " ") + cls;
+			clmod(ths[a], 'min', cls)
 		}
 		for (var a = 0; a < ncols; a++) {
 			var cls = has(ohidden, a) ? 'min' : '';
@@ -1407,8 +1386,8 @@ var filecols = (function () {
 		if (!min)
 			for (var a = 0, aa = rows.length; a < aa; a++) {
 				var c = rows[a].cells[i];
-				if (c)
-					var v = c.textContent = s2ms(c.textContent);
+				if (c && c.textContent)
+					c.textContent = s2ms(c.textContent);
 			}
 	}
 	catch (ex) { }
@@ -1479,8 +1458,11 @@ var mukey = (function () {
 	}
 
 	function render() {
-		var ci = find_file_col('Key'),
-			i = ci[0],
+		var ci = find_file_col('Key');
+		if (!ci)
+			return;
+
+		var i = ci[0],
 			min = ci[1],
 			rows = ebi('files').tBodies[0].rows;
 
@@ -1529,10 +1511,13 @@ var mukey = (function () {
 
 
 function addcrc() {
-	var links = document.querySelectorAll('#files>tbody>tr>td:nth-child(2)>a');
+	var links = document.querySelectorAll(
+		'#files>tbody>tr>td:first-child+td>' + (
+			ebi('unsearch') ? 'div>a:last-child' : 'a'));
+
 	for (var a = 0, aa = links.length; a < aa; a++)
 		if (!links[a].getAttribute('id'))
-			links[a].setAttribute('id', 'f-' + crc32(links[a].textContent));
+			links[a].setAttribute('id', 'f-' + crc32(links[a].textContent || links[a].innerText));
 }
 
 
@@ -1559,6 +1544,140 @@ function addcrc() {
 })();
 
 
+var arcfmt = (function () {
+	if (!ebi('arc_fmt'))
+		return { "render": function () { } };
+
+	var html = [],
+		arcfmts = ["tar", "zip", "zip_dos", "zip_crc"],
+		arcv = ["tar", "zip=utf8", "zip", "zip=crc"];
+
+	for (var a = 0; a < arcfmts.length; a++) {
+		var k = arcfmts[a];
+		html.push(
+			'<span><input type="radio" name="arcfmt" value="' + k + '" id="arcfmt_' + k + '">' +
+			'<label for="arcfmt_' + k + '">' + k + '</label></span>');
+	}
+	ebi('arc_fmt').innerHTML = html.join('\n');
+
+	var fmt = sread("arc_fmt") || "zip";
+	ebi('arcfmt_' + fmt).checked = true;
+
+	function render() {
+		var arg = arcv[arcfmts.indexOf(fmt)],
+			tds = document.querySelectorAll('#files tbody td:first-child a');
+
+		for (var a = 0, aa = tds.length; a < aa; a++) {
+			var o = tds[a], txt = o.textContent, href = o.getAttribute('href');
+			if (txt != 'tar' && txt != 'zip')
+				continue;
+
+			var ofs = href.lastIndexOf('?');
+			if (ofs < 0)
+				throw 'missing arg in url';
+
+			o.setAttribute("href", href.slice(0, ofs + 1) + arg);
+			o.textContent = fmt.split('_')[0];
+		}
+		ebi('selzip').textContent = fmt.split('_')[0];
+		ebi('selzip').setAttribute('fmt', arg);
+	}
+
+	function try_render() {
+		try {
+			render();
+		}
+		catch (ex) {
+			console.log("arcfmt failed: " + ex);
+		}
+	}
+
+	function change_fmt(e) {
+		ev(e);
+		fmt = this.getAttribute('value');
+		swrite("arc_fmt", fmt);
+		try_render();
+	}
+
+	var o = document.querySelectorAll('#arc_fmt input');
+	for (var a = 0; a < o.length; a++) {
+		o[a].onchange = change_fmt;
+	}
+
+	return {
+		"render": try_render
+	};
+})();
+
+
+var msel = (function () {
+	function getsel() {
+		var names = [];
+		var links = document.querySelectorAll('#files tbody tr.sel td:nth-child(2) a');
+		for (var a = 0, aa = links.length; a < aa; a++)
+			names.push(links[a].getAttribute('href').replace(/\/$/, "").split('/').slice(-1));
+
+		return names;
+	}
+	function selui() {
+		clmod(ebi('wtoggle'), 'sel', getsel().length);
+	}
+	function seltgl(e) {
+		ev(e);
+		var tr = this.parentNode;
+		clmod(tr, 'sel', 't');
+		selui();
+	}
+	function evsel(e, fun) {
+		ev(e);
+		var trs = document.querySelectorAll('#files tbody tr');
+		for (var a = 0, aa = trs.length; a < aa; a++)
+			clmod(trs[a], 'sel', fun);
+		selui();
+	}
+	ebi('selall').onclick = function (e) {
+		evsel(e, "add");
+	};
+	ebi('selinv').onclick = function (e) {
+		evsel(e, "t");
+	};
+	ebi('selzip').onclick = function (e) {
+		ev(e);
+		var names = getsel();
+		var arg = ebi('selzip').getAttribute('fmt');
+		var txt = names.join('\n');
+		var frm = document.createElement('form');
+		frm.setAttribute('action', '?' + arg);
+		frm.setAttribute('method', 'post');
+		frm.setAttribute('target', '_blank');
+		frm.setAttribute('enctype', 'multipart/form-data');
+		frm.innerHTML = '<input name="act" value="zip" />' +
+			'<textarea name="files" id="ziptxt"></textarea>';
+		frm.style.display = 'none';
+
+		var oldform = document.querySelector('#widgeti>form');
+		if (oldform)
+			oldform.parentNode.removeChild(oldform);
+
+		ebi('widgeti').appendChild(frm);
+		var obj = ebi('ziptxt');
+		obj.value = txt;
+		console.log(txt);
+		frm.submit();
+	};
+	function render() {
+		var tds = document.querySelectorAll('#files tbody td+td+td');
+		for (var a = 0, aa = tds.length; a < aa; a++) {
+			tds[a].onclick = seltgl;
+		}
+		arcfmt.render();
+	}
+	return {
+		"render": render
+	};
+})();
+
+
 function ev_row_tgl(e) {
 	ev(e);
 	filecols.toggle(this.parentElement.parentElement.getElementsByTagName('span')[0].textContent);
@@ -1611,3 +1730,4 @@ function reload_browser(not_mp) {
 }
 reload_browser(true);
 mukey.render();
+msel.render();

copyparty/web/md.html

@@ -147,7 +147,7 @@ var md_opt = {
 
 	</script>
     <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.full.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}
 	<script src="/.cpr/md2.js"></script>

copyparty/web/up2k.js

@@ -278,18 +278,27 @@ function up2k_init(have_crypto) {
         }
         else files = e.target.files;
 
-        if (files.length == 0)
+        if (!files || files.length == 0)
             return alert('no files selected??');
 
         more_one_file();
         var bad_files = [];
         var good_files = [];
+        var dirs = [];
         for (var a = 0; a < files.length; a++) {
             var fobj = files[a];
             if (is_itemlist) {
                 if (fobj.kind !== 'file')
                     continue;
 
+                try {
+                    var wi = fobj.webkitGetAsEntry();
+                    if (wi.isDirectory) {
+                        dirs.push(wi);
+                        continue;
+                    }
+                }
+                catch (ex) { }
                 fobj = fobj.getAsFile();
             }
             try {
@@ -300,12 +309,69 @@ function up2k_init(have_crypto) {
                 bad_files.push(fobj.name);
                 continue;
             }
-            good_files.push(fobj);
+            good_files.push([fobj, fobj.name]);
+        }
+        if (dirs) {
+            return read_dirs(null, [], dirs, good_files, bad_files);
+        }
+    }
+
+    function read_dirs(rd, pf, dirs, good, bad) {
+        if (!dirs.length) {
+            if (!pf.length)
+                return gotallfiles(good, bad);
+
+            console.log("retry pf, " + pf.length);
+            setTimeout(function () {
+                read_dirs(rd, pf, dirs, good, bad);
+            }, 50);
+            return;
         }
 
+        if (!rd)
+            rd = dirs[0].createReader();
+
+        rd.readEntries(function (ents) {
+            var ngot = 0;
+            ents.forEach(function (dn) {
+                if (dn.isDirectory) {
+                    dirs.push(dn);
+                }
+                else {
+                    var name = dn.fullPath;
+                    if (name.indexOf('/') === 0)
+                        name = name.slice(1);
+
+                    pf.push(name);
+                    dn.file(function (fobj) {
+                        var idx = pf.indexOf(name);
+                        pf.splice(idx, 1);
+                        try {
+                            if (fobj.size > 0) {
+                                good.push([fobj, name]);
+                                return;
+                            }
+                        }
+                        catch (ex) { }
+                        bad.push(name);
+                    });
+                }
+                ngot += 1;
+            });
+            // console.log("ngot: " + ngot);
+            if (!ngot) {
+                dirs.shift();
+                rd = null;
+            }
+            return read_dirs(rd, pf, dirs, good, bad);
+        });
+    }
+
+    function gotallfiles(good_files, bad_files) {
         if (bad_files.length > 0) {
-            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, files.length);
-            for (var a = 0; a < bad_files.length; a++)
+            var ntot = bad_files.length + good_files.length;
+            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
+            for (var a = 0, aa = Math.min(20, bad_files.length); a < aa; a++)
                 msg += '-- ' + bad_files[a] + '\n';
 
             if (files.length - bad_files.length <= 1 && /(android)/i.test(navigator.userAgent))
@@ -315,21 +381,21 @@ function up2k_init(have_crypto) {
         }
 
         var msg = ['upload these ' + good_files.length + ' files?'];
-        for (var a = 0; a < good_files.length; a++)
-            msg.push(good_files[a].name);
+        for (var a = 0, aa = Math.min(20, good_files.length); a < aa; a++)
+            msg.push(good_files[a][1]);
 
         if (ask_up && !fsearch && !confirm(msg.join('\n')))
             return;
 
         for (var a = 0; a < good_files.length; a++) {
-            var fobj = good_files[a];
+            var fobj = good_files[a][0];
             var now = new Date().getTime();
             var lmod = fobj.lastModified || now;
             var entry = {
                 "n": parseInt(st.files.length.toString()),
-                "t0": now,  // TODO remove probably
+                "t0": now,
                 "fobj": fobj,
-                "name": fobj.name,
+                "name": good_files[a][1],
                 "size": fobj.size,
                 "lmod": lmod / 1000,
                 "purl": get_evpath(),
@@ -458,7 +524,7 @@ function up2k_init(have_crypto) {
 
                 if (st.todo.handshake.length > 0 &&
                     st.busy.handshake.length == 0 && (
-                        st.todo.handshake[0].t3 || (
+                        st.todo.handshake[0].t4 || (
                             handshakes_permitted() &&
                             st.busy.upload.length < parallel_uploads
                         )
@@ -802,27 +868,32 @@ function up2k_init(have_crypto) {
                     t.done = true;
                     st.bytes.uploaded += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
-                    var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
+                    var spd2 = (t.size / ((t.t4 - t.t3) / 1000.)) / (1024 * 1024.);
                     ebi('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2));
                 }
-                else t.t3 = undefined;
+                else t.t4 = undefined;
 
                 tasker();
             }
             else {
-                var err = "";
-                var rsp = (xhr.responseText + '');
+                var err = "",
+                    rsp = (xhr.responseText + ''),
+                    ofs = rsp.lastIndexOf('\nURL: ');
+
+                if (ofs !== -1)
+                    rsp = rsp.slice(0, ofs);
+
+                if (rsp.indexOf('<pre>') === 0)
+                    rsp = rsp.slice(5);
+
+                st.bytes.uploaded += t.size;
                 if (rsp.indexOf('partial upload exists') !== -1 ||
                     rsp.indexOf('file already exists') !== -1) {
                     err = rsp;
-                    var ofs = err.lastIndexOf(' : ');
-                    if (ofs > 0)
-                        err = err.slice(0, ofs);
-
                     ofs = err.indexOf('\n/');
                     if (ofs !== -1) {
-                        err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2, -1)).join(' ');
+                        err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2)).join(' ');
                     }
                 }
                 if (err != "") {
@@ -895,7 +966,7 @@ function up2k_init(have_crypto) {
                     st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
                     t.postlist.splice(t.postlist.indexOf(npart), 1);
                     if (t.postlist.length == 0) {
-                        t.t3 = new Date().getTime();
+                        t.t4 = new Date().getTime();
                         ebi('f{0}t'.format(t.n)).innerHTML = 'verifying';
                         st.todo.handshake.unshift(t);
                     }
@@ -913,9 +984,14 @@ function up2k_init(have_crypto) {
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
             xhr.setRequestHeader('Content-Type', 'application/octet-stream');
-            xhr.overrideMimeType('Content-Type', 'application/octet-stream');
+            if (xhr.overrideMimeType)
+                xhr.overrideMimeType('Content-Type', 'application/octet-stream');
+
             xhr.responseType = 'text';
             xhr.send(e.target.result);
+
+            if (!t.t3)
+                t.t3 = new Date().getTime();
         };
 
         reader.readAsArrayBuffer(bobslice.call(t.fobj, car, cdr));

copyparty/web/upload.css

@@ -88,7 +88,7 @@
 	width: 30em;
 }
 #u2conf.has_btn {
-	width: 46em;
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;

copyparty/web/upload.html

@@ -24,7 +24,7 @@
         </form>
     </div>
 
-    <div id="op_msg" class="opview opbox">
+    <div id="op_msg" class="opview opbox act">
         <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
             <input type="text" name="msg" size="30">
             <input type="submit" value="send msg">
@@ -73,7 +73,8 @@
             <div id="u2btn_ct">
                 <div id="u2btn">
                     <span id="u2bm"></span><br />
-                    drop files here<br />
+                    drag/drop files<br />
+                    and folders here<br />
                     (or click me)
                 </div>
             </div>

copyparty/web/util.js

@@ -1,5 +1,11 @@
 "use strict";
 
+if (!window['console'])
+    window['console'] = {
+        "log": function (msg) { }
+    };
+
+
 // error handler for mobile devices
 function hcroak(msg) {
     document.body.innerHTML = msg;
@@ -113,6 +119,84 @@ function crc32(str) {
 };
 
 
+function clmod(obj, cls, add) {
+    var re = new RegExp('\\s*\\b' + cls + '\\s*\\b', 'g');
+    if (add == 't')
+        add = !re.test(obj.className);
+
+    obj.className = obj.className.replace(re, ' ') + (add ? ' ' + cls : '');
+}
+
+
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["lead", -1, ""], ["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v)
+                        v = uricom_dec(v)[0]
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map(function (x) { return x; });
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+    }
+    return nodes;
+}
+
+
 function sortTable(table, col, cb) {
     var tb = table.tBodies[0],
         th = table.tHead.rows[0].cells,
@@ -186,7 +270,6 @@ function makeSortable(table, cb) {
 }
 
 
-
 (function () {
     var ops = document.querySelectorAll('#ops>a');
     for (var a = 0; a < ops.length; a++) {
@@ -212,16 +295,16 @@ function opclick(e) {
 function goto(dest) {
     var obj = document.querySelectorAll('.opview.act');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     obj = document.querySelectorAll('#ops>a');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     if (dest) {
         var ui = ebi('op_' + dest);
-        ui.classList.add('act');
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+        clmod(ui, 'act', true);
+        document.querySelector('#ops>a[data-dest=' + dest + ']').className += " act";
 
         var fn = window['goto_' + dest];
         if (fn)
@@ -237,7 +320,10 @@ function goto(dest) {
     goto();
     var op = sread('opmode');
     if (op !== null && op !== '.')
-        goto(op);
+        try {
+            goto(op);
+        }
+        catch (ex) { }
 })();
 
 
@@ -405,8 +491,7 @@ function bcfg_upd_ui(name, val) {
     if (o.getAttribute('type') == 'checkbox')
         o.checked = val;
     else if (o) {
-        var fun = val ? 'add' : 'remove';
-        o.classList[fun]('on');
+        clmod(o, 'on', val);
     }
 }
 

docs/notes.sh

@@ -73,6 +73,13 @@ shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*10
 command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
 
 
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+
 ##
 ## sqlite3 stuff
 
@@ -83,6 +90,9 @@ sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2
 time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
 cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
 
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
 
 ##
 ## media
@@ -126,6 +136,16 @@ pip install virtualenv
 # readme toc
 cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
 
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
 
 ##
 ## http 206

scripts/make-sfx.sh

@@ -161,7 +161,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |

tests/test_vfs.py

@@ -18,7 +18,9 @@ from copyparty import util
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)