v0.10.9

.vscode/launch.json

@@ -14,6 +14,8 @@
                 "-emp",
                 "-e2dsa",
                 "-e2ts",
+                "-mtp",
+                ".bpm=f,bin/mtag/audio-bpm.py",
                 "-a",
                 "ed:wark",
                 "-v",

.vscode/launch.py

@@ -0,0 +1,35 @@
+# takes arguments from launch.json
+# is used by no_dbg in tasks.json
+# launches 10x faster than mspython debugpy
+# and is stoppable with ^C
+
+import os
+import sys
+import shlex
+
+sys.path.insert(0, os.getcwd())
+
+import jstyleson
+from copyparty.__main__ import main as copyparty
+
+with open(".vscode/launch.json", "r") as f:
+    tj = f.read()
+
+oj = jstyleson.loads(tj)
+argv = oj["configurations"][0]["args"]
+
+try:
+    sargv = " ".join([shlex.quote(x) for x in argv])
+    print(sys.executable + " -m copyparty " + sargv + "\n")
+except:
+    pass
+
+argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
+try:
+    copyparty(["a"] + argv)
+except SystemExit as ex:
+    if ex.code:
+        raise
+
+print("\n\033[32mokke\033[0m")
+sys.exit(1)

.vscode/tasks.json

@@ -9,9 +9,7 @@
         {
             "label": "no_dbg",
             "type": "shell",
-            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -e2ts -a ed:wark -v srv::r:aed:cnodupe -v dist:dist:r ;exit 1"
+            "command": "${config:python.pythonPath} .vscode/launch.py"
-            // -v ~/Music/mt:mt:r:cmtp=.bpm=~/dev/copyparty/bin/mtag/audio-bpm.py:cmtp=key=~/dev/copyparty/bin/mtag/audio-key.py:ce2tsr 
-            // -v ~/Music/mt:mt:r:cmtp=.bpm=~/dev/copyparty/bin/mtag/audio-bpm.py:ce2tsr
         }
     ]
 }

README.md

@@ -21,11 +21,13 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [status](#status)
 * [bugs](#bugs)
 * [usage](#usage)
+    * [zip downloads](#zip-downloads)
 * [searching](#searching)
     * [search configuration](#search-configuration)
     * [metadata from audio files](#metadata-from-audio-files)
     * [file parser plugins](#file-parser-plugins)
     * [complete examples](#complete-examples)
+* [browser support](#browser-support)
 * [client examples](#client-examples)
 * [dependencies](#dependencies)
     * [optional gpl stuff](#optional-gpl-stuff)
@@ -72,7 +74,7 @@ you may also want these, especially on servers:
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ folders as zip / tar files
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view
@@ -95,6 +97,8 @@ summary: it works! you can use it! (but technically not even close to beta)
 
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
 
@@ -108,6 +112,23 @@ the browser has the following hotkeys
 * `P` parent folder
 
 
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+
 # searching
 
 when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
@@ -176,6 +197,41 @@ copyparty can invoke external programs to collect additional metadata for files
   `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
 
 
+# browser support
+
+`ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
+
+| feature         | ie6 | ie9 | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+| --------------- | --- | --- | ---- | ---- | ----- | ---- | --- | ---- |
+| browse files    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| basic uploader  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| make directory  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| send message    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| set sort order  |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| zip selection   |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| directory tree  |  -  |  -  | `*1` | yep  | yep   | yep  | yep | yep  |
+| up2k            |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| icons work      |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown editor |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown viewer |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| play mp3/m4a    |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| play ogg/opus   |  -  |  -  |  -   |  -   | yep   | yep  | `*2` | yep |
+
+* internet explorer 6 to 8 behave the same
+* firefox 52 and chrome 49 are the last winxp versions
+* `*1` only public folders (login session is dropped) and no history / back-button
+* `*2` using a wasm decoder which can sometimes get stuck and consumes a bit more power
+
+quick summary of more eccentric web-browsers trying to view a directory index:
+* safari (14.0.3/macos) is chrome with janky wasm, so playing opus can deadlock the javascript engine
+* safari (14.0.1/iOS) same as macos, except it recovers from the deadlocks if you poke it a bit
+* links (2.21/macports) can browse, login, upload/mkdir/msg
+* lynx (2.8.9/macports) can browse, login, upload/mkdir/msg
+* w3m (0.5.3/macports) can browse, login, upload at 100kB/s, mkdir/msg
+* netsurf (3.10/arch) is basically ie6 with much better css (javascript has almost no effect)
+* netscape 4.0 and 4.5 can browse (text is yellow on white), upload with `?b=u`
+* SerenityOS (22d13d8) hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)
@@ -283,15 +339,21 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
+* audio link with timestamp
+* separate sqlite table per tag
+* audio fingerprinting
+* readme.md as epilogue
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* drop onto folders
 * `os.copy_file_range` for up2k cloning
-* up2k partials ui
 * support pillow-simd
-* cache sha512 chunks on client
-* comment field
-* ~~look into android thumbnail cache file format~~ bad idea
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||
+
+discarded ideas
+
+* up2k partials ui
+* cache sha512 chunks on client
+* comment field
+* look into android thumbnail cache file format

copyparty/__main__.py

@@ -12,7 +12,6 @@ import re
 import os
 import sys
 import time
-import signal
 import shutil
 import filecmp
 import locale
@@ -56,6 +55,12 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+class Dodge11874(RiceFormatter):
+    def __init__(self, *args, **kwargs):
+        kwargs["width"] = 9003
+        super(Dodge11874, self).__init__(*args, **kwargs)
+
+
 def warn(msg):
     print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
@@ -167,7 +172,7 @@ def configure_ssl_ciphers(al):
         sys.exit(0)
 
 
-def sighandler(signal=None, frame=None):
+def sighandler(sig=None, frame=None):
     msg = [""] * 5
     for th in threading.enumerate():
         msg.append(str(th))
@@ -177,34 +182,9 @@ def sighandler(signal=None, frame=None):
     print("\n".join(msg))
 
 
-def main():
+def run_argparse(argv, formatter):
-    time.strptime("19970815", "%Y%m%d")  # python#7980
-    if WINDOWS:
-        os.system("rem")  # enables colors
-
-    desc = py_desc().replace("[", "\033[1;30m[")
-
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
-
-    ensure_locale()
-    if HAVE_SSL:
-        ensure_cert()
-
-    deprecated = [["-e2s", "-e2ds"]]
-    for dk, nk in deprecated:
-        try:
-            idx = sys.argv.index(dk)
-        except:
-            continue
-
-        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
-        sys.argv[idx] = nk
-        time.sleep(2)
-
     ap = argparse.ArgumentParser(
-        formatter_class=RiceFormatter,
+        formatter_class=formatter,
         prog="copyparty",
         description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
         epilog=dedent(
@@ -216,6 +196,9 @@ def main():
             
             list of cflags:
               "cnodupe" rejects existing files (instead of symlinking them)
+              "ce2d" sets -e2d (all -e2* args can be set using ce2* cflags)
+              "cd2t" disables metadata collection, overrides -e2t*
+              "cd2d" disables all database stuff, overrides -e2*
 
             example:\033[35m
               -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
@@ -261,6 +244,7 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
     ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
@@ -289,9 +273,44 @@ def main():
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     
-    al = ap.parse_args()
+    return ap.parse_args(args=argv[1:])
     # fmt: on
 
+
+def main(argv=None):
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if WINDOWS:
+        os.system("rem")  # enables colors
+
+    if argv is None:
+        argv = sys.argv
+
+    desc = py_desc().replace("[", "\033[1;30m[")
+
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
+    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+
+    ensure_locale()
+    if HAVE_SSL:
+        ensure_cert()
+
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        argv[idx] = nk
+        time.sleep(2)
+
+    try:
+        al = run_argparse(argv, RiceFormatter)
+    except AssertionError:
+        al = run_argparse(argv, Dodge11874)
+
     # propagate implications
     for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 9, 13)
+VERSION = (0, 10, 9)
-CODENAME = "the strongest music server"
+CODENAME = "zip it"
-BUILD_DT = (2021, 3, 23)
+BUILD_DT = (2021, 4, 17)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import os
 import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
@@ -53,6 +54,7 @@ class VFS(object):
                 self.uwrite,
                 self.flags,
             )
+            self._trk(vn)
             self.nodes[name] = vn
             return self._trk(vn.add(src, dst))
 
@@ -109,7 +111,27 @@ class VFS(object):
         if rem:
             rp += "/" + rem
 
+        try:
             return fsdec(os.path.realpath(fsenc(rp)))
+        except:
+            if not WINDOWS:
+                raise
+
+            # cpython bug introduced in 3.8, still exists in 3.9.1;
+            # some win7sp1 and win10:20H2 boxes cannot realpath a
+            # networked drive letter such as b"n:" or b"n:\\"
+            #
+            # requirements to trigger:
+            #  * bytestring (not unicode str)
+            #  * just the drive letter (subfolders are ok)
+            #  * networked drive (regular disks and vmhgfs are ok)
+            #  * on an enterprise network (idk, cannot repro with samba)
+            #
+            # hits the following exceptions in succession:
+            #  * access denied at L601: "path = _getfinalpathname(path)"
+            #  * "cant concat str to bytes" at L621: "return path + tail"
+            #
+            return os.path.realpath(rp)
 
     def ls(self, rem, uname, scandir, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
@@ -127,6 +149,78 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+
+                rm = [x for x in rd if x[0] not in flt]
+                [rd.remove(x) for x in rm]
+
+                rm = [x for x in vd.keys() if x not in flt]
+                [vd.pop(x) for x in rm]
+
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)
@@ -159,12 +253,6 @@ class AuthSrv(object):
     def log(self, msg, c=0):
         self.log_func("auth", msg, c)
 
-    def invert(self, orig):
-        if PY2:
-            return {v: k for k, v in orig.iteritems()}
-        else:
-            return {v: k for k, v in orig.items()}
-
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
         it = iter(iterable)
@@ -421,7 +509,7 @@ class AuthSrv(object):
         with self.mutex:
             self.vfs = vfs
             self.user = user
-            self.iuser = self.invert(user)
+            self.iuser = {v: k for k, v in user.items()}
 
         # import pprint
         # pprint.pprint({"usr": user, "rd": mread, "wr": mwrite, "mnt": mount})

copyparty/broker_mp.py

@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()

copyparty/httpcli.py

@@ -7,6 +7,7 @@ import gzip
 import time
 import copy
 import json
+import string
 import socket
 import ctypes
 from datetime import datetime
@@ -14,6 +15,8 @@ import calendar
 
 from .__init__ import E, PY2, WINDOWS
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .szip import StreamZip
+from .star import StreamTar
 
 if not PY2:
     unicode = str
@@ -52,6 +55,10 @@ class HttpCli(object):
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def j2(self, name, **kwargs):
+        tpl = self.conn.hsrv.j2[name]
+        return tpl.render(**kwargs) if kwargs else tpl
+
     def run(self):
         """returns true if connection can be reused"""
         self.keepalive = False
@@ -67,7 +74,7 @@ class HttpCli(object):
                 headerlines.pop(0)
 
             try:
-                self.mode, self.req, _ = headerlines[0].split(" ")
+                self.mode, self.req, self.http_ver = headerlines[0].split(" ")
             except:
                 raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
 
@@ -86,30 +93,13 @@ class HttpCli(object):
             self.headers[k.lower()] = v.strip()
 
         v = self.headers.get("connection", "").lower()
-        self.keepalive = not v.startswith("close")
+        self.keepalive = not v.startswith("close") and self.http_ver != "HTTP/1.0"
 
         v = self.headers.get("x-forwarded-for", None)
         if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
             self.ip = v.split(",")[0]
             self.log_src = self.conn.set_rproxy(self.ip)
 
-        self.uname = "*"
-        if "cookie" in self.headers:
-            cookies = self.headers["cookie"].split(";")
-            for k, v in [x.split("=", 1) for x in cookies]:
-                if k.strip() != "cppwd":
-                    continue
-
-                v = unescape_cookie(v)
-                if v in self.auth.iuser:
-                    self.uname = self.auth.iuser[v]
-
-                break
-
-        if self.uname:
-            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
-            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
-
         # split req into vpath + uparam
         uparam = {}
         if "?" not in self.req:
@@ -133,6 +123,22 @@ class HttpCli(object):
         self.uparam = uparam
         self.vpath = unquotep(vpath)
 
+        pwd = None
+        if "cookie" in self.headers:
+            cookies = self.headers["cookie"].split(";")
+            for k, v in [x.split("=", 1) for x in cookies]:
+                if k.strip() != "cppwd":
+                    continue
+
+                pwd = unescape_cookie(v)
+                break
+
+        pwd = uparam.get("pw", pwd)
+        self.uname = self.auth.iuser.get(pwd, "*")
+        if self.uname:
+            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
+            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
+
         ua = self.headers.get("user-agent", "")
         if ua.startswith("rclone/"):
             uparam["raw"] = False
@@ -153,14 +159,18 @@ class HttpCli(object):
         except Pebkac as ex:
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
-                self.keepalive = self._check_nonfatal(ex)
+                if not self._check_nonfatal(ex):
-                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
+                    self.keepalive = False
+
+                self.log("{}\033[0m, {}".format(str(ex), self.vpath), 3)
+                msg = "<pre>{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                self.reply(msg.encode("utf-8", "replace"), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
 
     def send_headers(self, length, status=200, mime=None, headers={}):
-        response = ["HTTP/1.1 {} {}".format(status, HTTPCODE[status])]
+        response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
 
         if length is not None:
             response.append("Content-Length: " + unicode(length))
@@ -204,6 +214,20 @@ class HttpCli(object):
         self.log(body.rstrip())
         self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
+    def urlq(self, add={}, rm=[]):
+        """
+        generates url query based on uparam (b, pw, all others)
+        removing anything in rm, adding pairs in add
+        """
+
+        kv = {k: v for k, v in self.uparam.items() if k not in rm}
+        kv.update(add)
+        if not kv:
+            return ""
+
+        r = ["{}={}".format(k, quotep(v)) if v else k for k, v in kv.items()]
+        return "?" + "&amp;".join(r)
+
     def handle_get(self):
         logmsg = "{:4} {}".format(self.mode, self.req)
 
@@ -312,8 +336,19 @@ class HttpCli(object):
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
                 for buf in reader:
-                    buf = buf.decode("utf-8", "replace")
+                    orig = buf.decode("utf-8", "replace")
-                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
+                    m = "urlform_raw {} @ {}\n  {}\n"
+                    self.log(m.format(len(orig), self.vpath, orig))
+                    try:
+                        plain = unquote(buf.replace(b"+", b" "))
+                        plain = plain.decode("utf-8", "replace")
+                        if buf.startswith(b"msg="):
+                            plain = plain[4:]
+
+                        m = "urlform_dec {} @ {}\n  {}\n"
+                        self.log(m.format(len(plain), self.vpath, plain))
+                    except Exception as ex:
+                        self.log(repr(ex))
 
             if "get" in opt:
                 return self.handle_get()
@@ -388,8 +423,30 @@ class HttpCli(object):
         if act == "tput":
             return self.handle_text_upload()
 
+        if act == "zip":
+            return self.handle_zip_post()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
+    def handle_zip_post(self):
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                break
+
+        if v is None:
+            raise Pebkac(422, "need zip or tar keyword")
+
+        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        items = self.parser.require("files", 1024 * 1024)
+        if not items:
+            raise Pebkac(422, "need files list")
+
+        items = items.replace("\r", "").split("\n")
+        items = [unquotep(x) for x in items if items]
+
+        return self.tx_zip(k, v, vn, rem, items, self.args.ed)
+
     def handle_post_json(self):
         try:
             remains = int(self.headers["content-length"])
@@ -417,15 +474,18 @@ class HttpCli(object):
         if "srch" in self.uparam or "srch" in body:
             return self.handle_search(body)
 
-        # prefer this over undot; no reason to allow traversion
-        if "/" in body["name"]:
-            raise Pebkac(400, "folders verboten")
-
         # up2k-php compat
         for k in "chunkpit.php", "handshake.php":
             if self.vpath.endswith(k):
                 self.vpath = self.vpath[: -len(k)]
 
+        sub = None
+        name = undot(body["name"])
+        if "/" in name:
+            sub, name = name.rsplit("/", 1)
+            self.vpath = "/".join([self.vpath, sub]).strip("/")
+            body["name"] = name
+
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
 
         body["vtop"] = vfs.vpath
@@ -434,12 +494,22 @@ class HttpCli(object):
         body["addr"] = self.ip
         body["vcfg"] = vfs.flags
 
-        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
+        if sub:
-        response = x.get()
+            try:
-        response = json.dumps(response)
+                dst = os.path.join(vfs.realpath, rem)
+                os.makedirs(dst)
+            except:
+                if not os.path.isdir(dst):
+                    raise Pebkac(400, "some file got your folder name")
 
-        self.log(response)
+        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
-        self.reply(response.encode("utf-8"), mime="application/json")
+        ret = x.get()
+        if sub:
+            ret["name"] = "/".join([sub, ret["name"]])
+
+        ret = json.dumps(ret)
+        self.log(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
         return True
 
     def handle_search(self, body):
@@ -580,7 +650,7 @@ class HttpCli(object):
             pwd = "x"  # nosec
 
         h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
-        html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
+        html = self.j2("msg", h1=msg, h2='<a href="/">ack</a>', redir="/")
         self.reply(html.encode("utf-8"), headers=h)
         return True
 
@@ -611,7 +681,8 @@ class HttpCli(object):
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         esc_paths = [quotep(vpath), html_escape(vpath)]
-        html = self.conn.tpl_msg.render(
+        html = self.j2(
+            "msg",
             h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
             pre="aight",
             click=True,
@@ -643,7 +714,8 @@ class HttpCli(object):
                 f.write(b"`GRUNNUR`\n")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        html = self.conn.tpl_msg.render(
+        html = self.j2(
+            "msg",
             h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
                 quotep(vpath), html_escape(vpath)
             ),
@@ -749,7 +821,8 @@ class HttpCli(object):
                     ).encode("utf-8")
                 )
 
-        html = self.conn.tpl_msg.render(
+        html = self.j2(
+            "msg",
             h2='<a href="/{}">return to /{}</a>'.format(
                 quotep(self.vpath), html_escape(self.vpath)
             ),
@@ -1037,16 +1110,75 @@ class HttpCli(object):
         self.log("{},  {}".format(logmsg, spd))
         return ret
 
+    def tx_zip(self, fmt, uarg, vn, rem, items, dots):
+        if self.args.no_zip:
+            raise Pebkac(400, "not enabled")
+
+        logmsg = "{:4} {} ".format("", self.req)
+        self.keepalive = False
+
+        if not uarg:
+            uarg = ""
+
+        if fmt == "tar":
+            mime = "application/x-tar"
+            packer = StreamTar
+        else:
+            mime = "application/zip"
+            packer = StreamZip
+
+        fn = items[0] if items and items[0] else self.vpath
+        if fn:
+            fn = fn.rstrip("/").split("/")[-1]
+        else:
+            fn = self.headers.get("host", "hey")
+
+        afn = "".join(
+            [x if x in (string.ascii_letters + string.digits) else "_" for x in fn]
+        )
+
+        bascii = unicode(string.ascii_letters + string.digits).encode("utf-8")
+        ufn = fn.encode("utf-8", "xmlcharrefreplace")
+        if PY2:
+            ufn = [unicode(x) if x in bascii else "%{:02x}".format(ord(x)) for x in ufn]
+        else:
+            ufn = [
+                chr(x).encode("utf-8")
+                if x in bascii
+                else "%{:02x}".format(x).encode("ascii")
+                for x in ufn
+            ]
+        ufn = b"".join(ufn).decode("ascii")
+
+        cdis = "attachment; filename=\"{}.{}\"; filename*=UTF-8''{}.{}"
+        cdis = cdis.format(afn, fmt, ufn, fmt)
+        self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis})
+
+        fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
+        # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
+        bsent = 0
+        for buf in bgen.gen():
+            if not buf:
+                break
+
+            try:
+                self.s.sendall(buf)
+                bsent += len(buf)
+            except:
+                logmsg += " \033[31m" + unicode(bsent) + "\033[0m"
+                break
+
+        spd = self._spd(bsent)
+        self.log("{},  {}".format(logmsg, spd))
+        return True
+
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
-        if "edit2" in self.uparam:
-            html_path = "web/mde.html"
-            template = self.conn.tpl_mde
-        else:
-            html_path = "web/md.html"
-            template = self.conn.tpl_md
 
-        html_path = os.path.join(E.mod, html_path)
+        tpl = "mde" if "edit2" in self.uparam else "md"
+        html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
+        template = self.j2(tpl)
 
         st = os.stat(fsenc(fs_path))
         # sz_md = st.st_size
@@ -1096,9 +1228,10 @@ class HttpCli(object):
         return True
 
     def tx_mounts(self):
+        suf = self.urlq(rm=["h"])
         rvol = [x + "/" if x else x for x in self.rvol]
         wvol = [x + "/" if x else x for x in self.wvol]
-        html = self.conn.tpl_mounts.render(this=self, rvol=rvol, wvol=wvol)
+        html = self.j2("splash", this=self, rvol=rvol, wvol=wvol, url_suf=suf)
         self.reply(html.encode("utf-8"))
         return True
 
@@ -1187,6 +1320,11 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                return self.tx_zip(k, v, vn, rem, [], self.args.ed)
+
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
@@ -1223,6 +1361,8 @@ class HttpCli(object):
             idx = self.conn.get_u2idx()
             icur = idx.get_cur(vn.realpath)
 
+        url_suf = self.urlq()
+
         dirs = []
         files = []
         for fn in vfs_ls:
@@ -1247,8 +1387,11 @@ class HttpCli(object):
 
             is_dir = stat.S_ISDIR(inf.st_mode)
             if is_dir:
-                margin = "DIR"
                 href += "/"
+                if self.args.no_zip:
+                    margin = "DIR"
+                else:
+                    margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
                     base, html_escape(hist[fn][2], quote=True), hist[fn][0]
@@ -1372,7 +1515,12 @@ class HttpCli(object):
 
         dirs.extend(files)
 
-        html = self.conn.tpl_browser.render(
+        tpl = "browser"
+        if "b" in self.uparam:
+            tpl = "browser2"
+
+        html = self.j2(
+            tpl,
             vdir=quotep(self.vpath),
             vpnodes=vpnodes,
             files=dirs,
@@ -1384,6 +1532,9 @@ class HttpCli(object):
             ),
             have_up2k_idx=("e2d" in vn.flags),
             have_tags_idx=("e2t" in vn.flags),
+            have_zip=(not self.args.no_zip),
+            have_b_u=(self.writable and self.uparam.get("b") == "u"),
+            url_suf=url_suf,
             logues=logues,
             title=html_escape(self.vpath),
             srv_info=srv_info,

copyparty/httpconn.py

@@ -12,23 +12,6 @@ try:
 except:
     HAVE_SSL = False
 
-try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * {} -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-""".format(
-            os.path.basename(sys.executable)
-        )
-    )
-    sys.exit(1)
-
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
@@ -57,14 +40,6 @@ class HttpConn(object):
         self.log_func = hsrv.log
         self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
-
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -112,7 +87,9 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
+                if method:
                     self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 

copyparty/httpsrv.py

@@ -2,10 +2,28 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
 from .__init__ import E, MACOS
 from .httpconn import HttpConn
 from .authsrv import AuthSrv
@@ -30,6 +48,13 @@ class HttpSrv(object):
         self.workload_thr_alive = False
         self.auth = AuthSrv(self.args, self.log)
 
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+        }
+
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
             self.cert_path = cert_path

copyparty/star.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen)
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/sutil.py

@@ -0,0 +1,25 @@
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/szip.py

@@ -0,0 +1,271 @@
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, "/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/up2k.py

@@ -232,6 +232,7 @@ class Up2k(object):
                 (ft if v is True else ff if v is False else fv).format(k, str(v))
                 for k, v in flags.items()
             ]
+            if a:
                 self.log(" ".join(sorted(a)) + "\033[0m")
 
             reg = {}
@@ -1067,6 +1068,8 @@ class Up2k(object):
         with self.mutex:
             job = self.registry[ptop].get(wark, None)
             if not job:
+                known = " ".join([x for x in self.registry[ptop].keys()])
+                self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
@@ -1309,6 +1312,7 @@ class Up2k(object):
                     self.log("no cursor to write tags with??", c=1)
                     continue
 
+                # TODO is undef if vol 404 on startup
                 entags = self.entags[ptop]
                 if not entags:
                     self.log("no entags okay.jpg", c=3)

copyparty/util.py

@@ -576,11 +576,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
+def sanitize_fn(fn, ok=""):
+    if "/" not in ok:
         fn = fn.replace("\\", "/").split("/")[-1]
 
     if WINDOWS:
-        for bad, good in [
+        for bad, good in [x for x in [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -590,7 +591,7 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
+        ] if x[0] not in ok]:
             fn = fn.replace(bad, good)
 
         bad = ["con", "prn", "aux", "nul"]
@@ -780,6 +781,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()

copyparty/web/browser.css

@@ -182,6 +182,11 @@ a, #files tbody div a:last-child {
 	color: #840;
 	text-shadow: 0 0 .3em #b80;
 }
+#files tbody tr.sel td {
+	color: #fff;
+	background: #925;
+	border-color: #c37;
+}
 #blocked {
 	position: fixed;
 	top: 0;
@@ -238,7 +243,7 @@ a, #files tbody div a:last-child {
 	height: 100%;
 	background: #3c3c3c;
 }
-#wtoggle {
+#wtico {
 	cursor: url(/.cpr/dd/1.png), pointer;
 	animation: cursor 500ms infinite;
 }
@@ -268,6 +273,33 @@ a, #files tbody div a:last-child {
 	padding: .2em 0 0 .07em;
 	color: #fff;
 }
+#wzip {
+	display: none;
+	margin-right: .3em;
+	padding-right: .3em;
+	border-right: .1em solid #555;
+}
+#wtoggle,
+#wtoggle * {
+	line-height: 1em;
+}
+#wtoggle.sel {
+	width: 6em;
+}
+#wtoggle.sel #wzip {
+	display: inline-block;
+}
+#wtoggle.sel #wzip a {
+	font-size: .4em;
+	padding: 0 .3em;
+	margin: -.3em .2em;
+	position: relative;
+	display: inline-block;
+}
+#wtoggle.sel #wzip #selzip {
+	top: -.6em;
+	padding: .4em .3em;
+}
 #barpos,
 #barbuf {
 	position: absolute;
@@ -463,7 +495,7 @@ input[type="checkbox"]:checked+label {
 }
 #tree {
 	display: none;
-	position: fixed;
+	position: absolute;
 	left: 0;
 	bottom: 0;
 	top: 7em;
@@ -598,7 +630,8 @@ input[type="checkbox"]:checked+label {
 #files td.min a {
 	display: none;
 }
-#files tr.play td {
+#files tr.play td,
+#files tr.play div a {
 	background: #fc4;
 	border-color: transparent;
 	color: #400;

copyparty/web/browser.html

@@ -13,8 +13,8 @@
 <body>
     <div id="ops">
         <a href="#" data-dest="" data-desc="close submenu">---</a>
-        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
         {%- if have_up2k_idx %}
+        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
         <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
         {%- else %}
         <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
@@ -41,10 +41,12 @@
     <div id="op_cfg" class="opview opbox">
         <h3>key notation</h3>
         <div id="key_notation"></div>
+        {%- if have_zip %}
+        <h3>folder download</h3>
+        <div id="arc_fmt"></div>
+        {%- endif %}
         <h3>tooltips</h3>
-        <div>
+        <div><a id="tooltips" class="tglbtn" href="#">enable</a></div>
-            <a id="tooltips" class="tglbtn" href="#">enable</a>
-        </div>
     </div>
     
     <h1 id="path">
@@ -70,7 +72,7 @@
     <table id="files">
         <thead>
             <tr>
-                <th></th>
+                <th name="lead"><span>c</span></th>
                 <th name="href"><span>File Name</span></th>
                 <th name="sz" sort="int"><span>Size</span></th>
                 {%- for k in taglist %}
@@ -110,7 +112,14 @@
     {%- endif %}
 
     <div id="widget">
-        <div id="wtoggle">♫</div>
+        <div id="wtoggle">
+            <span id="wzip">
+                <a href="#" id="selall">sel.<br />all</a>
+                <a href="#" id="selinv">sel.<br />inv.</a>
+                <a href="#" id="selzip">zip</a>
+            </span><a
+                href="#" id="wtico">♫</a>
+        </div>
         <div id="widgeti">
             <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
             <canvas id="pvol" width="288" height="38"></canvas>

copyparty/web/browser.js

@@ -75,7 +75,7 @@ makeSortable(ebi('files'), mp.read_order.bind(mp));
 var widget = (function () {
 	var ret = {};
 	var widget = ebi('widget');
-	var wtoggle = ebi('wtoggle');
+	var wtico = ebi('wtico');
 	var touchmode = false;
 	var side_open = false;
 	var was_paused = true;
@@ -113,14 +113,7 @@ var widget = (function () {
 
 		return false;
 	};
-	if (window.Touch) {
+	wtico.onclick = click_handler;
-		var touch_handler = function (e) {
-			touchmode = true;
-			return ret.toggle(e);
-		};
-		wtoggle.addEventListener('touchstart', touch_handler, false);
-	}
-	wtoggle.onclick = click_handler;
 	return ret;
 })();
 
@@ -324,7 +317,7 @@ function seek_au_sec(seek) {
 	if (mp.au === mp.au_native)
 		// hack: ogv.js breaks on .play() during playback
 		mp.au.play();
-};
+}
 
 
 function song_skip(n) {
@@ -336,7 +329,7 @@ function song_skip(n) {
 		play(mp.order.indexOf(tid) + n);
 	else
 		play(mp.order[0]);
-};
+}
 
 
 // hook up the widget buttons
@@ -500,7 +493,7 @@ function play(tid, call_depth) {
 	setclass(oid, 'play act');
 	var trs = ebi('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr');
 	for (var a = 0, aa = trs.length; a < aa; a++) {
-		trs[a].className = trs[a].className.replace(/ *play */, "");
+		clmod(trs[a], 'play');
 	}
 	ebi(oid).parentElement.parentElement.className += ' play';
 
@@ -625,13 +618,16 @@ function tree_neigh(n) {
 			break;
 		}
 	}
-	a += n;
+	if (act == -1)
-	if (a < 0)
+		return;
-		a = links.length - 1;
-	if (a >= links.length)
-		a = 0;
 
-	links[a].click();
+	act += n;
+	if (act < 0)
+		act = links.length - 1;
+	if (act >= links.length)
+		act = 0;
+
+	links[act].click();
 }
 
 
@@ -649,10 +645,13 @@ function tree_up() {
 
 
 document.onkeydown = function (e) {
-	if (document.activeElement != document.body && document.activeElement.nodeName.toLowerCase() != 'a')
+	if (!document.activeElement || document.activeElement != document.body && document.activeElement.nodeName.toLowerCase() != 'a')
 		return;
 
-	var k = e.code, pos = -1;
+	if (e.ctrlKey || e.altKey || e.shiftKey || e.metaKey || e.isComposing)
+		return;
+
+	var k = (e.code + ''), pos = -1;
 	if (k.indexOf('Digit') === 0)
 		pos = parseInt(k.slice(-1)) * 0.1;
 
@@ -753,7 +752,7 @@ document.onkeydown = function (e) {
 		clearTimeout(search_timeout);
 		var now = new Date().getTime();
 		if (now - search_in_progress > 30 * 1000)
-			search_timeout = setTimeout(do_search, 100);
+			search_timeout = setTimeout(do_search, 200);
 	}
 
 	function do_search() {
@@ -772,6 +771,7 @@ document.onkeydown = function (e) {
 		// ebi('srch_q').textContent = JSON.stringify(params, null, 4);
 		var xhr = new XMLHttpRequest();
 		xhr.open('POST', '/?srch', true);
+		xhr.setRequestHeader('Content-Type', 'text/plain');
 		xhr.onreadystatechange = xhr_search_results;
 		xhr.ts = new Date().getTime();
 		xhr.send(JSON.stringify(params));
@@ -796,6 +796,8 @@ document.onkeydown = function (e) {
 		var res = JSON.parse(this.responseText),
 			tagord = res.tag_order;
 
+		sortfiles(res.hits);
+
 		var ofiles = ebi('files');
 		if (ofiles.getAttribute('ts') > this.ts)
 			return;
@@ -814,7 +816,7 @@ document.onkeydown = function (e) {
 
 		var html = mk_files_header(tagord);
 		html.push('<tbody>');
-		html.push('<tr><td>-</td><td colspan="42"><a href="#" id="unsearch">close search results</a></td></tr>');
+		html.push('<tr><td>-</td><td colspan="42"><a href="#" id="unsearch">! close search results</a></td></tr>');
 		for (var a = 0; a < res.hits.length; a++) {
 			var r = res.hits[a],
 				ts = parseInt(r.ts),
@@ -833,7 +835,7 @@ document.onkeydown = function (e) {
 					v = r.tags[k] || "";
 
 				if (k == ".dur") {
-					var sv = s2ms(v);
+					var sv = v ? s2ms(v) : "";
 					nodes[nodes.length - 1] += '</td><td sortv="' + v + '">' + sv;
 					continue;
 				}
@@ -867,6 +869,7 @@ document.onkeydown = function (e) {
 		oldcfg = [];
 		ebi('files').innerHTML = orig_html;
 		orig_html = null;
+		msel.render();
 		reload_browser();
 	}
 })();
@@ -875,12 +878,16 @@ document.onkeydown = function (e) {
 var treectl = (function () {
 	var treectl = {
 		"hidden": false
-	};
+	},
-	var dyn = bcfg_get('dyntree', true);
+		entreed = false,
-	var treesz = icfg_get('treesz', 16);
+		fixedpos = false,
+		prev_atop = null,
+		prev_winh = null,
+		dyn = bcfg_get('dyntree', true),
+		treesz = icfg_get('treesz', 16);
+
 	treesz = Math.min(Math.max(treesz, 4), 50);
 	console.log('treesz [' + treesz + ']');
-	var entreed = false;
 
 	function entree(e) {
 		ev(e);
@@ -912,13 +919,43 @@ var treectl = (function () {
 		if (!entreed || treectl.hidden)
 			return;
 
-		var top = ebi('wrap').getBoundingClientRect().top;
+		var tree = ebi('tree'),
-		ebi('tree').style.top = Math.max(0, parseInt(top)) + 'px';
+			wrap = ebi('wrap'),
+			atop = wrap.getBoundingClientRect().top,
+			winh = window.innerHeight;
+
+		if (atop === prev_atop && winh === prev_winh)
+			return;
+
+		prev_atop = atop;
+		prev_winh = winh;
+
+		if (fixedpos && atop >= 0) {
+			tree.style.position = 'absolute';
+			tree.style.bottom = '';
+			fixedpos = false;
+		}
+		else if (!fixedpos && atop < 0) {
+			tree.style.position = 'fixed';
+			tree.style.height = 'auto';
+			fixedpos = true;
+		}
+
+		if (fixedpos) {
+			tree.style.top = Math.max(0, parseInt(atop)) + 'px';
+		}
+		else {
+			var top = Math.max(0, parseInt(wrap.offsetTop)),
+				treeh = (winh - atop) - 4;
+
+			tree.style.top = top + 'px';
+			tree.style.height = treeh < 10 ? '' : treeh + 'px';
+		}
 	}
 
 	function periodic() {
 		onscroll();
-		setTimeout(periodic, document.visibilityState ? 200 : 5000);
+		setTimeout(periodic, document.visibilityState ? 100 : 5000);
 	}
 	periodic();
 
@@ -995,8 +1032,6 @@ var treectl = (function () {
 					var o = links[a].parentNode;
 					if (!o.getElementsByTagName('li').length)
 						o.innerHTML = html;
-					//else
-					//	links[a].previousSibling.textContent = '-';
 				}
 			}
 		}
@@ -1085,32 +1120,8 @@ var treectl = (function () {
 		}
 
 		ebi('srv_info').innerHTML = '<span>' + res.srvinf + '</span>';
-		var nodes = res.dirs.concat(res.files),
+		var nodes = res.dirs.concat(res.files);
-			sopts = jread('fsort', []);
+		nodes = sortfiles(nodes);
-
-		try {
-			for (var a = sopts.length - 1; a >= 0; a--) {
-				var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
-				if (name.indexOf('tags/') == -1) {
-					nodes.sort(function (v1, v2) {
-						if (!v1[name]) return -1 * rev;
-						if (!v2[name]) return 1 * rev;
-						return rev * (typ == 'int' ? (v1[name] - v2[name]) : (v1[name].localeCompare(v2[name])));
-					});
-				}
-				else {
-					name = name.slice(5);
-					nodes.sort(function (v1, v2) {
-						if (!v1.tags[name]) return -1 * rev;
-						if (!v2.tags[name]) return 1 * rev;
-						return rev * (typ == 'int' ? (v1.tags[name] - v2.tags[name]) : (v1.tags[name].localeCompare(v2.tags[name])));
-					});
-				}
-			}
-		}
-		catch (ex) {
-			console.log("failed to apply sort config: " + ex);
-		}
 
 		var top = this.top;
 		var html = mk_files_header(res.taglist);
@@ -1125,7 +1136,7 @@ var treectl = (function () {
 					v = (r.tags || {})[k] || "";
 
 				if (k == ".dur") {
-					var sv = s2ms(v);
+					var sv = v ? s2ms(v) : "";
 					ln[ln.length - 1] += '</td><td sortv="' + v + '">' + sv;
 					continue;
 				}
@@ -1149,6 +1160,7 @@ var treectl = (function () {
 
 		filecols.set_style();
 		mukey.render();
+		msel.render();
 		reload_tree();
 		reload_browser();
 	}
@@ -1303,8 +1315,8 @@ function find_file_col(txt) {
 
 function mk_files_header(taglist) {
 	var html = [
-		'<thead>',
+		'<thead><tr>',
-		'<th></th>',
+		'<th name="lead"><span>c</span></th>',
 		'<th name="href"><span>File Name</span></th>',
 		'<th name="sz" sort="int"><span>Size</span></th>'
 	];
@@ -1322,7 +1334,7 @@ function mk_files_header(taglist) {
 	html = html.concat([
 		'<th name="ext"><span>T</span></th>',
 		'<th name="ts"><span>Date</span></th>',
-		'</thead>',
+		'</tr></thead>',
 	]);
 	return html;
 }
@@ -1356,13 +1368,13 @@ var filecols = (function () {
 				continue;
 
 			var name = span[0].textContent,
-				cls = '';
+				cls = false;
 
 			if (has(hidden, name)) {
 				ohidden.push(a);
-				cls = ' min';
+				cls = true;
 			}
-			ths[a].className = ths[a].className.replace(/ *min */, " ") + cls;
+			clmod(ths[a], 'min', cls)
 		}
 		for (var a = 0; a < ncols; a++) {
 			var cls = has(ohidden, a) ? 'min' : '';
@@ -1407,8 +1419,8 @@ var filecols = (function () {
 		if (!min)
 			for (var a = 0, aa = rows.length; a < aa; a++) {
 				var c = rows[a].cells[i];
-				if (c)
+				if (c && c.textContent)
-					var v = c.textContent = s2ms(c.textContent);
+					c.textContent = s2ms(c.textContent);
 			}
 	}
 	catch (ex) { }
@@ -1479,8 +1491,11 @@ var mukey = (function () {
 	}
 
 	function render() {
-		var ci = find_file_col('Key'),
+		var ci = find_file_col('Key');
-			i = ci[0],
+		if (!ci)
+			return;
+
+		var i = ci[0],
 			min = ci[1],
 			rows = ebi('files').tBodies[0].rows;
 
@@ -1529,10 +1544,13 @@ var mukey = (function () {
 
 
 function addcrc() {
-	var links = document.querySelectorAll('#files>tbody>tr>td:nth-child(2)>a');
+	var links = document.querySelectorAll(
+		'#files>tbody>tr>td:first-child+td>' + (
+			ebi('unsearch') ? 'div>a:last-child' : 'a'));
+
 	for (var a = 0, aa = links.length; a < aa; a++)
 		if (!links[a].getAttribute('id'))
-			links[a].setAttribute('id', 'f-' + crc32(links[a].textContent));
+			links[a].setAttribute('id', 'f-' + crc32(links[a].textContent || links[a].innerText));
 }
 
 
@@ -1559,6 +1577,140 @@ function addcrc() {
 })();
 
 
+var arcfmt = (function () {
+	if (!ebi('arc_fmt'))
+		return { "render": function () { } };
+
+	var html = [],
+		arcfmts = ["tar", "zip", "zip_dos", "zip_crc"],
+		arcv = ["tar", "zip=utf8", "zip", "zip=crc"];
+
+	for (var a = 0; a < arcfmts.length; a++) {
+		var k = arcfmts[a];
+		html.push(
+			'<span><input type="radio" name="arcfmt" value="' + k + '" id="arcfmt_' + k + '">' +
+			'<label for="arcfmt_' + k + '">' + k + '</label></span>');
+	}
+	ebi('arc_fmt').innerHTML = html.join('\n');
+
+	var fmt = sread("arc_fmt") || "zip";
+	ebi('arcfmt_' + fmt).checked = true;
+
+	function render() {
+		var arg = arcv[arcfmts.indexOf(fmt)],
+			tds = document.querySelectorAll('#files tbody td:first-child a');
+
+		for (var a = 0, aa = tds.length; a < aa; a++) {
+			var o = tds[a], txt = o.textContent, href = o.getAttribute('href');
+			if (txt != 'tar' && txt != 'zip')
+				continue;
+
+			var ofs = href.lastIndexOf('?');
+			if (ofs < 0)
+				throw 'missing arg in url';
+
+			o.setAttribute("href", href.slice(0, ofs + 1) + arg);
+			o.textContent = fmt.split('_')[0];
+		}
+		ebi('selzip').textContent = fmt.split('_')[0];
+		ebi('selzip').setAttribute('fmt', arg);
+	}
+
+	function try_render() {
+		try {
+			render();
+		}
+		catch (ex) {
+			console.log("arcfmt failed: " + ex);
+		}
+	}
+
+	function change_fmt(e) {
+		ev(e);
+		fmt = this.getAttribute('value');
+		swrite("arc_fmt", fmt);
+		try_render();
+	}
+
+	var o = document.querySelectorAll('#arc_fmt input');
+	for (var a = 0; a < o.length; a++) {
+		o[a].onchange = change_fmt;
+	}
+
+	return {
+		"render": try_render
+	};
+})();
+
+
+var msel = (function () {
+	function getsel() {
+		var names = [];
+		var links = document.querySelectorAll('#files tbody tr.sel td:nth-child(2) a');
+		for (var a = 0, aa = links.length; a < aa; a++)
+			names.push(links[a].getAttribute('href').replace(/\/$/, "").split('/').slice(-1));
+
+		return names;
+	}
+	function selui() {
+		clmod(ebi('wtoggle'), 'sel', getsel().length);
+	}
+	function seltgl(e) {
+		ev(e);
+		var tr = this.parentNode;
+		clmod(tr, 'sel', 't');
+		selui();
+	}
+	function evsel(e, fun) {
+		ev(e);
+		var trs = document.querySelectorAll('#files tbody tr');
+		for (var a = 0, aa = trs.length; a < aa; a++)
+			clmod(trs[a], 'sel', fun);
+		selui();
+	}
+	ebi('selall').onclick = function (e) {
+		evsel(e, "add");
+	};
+	ebi('selinv').onclick = function (e) {
+		evsel(e, "t");
+	};
+	ebi('selzip').onclick = function (e) {
+		ev(e);
+		var names = getsel();
+		var arg = ebi('selzip').getAttribute('fmt');
+		var txt = names.join('\n');
+		var frm = document.createElement('form');
+		frm.setAttribute('action', '?' + arg);
+		frm.setAttribute('method', 'post');
+		frm.setAttribute('target', '_blank');
+		frm.setAttribute('enctype', 'multipart/form-data');
+		frm.innerHTML = '<input name="act" value="zip" />' +
+			'<textarea name="files" id="ziptxt"></textarea>';
+		frm.style.display = 'none';
+
+		var oldform = document.querySelector('#widgeti>form');
+		if (oldform)
+			oldform.parentNode.removeChild(oldform);
+
+		ebi('widgeti').appendChild(frm);
+		var obj = ebi('ziptxt');
+		obj.value = txt;
+		console.log(txt);
+		frm.submit();
+	};
+	function render() {
+		var tds = document.querySelectorAll('#files tbody td+td+td');
+		for (var a = 0, aa = tds.length; a < aa; a++) {
+			tds[a].onclick = seltgl;
+		}
+		arcfmt.render();
+	}
+	return {
+		"render": render
+	};
+})();
+
+
 function ev_row_tgl(e) {
 	ev(e);
 	filecols.toggle(this.parentElement.parentElement.getElementsByTagName('span')[0].textContent);
@@ -1611,3 +1763,4 @@ function reload_browser(not_mp) {
 }
 reload_browser(true);
 mukey.render();
+msel.render();

copyparty/web/browser2.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>{{ title }}</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+</head>
+
+<body>
+    {%- if srv_info %}
+    <p><span>{{ srv_info }}</span></p>
+    {%- endif %}
+
+    {%- if have_b_u %}
+    <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+        <input type="hidden" name="act" value="bput" />
+        <input type="file" name="f" multiple /><br />
+        <input type="submit" value="start upload" />
+    </form>
+    <br />
+    {%- endif %}
+
+    {%- if logues[0] %}
+    <div>{{ logues[0] }}</div><br />
+    {%- endif %}
+
+    <table id="files">
+        <thead>
+            <tr>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
+                <th name="ts"><span>Date</span></th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+
+{%- for f in files %}
+    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+{%- endfor %}
+
+        </tbody>
+    </table>
+    
+    {%- if logues[1] %}
+    <div>{{ logues[1] }}</div><br />
+    {%- endif %}
+    
+    <h2><a href="{{ url_suf }}&amp;h">control-panel</a></h2>
+
+</body>
+</html>

copyparty/web/md.html

@@ -147,7 +147,7 @@ var md_opt = {
 
 	</script>
     <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.full.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}
 	<script src="/.cpr/md2.js"></script>

copyparty/web/splash.html

@@ -16,20 +16,20 @@
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
 
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/">
+            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />

copyparty/web/up2k.js

@@ -31,6 +31,7 @@ catch (ex) {
     }
     catch (ex) { }
 }
+treectl.onscroll();
 
 
 function up2k_flagbus() {
@@ -131,6 +132,280 @@ function up2k_flagbus() {
     return flag;
 }
 
+
+function U2pvis(act, btns) {
+    this.act = act;
+    this.ctr = { "ok": 0, "ng": 0, "bz": 0, "q": 0 };
+    this.tab = [];
+    this.head = 0;
+    this.tail = -1;
+    this.wsz = 3;
+
+    this.addfile = function (entry) {
+        this.tab.push({
+            "hn": entry[0],
+            "ht": entry[1],
+            "hp": entry[2],
+            "in": 'q',
+            "nh": 0, //hashed
+            "nd": 0, //done
+            "pa": [], //percents
+            "pb": []  //active-list
+        });
+        this.ctr["q"]++;
+        this.drawcard("q");
+        if (this.act == "q") {
+            this.addrow(this.tab.length - 1);
+        }
+        if (this.act == "bz") {
+            this.bzw();
+        }
+    };
+
+    this.is_act = function (card) {
+        if (this.act == "done")
+            return card == "ok" || card == "ng";
+
+        return this.act == card;
+    }
+
+    this.seth = function (nfile, field, html) {
+        var fo = this.tab[nfile];
+        field = ['hn', 'ht', 'hp'][field];
+        fo[field] = html;
+        if (!this.is_act(fo.in))
+            return;
+
+        var obj = ebi('f{0}{1}'.format(nfile, field.slice(1)));
+        obj.innerHTML = html;
+        if (field == 'hp') {
+            obj.style.color = '';
+            obj.style.background = '';
+        }
+    };
+
+    this.setab = function (nfile, blocks) {
+        var t = [];
+        for (var a = 0; a < blocks; a++)
+            t.push(0);
+
+        this.tab[nfile].pa = t;
+    };
+
+    this.perc = function (n, t, e, sz, t0) {
+        var p = (n + e) * 100.0 / t,
+            td = new Date().getTime() - t0,
+            pp = (td / 1000) / p,
+            spd = (sz / 100) / pp,
+            eta = pp * (100 - p);
+
+        return [p, s2ms(eta), spd / (1024 * 1024)];
+    };
+
+    this.hashed = function (fobj) {
+        var fo = this.tab[fobj.n];
+        fo.nh++;
+        var p = this.perc(fo.nh, fo.pa.length, 0, fobj.size, fobj.t1);
+        fo.hp = '{0}%, {1}, {2} MB/s'.format(
+            p[0].toFixed(2), p[1], p[2].toFixed(2)
+        );
+        if (!this.is_act(fo.in))
+            return;
+
+        var obj = ebi('f{0}p'.format(fobj.n));
+        obj.innerHTML = fo.hp;
+        obj.style.color = '#fff';
+        var o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
+        obj.style.background = 'linear-gradient(90deg, #025, #06a ' + o1 + '%, #08d ' + o2 + '%, #333 ' + o3 + '%)';
+    };
+
+    this.prog = function (fobj, nchunk, percent) {
+        var fo = this.tab[fobj.n], pb = fo.pb;
+        var i = pb.indexOf(nchunk);
+        fo.pa[nchunk] = percent;
+        if (percent == 101) {
+            fo.nd++;
+            if (i >= 0)
+                pb.splice(i);
+        }
+        else if (i == -1) {
+            pb.push(nchunk);
+        }
+
+        var extra = 0;
+        for (var a = 0; a < pb.length; a++)
+            extra += fo.pa[a];
+
+        extra /= fo.pa.length;
+
+        var p = this.perc(fo.nd, fo.pa.length, extra, fobj.size, fobj.t3);
+        fo.hp = '{0}%, {1}, {2} MB/s'.format(
+            p[0].toFixed(2), p[1], p[2].toFixed(2)
+        );
+
+        if (!this.is_act(fo.in))
+            return;
+
+        var obj = ebi('f{0}p'.format(fobj.n));
+        obj.innerHTML = fo.hp;
+        obj.style.color = '#fff';
+        var o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
+        obj.style.background = 'linear-gradient(90deg, #050, #270 ' + o1 + '%, #4b0 ' + o2 + '%, #333 ' + o3 + '%)';
+    };
+
+    this.move = function (nfile, newcat) {
+        var fo = this.tab[nfile],
+            oldcat = fo.in,
+            bz_act = this.act == "bz";
+
+        if (oldcat == newcat) {
+            throw 42;
+        }
+
+        fo.in = newcat;
+        this.ctr[oldcat]--;
+        this.ctr[newcat]++;
+        this.drawcard(oldcat);
+        this.drawcard(newcat);
+        if (this.is_act(newcat)) {
+            this.tail++;
+            if (!ebi('f' + nfile))
+                this.addrow(nfile);
+        }
+        else if (this.is_act(oldcat)) {
+            this.head++;
+            if (!bz_act) {
+                var tr = ebi("f" + nfile);
+                tr.parentNode.removeChild(tr);
+            }
+        }
+        if (bz_act) {
+            this.bzw();
+        }
+    };
+
+    this.bzw_log = function (first, last) {
+        console.log("first %d   head %d   tail %d   last %d", first, this.head, this.tail, last);
+        var trs = document.querySelectorAll('#u2tab>tbody>tr'), msg = [];
+        for (var a = 0; a < trs.length; a++)
+            msg.push(trs[a].getAttribute('id'));
+
+        console.log(msg.join(' '));
+    }
+
+    this.bzw = function () {
+        var first = document.querySelector('#u2tab>tbody>tr:first-child');
+        if (!first)
+            return;
+
+        var last = document.querySelector('#u2tab>tbody>tr:last-child');
+        first = parseInt(first.getAttribute('id').slice(1));
+        last = parseInt(last.getAttribute('id').slice(1));
+        //this.bzw_log(first, last);
+
+        while (this.head - first > this.wsz) {
+            var obj = ebi('f' + (first++));
+            obj.parentNode.removeChild(obj);
+        }
+        while (last - this.tail < this.wsz && last < this.tab.length - 2) {
+            var obj = ebi('f' + (++last));
+            if (!obj)
+                this.addrow(last);
+        }
+        //this.bzw_log(first, last);
+        //console.log('--');
+    };
+
+    this.drawcard = function (cat) {
+        var cards = document.querySelectorAll('#u2cards>a>span');
+
+        if (cat == "q") {
+            cards[4].innerHTML = this.ctr[cat];
+            return;
+        }
+        if (cat == "bz") {
+            cards[3].innerHTML = this.ctr[cat];
+            return;
+        }
+
+        cards[2].innerHTML = this.ctr["ok"] + this.ctr["ng"];
+
+        if (cat == "ng") {
+            cards[1].innerHTML = this.ctr[cat];
+        }
+        if (cat == "ok") {
+            cards[0].innerHTML = this.ctr[cat];
+        }
+    };
+
+    this.changecard = function (card) {
+        this.act = card;
+        var html = [];
+        this.head = -1;
+        this.tail = -1;
+        for (var a = 0; a < this.tab.length; a++) {
+            var rt = this.tab[a].in;
+            if (this.is_act(rt)) {
+                html.push(this.genrow(a, true));
+
+                this.tail = a;
+                if (this.head == -1)
+                    this.head = a;
+            }
+        }
+        if (this.head == -1) {
+            this.head = this.tab.length;
+            this.tail = this.head - 1;
+        }
+        if (card == "bz") {
+            for (var a = this.head - 1; a >= this.head - this.wsz && a >= 0; a--) {
+                html.unshift(this.genrow(a, true).replace(/><td>/, "><td>a "));
+            }
+            for (var a = this.tail + 1; a <= this.tail + this.wsz && a < this.tab.length; a++) {
+                html.push(this.genrow(a, true).replace(/><td>/, "><td>b "));
+            }
+        }
+        ebi('u2tab').tBodies[0].innerHTML = html.join('\n');
+    };
+
+    this.genrow = function (nfile, as_html) {
+        var r = this.tab[nfile],
+            td1 = '<td id="f' + nfile,
+            td = '</td>' + td1,
+            ret = td1 + 'n">' + r.hn +
+                td + 't">' + r.ht +
+                td + 'p" class="prog">' + r.hp + '</td>';
+
+        if (as_html)
+            return '<tr id="f' + nfile + '">' + ret + '</tr>';
+
+        var obj = document.createElement('tr');
+        obj.setAttribute('id', 'f' + nfile);
+        obj.innerHTML = ret;
+        return obj;
+    };
+
+    this.addrow = function (nfile) {
+        var tr = this.genrow(nfile);
+        ebi('u2tab').tBodies[0].appendChild(tr);
+    };
+
+    var that = this;
+    btns = document.querySelectorAll(btns + '>a[act]');
+    for (var a = 0; a < btns.length; a++) {
+        btns[a].onclick = function (e) {
+            ev(e);
+            var newtab = this.getAttribute('act');
+            for (var b = 0; b < btns.length; b++) {
+                btns[b].className = (
+                    btns[b].getAttribute('act') == newtab) ? 'act' : '';
+            }
+            that.changecard(newtab);
+        };
+    }
+}
+
+
 function up2k_init(have_crypto) {
     //have_crypto = false;
     var need_filereader_cache = undefined;
@@ -215,10 +490,6 @@ function up2k_init(have_crypto) {
     var flag_en = bcfg_get('flag_en', false);
     var fsearch = bcfg_get('fsearch', false);
 
-    var col_hashing = '#00bbff';
-    var col_hashed = '#004466';
-    var col_uploading = '#ffcc44';
-    var col_uploaded = '#00bb00';
     var fdom_ctr = 0;
     var st = {
         "files": [],
@@ -238,6 +509,8 @@ function up2k_init(have_crypto) {
         }
     };
 
+    var pvis = new U2pvis("bz", '#u2cards');
+
     var bobslice = null;
     if (window.File)
         bobslice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
@@ -278,18 +551,27 @@ function up2k_init(have_crypto) {
         }
         else files = e.target.files;
 
-        if (files.length == 0)
+        if (!files || files.length == 0)
             return alert('no files selected??');
 
         more_one_file();
         var bad_files = [];
         var good_files = [];
+        var dirs = [];
         for (var a = 0; a < files.length; a++) {
             var fobj = files[a];
             if (is_itemlist) {
                 if (fobj.kind !== 'file')
                     continue;
 
+                try {
+                    var wi = fobj.webkitGetAsEntry();
+                    if (wi.isDirectory) {
+                        dirs.push(wi);
+                        continue;
+                    }
+                }
+                catch (ex) { }
                 fobj = fobj.getAsFile();
             }
             try {
@@ -300,36 +582,93 @@ function up2k_init(have_crypto) {
                 bad_files.push(fobj.name);
                 continue;
             }
-            good_files.push(fobj);
+            good_files.push([fobj, fobj.name]);
+        }
+        if (dirs) {
+            return read_dirs(null, [], dirs, good_files, bad_files);
+        }
     }
 
+    function read_dirs(rd, pf, dirs, good, bad) {
+        if (!dirs.length) {
+            if (!pf.length)
+                return gotallfiles(good, bad);
+
+            console.log("retry pf, " + pf.length);
+            setTimeout(function () {
+                read_dirs(rd, pf, dirs, good, bad);
+            }, 50);
+            return;
+        }
+
+        if (!rd)
+            rd = dirs[0].createReader();
+
+        rd.readEntries(function (ents) {
+            var ngot = 0;
+            ents.forEach(function (dn) {
+                if (dn.isDirectory) {
+                    dirs.push(dn);
+                }
+                else {
+                    var name = dn.fullPath;
+                    if (name.indexOf('/') === 0)
+                        name = name.slice(1);
+
+                    pf.push(name);
+                    dn.file(function (fobj) {
+                        var idx = pf.indexOf(name);
+                        pf.splice(idx, 1);
+                        try {
+                            if (fobj.size > 0) {
+                                good.push([fobj, name]);
+                                return;
+                            }
+                        }
+                        catch (ex) { }
+                        bad.push(name);
+                    });
+                }
+                ngot += 1;
+            });
+            // console.log("ngot: " + ngot);
+            if (!ngot) {
+                dirs.shift();
+                rd = null;
+            }
+            return read_dirs(rd, pf, dirs, good, bad);
+        });
+    }
+
+    function gotallfiles(good_files, bad_files) {
         if (bad_files.length > 0) {
-            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, files.length);
+            var ntot = bad_files.length + good_files.length;
-            for (var a = 0; a < bad_files.length; a++)
+            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
+            for (var a = 0, aa = Math.min(20, bad_files.length); a < aa; a++)
                 msg += '-- ' + bad_files[a] + '\n';
 
-            if (files.length - bad_files.length <= 1 && /(android)/i.test(navigator.userAgent))
+            if (good_files.length - bad_files.length <= 1 && /(android)/i.test(navigator.userAgent))
                 msg += '\nFirefox-Android has a bug which prevents selecting multiple files. Try selecting one file at a time. For more info, see firefox bug 1456557';
 
             alert(msg);
         }
 
         var msg = ['upload these ' + good_files.length + ' files?'];
-        for (var a = 0; a < good_files.length; a++)
+        for (var a = 0, aa = Math.min(20, good_files.length); a < aa; a++)
-            msg.push(good_files[a].name);
+            msg.push(good_files[a][1]);
 
         if (ask_up && !fsearch && !confirm(msg.join('\n')))
             return;
 
         for (var a = 0; a < good_files.length; a++) {
-            var fobj = good_files[a];
+            var fobj = good_files[a][0];
             var now = new Date().getTime();
             var lmod = fobj.lastModified || now;
             var entry = {
                 "n": parseInt(st.files.length.toString()),
-                "t0": now,  // TODO remove probably
+                "t0": now,
                 "fobj": fobj,
-                "name": fobj.name,
+                "name": good_files[a][1],
                 "size": fobj.size,
                 "lmod": lmod / 1000,
                 "purl": get_evpath(),
@@ -346,11 +685,12 @@ function up2k_init(have_crypto) {
             if (skip)
                 continue;
 
-            var tr = document.createElement('tr');
+            pvis.addfile([
-            tr.innerHTML = '<td id="f{0}n"></td><td id="f{0}t">hashing</td><td id="f{0}p" class="prog"></td>'.format(st.files.length);
+                fsearch ? esc(entry.name) : linksplit(
-            tr.getElementsByTagName('td')[0].innerHTML = fsearch ? entry.name : linksplit(esc(entry.purl + entry.name)).join(' ');
+                    esc(uricom_dec(entry.purl)[0] + entry.name)).join(' '),
-            ebi('u2tab').appendChild(tr);
+                '📐 hash',
-
+                ''
+            ]);
             st.files.push(entry);
             st.todo.hash.push(entry);
         }
@@ -371,7 +711,10 @@ function up2k_init(have_crypto) {
         for (var a = 0; a < st.files.length; a++) {
             var t = st.files[a];
             if (t.done && t.name) {
-                var tr = ebi('f{0}p'.format(t.n)).parentNode;
+                var tr = ebi('f' + t.n);
+                if (!tr)
+                    continue;
+
                 tr.parentNode.removeChild(tr);
                 t.name = undefined;
             }
@@ -394,7 +737,8 @@ function up2k_init(have_crypto) {
     function hashing_permitted() {
         if (multitask) {
             var ahead = st.bytes.hashed - st.bytes.uploaded;
-            return ahead < 1024 * 1024 * 128;
+            return ahead < 1024 * 1024 * 128 &&
+                st.todo.handshake.length + st.busy.handshake.length < 16;
         }
         return handshakes_permitted() && 0 ==
             st.todo.handshake.length +
@@ -458,7 +802,7 @@ function up2k_init(have_crypto) {
 
                 if (st.todo.handshake.length > 0 &&
                     st.busy.handshake.length == 0 && (
-                        st.todo.handshake[0].t3 || (
+                        st.todo.handshake[0].t4 || (
                             handshakes_permitted() &&
                             st.busy.upload.length < parallel_uploads
                         )
@@ -627,13 +971,8 @@ function up2k_init(have_crypto) {
         if (!need_filereader_cache)
             subchunks = 1;
 
-        var pb_html = '';
+        pvis.setab(t.n, nchunks);
-        var pb_perc = 99.9 / nchunks;
+        pvis.move(t.n, 'bz');
-        for (var a = 0; a < nchunks; a++)
-            pb_html += '<div id="f{0}p{1}" style="width:{2}%"><div></div></div>'.format(
-                t.n, a, pb_perc);
-
-        ebi('f{0}p'.format(t.n)).innerHTML = pb_html;
 
         var reader = new FileReader();
 
@@ -651,8 +990,6 @@ function up2k_init(have_crypto) {
 
             reader.readAsArrayBuffer(
                 bobslice.call(t.fobj, car, cdr));
-
-            prog(t.n, nchunk, col_hashing);
         };
 
         var segm_load = function (e) {
@@ -696,9 +1033,8 @@ function up2k_init(have_crypto) {
             var b64str = buf2b64(hslice).replace(/=$/, '');
             t.hash.push(b64str);
 
-            prog(t.n, nchunk, col_hashed);
+            pvis.hashed(t);
             if (++nchunk < nchunks) {
-                prog(t.n, nchunk, col_hashing);
                 return segm_next();
             }
 
@@ -708,7 +1044,8 @@ function up2k_init(have_crypto) {
                 alert('{0} ms, {1} MB/s\n'.format(t.t2 - t.t1, spd.toFixed(3)) + t.hash.join('\n'));
             }
 
-            ebi('f{0}t'.format(t.n)).innerHTML = 'connecting';
+            pvis.seth(t.n, 2, 'hashing done');
+            pvis.seth(t.n, 1, '📦 wait');
             st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
             st.todo.handshake.push(t);
         };
@@ -744,7 +1081,7 @@ function up2k_init(have_crypto) {
                     else {
                         smsg = 'found';
                         var hit = response.hits[0],
-                            msg = linksplit(hit.rp).join(''),
+                            msg = linksplit(esc(hit.rp)).join(''),
                             tr = unix2iso(hit.ts),
                             tu = unix2iso(t.lmod),
                             diff = parseInt(t.lmod) - parseInt(hit.ts),
@@ -753,8 +1090,9 @@ function up2k_init(have_crypto) {
 
                         msg += '<br /><small>' + tr + ' (srv), ' + tu + ' (You), ' + sdiff + '</span></span>';
                     }
-                    ebi('f{0}p'.format(t.n)).innerHTML = msg;
+                    pvis.seth(t.n, 2, msg);
-                    ebi('f{0}t'.format(t.n)).innerHTML = smsg;
+                    pvis.seth(t.n, 1, smsg);
+                    pvis.move(t.n, smsg == '404' ? 'ng' : 'ok');
                     st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
                     st.bytes.uploaded += t.size;
                     t.done = true;
@@ -765,7 +1103,7 @@ function up2k_init(have_crypto) {
                 if (response.name !== t.name) {
                     // file exists; server renamed us
                     t.name = response.name;
-                    ebi('f{0}n'.format(t.n)).innerHTML = linksplit(esc(t.purl + t.name)).join(' ');
+                    pvis.seth(t.n, 0, linksplit(esc(t.purl + t.name)).join(' '));
                 }
 
                 t.postlist = [];
@@ -779,9 +1117,6 @@ function up2k_init(have_crypto) {
 
                     t.postlist.push(idx);
                 }
-                for (var a = 0; a < t.hash.length; a++)
-                    prog(t.n, a, (t.postlist.indexOf(a) == -1)
-                        ? col_uploaded : col_hashed);
 
                 var done = true;
                 var msg = '&#x1f3b7;&#x1f41b;';
@@ -795,46 +1130,53 @@ function up2k_init(have_crypto) {
                     msg = 'uploading';
                     done = false;
                 }
-                ebi('f{0}t'.format(t.n)).innerHTML = msg;
+                pvis.seth(t.n, 1, msg);
                 st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
 
                 if (done) {
                     t.done = true;
                     st.bytes.uploaded += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
-                    var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
+                    var spd2 = (t.size / ((t.t4 - t.t3) / 1000.)) / (1024 * 1024.);
-                    ebi('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
+                    pvis.seth(t.n, 2, 'hash {0}, up {1} MB/s'.format(
-                        spd1.toFixed(2), spd2.toFixed(2));
+                        spd1.toFixed(2), spd2.toFixed(2)));
+                    pvis.move(t.n, 'ok');
                 }
-                else t.t3 = undefined;
+                else t.t4 = undefined;
 
                 tasker();
             }
             else {
-                var err = "";
+                var err = "",
-                var rsp = (xhr.responseText + '');
+                    rsp = (xhr.responseText + ''),
+                    ofs = rsp.lastIndexOf('\nURL: ');
+
+                if (ofs !== -1)
+                    rsp = rsp.slice(0, ofs);
+
+                if (rsp.indexOf('<pre>') === 0)
+                    rsp = rsp.slice(5);
+
+                st.bytes.uploaded += t.size;
                 if (rsp.indexOf('partial upload exists') !== -1 ||
                     rsp.indexOf('file already exists') !== -1) {
                     err = rsp;
-                    var ofs = err.lastIndexOf(' : ');
-                    if (ofs > 0)
-                        err = err.slice(0, ofs);
-
                     ofs = err.indexOf('\n/');
                     if (ofs !== -1) {
-                        err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2, -1)).join(' ');
+                        err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2)).join(' ');
                     }
                 }
                 if (err != "") {
-                    ebi('f{0}t'.format(t.n)).innerHTML = "ERROR";
+                    pvis.seth(t.n, 1, "ERROR");
-                    ebi('f{0}p'.format(t.n)).innerHTML = err;
+                    pvis.seth(t.n, 2, err);
+                    pvis.move(t.n, 'ng');
 
                     st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
                     tasker();
                     return;
                 }
-                alert("server broke (error {0}):\n\"{1}\"\n".format(
+                alert("server broke; hs-err {0} on file [{1}]:\n".format(
-                    xhr.status,
+                    xhr.status, t.name) + (
                         (xhr.response && xhr.response.err) ||
                         (xhr.responseText && xhr.responseText) ||
                         "no further information"));
@@ -867,7 +1209,7 @@ function up2k_init(have_crypto) {
         var npart = upt.npart;
         var t = st.files[upt.nfile];
 
-        prog(t.n, npart, col_uploading);
+        pvis.seth(t.n, 1, "🚀 send");
 
         var chunksize = get_chunksize(t.size);
         var car = npart * chunksize;
@@ -885,25 +1227,25 @@ function up2k_init(have_crypto) {
             var xhr = new XMLHttpRequest();
             xhr.upload.onprogress = function (xev) {
                 var perc = xev.loaded / (cdr - car) * 100;
-                prog(t.n, npart, '', perc);
+                pvis.prog(t, npart, perc, t);
             };
             xhr.onload = function (xev) {
                 if (xhr.status == 200) {
-                    prog(t.n, npart, col_uploaded);
+                    pvis.prog(t, npart, 101, t);
                     st.bytes.uploaded += cdr - car;
                     t.bytes_uploaded += cdr - car;
                     st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
                     t.postlist.splice(t.postlist.indexOf(npart), 1);
                     if (t.postlist.length == 0) {
-                        t.t3 = new Date().getTime();
+                        t.t4 = new Date().getTime();
-                        ebi('f{0}t'.format(t.n)).innerHTML = 'verifying';
+                        pvis.seth(t.n, 1, 'verifying');
                         st.todo.handshake.unshift(t);
                     }
                     tasker();
                 }
                 else
-                    alert("server broke (error {0}):\n\"{1}\"\n".format(
+                    alert("server broke; cu-err {0} on file [{1}]:\n".format(
-                        xhr.status,
+                        xhr.status, t.name) + (
                             (xhr.response && xhr.response.err) ||
                             (xhr.responseText && xhr.responseText) ||
                             "no further information"));
@@ -913,32 +1255,19 @@ function up2k_init(have_crypto) {
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
             xhr.setRequestHeader('Content-Type', 'application/octet-stream');
+            if (xhr.overrideMimeType)
                 xhr.overrideMimeType('Content-Type', 'application/octet-stream');
+
             xhr.responseType = 'text';
             xhr.send(e.target.result);
+
+            if (!t.t3)
+                t.t3 = new Date().getTime();
         };
 
         reader.readAsArrayBuffer(bobslice.call(t.fobj, car, cdr));
     }
 
-    /////
-    ////
-    ///   progress bar
-    //
-
-    function prog(nfile, nchunk, color, percent) {
-        var n1 = ebi('f{0}p{1}'.format(nfile, nchunk));
-        var n2 = n1.getElementsByTagName('div')[0];
-        if (percent === undefined) {
-            n1.style.background = color;
-            n2.style.display = 'none';
-        }
-        else {
-            n2.style.width = percent + '%';
-            n2.style.display = 'block';
-        }
-    }
-
     /////
     ////
     ///   config ui
@@ -957,6 +1286,7 @@ function up2k_init(have_crypto) {
         if (btn.parentNode !== parent) {
             parent.appendChild(btn);
             ebi('u2conf').setAttribute('class', wide ? 'has_btn' : '');
+            ebi('u2cards').setAttribute('class', wide ? 'w' : '');
         }
     }
     window.addEventListener('resize', onresize);

copyparty/web/upload.css

@@ -47,6 +47,11 @@
 	margin: -1.5em 0;
 	padding: .8em 0;
 	width: 100%;
+	max-width: 12em;
+	display: inline-block;
+}
+#u2conf #u2btn_cw {
+	text-align: right;
 }
 #u2notbtn {
 	display: none;
@@ -72,6 +77,7 @@
 }
 #u2tab td:nth-child(2) {
 	width: 5em;
+	white-space: nowrap;
 }
 #u2tab td:nth-child(3) {
 	width: 40%;
@@ -83,12 +89,41 @@
 #u2tab tr+tr:hover td {
 	background: #222;
 }
+#u2cards {
+	margin: 2.5em auto -2.5em auto;
+	text-align: center;
+}
+#u2cards.w {
+	width: 45em;
+	text-align: left;
+}
+#u2cards a {
+	padding: .2em 1em;
+	border: 1px solid #777;
+	border-width: 0 0 1px 0;
+	background: linear-gradient(to bottom, #333, #222);
+}
+#u2cards a:first-child {
+	border-radius: .4em 0 0 0;
+}
+#u2cards a:last-child {
+	border-radius: 0 .4em 0 0;
+}
+#u2cards a.act {
+	border-width: 1px 1px 0 1px;
+	border-radius: .3em .3em 0 0;
+	margin-left: -1px;
+	background: transparent;
+}
+#u2cards span {
+	color: #fff;
+}
 #u2conf {
 	margin: 1em auto;
 	width: 30em;
 }
 #u2conf.has_btn {
-	width: 46em;
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;
@@ -193,24 +228,6 @@
 .prog {
 	font-family: monospace;
 }
-.prog>div {
-	display: inline-block;
-	position: relative;
-	overflow: hidden;
-	margin: 0;
-	padding: 0;
-	height: 1.1em;
-	margin-bottom: -.15em;
-	box-shadow: -1px -1px 0 inset rgba(255,255,255,0.1);
-}
-.prog>div>div {
-	width: 0%;
-	position: absolute;
-	left: 0;
-	top: 0;
-	bottom: 0;
-	background: #0a0;
-}
 #u2tab a>span {
 	font-weight: bold;
 	font-style: italic;

copyparty/web/upload.html

@@ -1,7 +1,7 @@
 
     <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
@@ -9,7 +9,7 @@
     </div>
 
     <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="mkdir" />
             <input type="text" name="name" size="30">
             <input type="submit" value="mkdir">
@@ -17,15 +17,15 @@
     </div>
 
     <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="new_md" />
             <input type="text" name="name" size="30">
             <input type="submit" value="create doc">
         </form>
     </div>
 
-    <div id="op_msg" class="opview opbox">
+    <div id="op_msg" class="opview opbox act">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="text" name="msg" size="30">
             <input type="submit" value="send msg">
         </form>
@@ -73,17 +73,29 @@
             <div id="u2btn_ct">
                 <div id="u2btn">
                     <span id="u2bm"></span><br />
-                    drop files here<br />
+                    drag/drop files<br />
+                    and folders here<br />
                     (or click me)
                 </div>
             </div>
 
+            <div id="u2cards">
+                <a href="#" act="ok">ok <span>0</span></a><a
+                href="#" act="ng">ng <span>0</span></a><a
+                href="#" act="done">done <span>0</span></a><a
+                href="#" act="bz" class="act">busy <span>0</span></a><a
+                href="#" act="q">que <span>0</span></a>
+            </div>
+
             <table id="u2tab">
+                <thead>
                     <tr>
                         <td>filename</td>
                         <td>status</td>
                         <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
                     </tr>
+                </thead>
+                <tbody></tbody>
             </table>
 
             <p id="u2foot"></p>

copyparty/web/util.js

@@ -1,5 +1,14 @@
 "use strict";
 
+if (!window['console'])
+    window['console'] = {
+        "log": function (msg) { }
+    };
+
+
+var clickev = window.Touch ? 'touchstart' : 'click';
+
+
 // error handler for mobile devices
 function hcroak(msg) {
     document.body.innerHTML = msg;
@@ -110,7 +119,85 @@ function crc32(str) {
         crc = (crc >>> 8) ^ crctab[(crc ^ str.charCodeAt(i)) & 0xFF];
     }
     return ((crc ^ (-1)) >>> 0).toString(16);
-};
+}
+
+
+function clmod(obj, cls, add) {
+    var re = new RegExp('\\s*\\b' + cls + '\\s*\\b', 'g');
+    if (add == 't')
+        add = !re.test(obj.className);
+
+    obj.className = obj.className.replace(re, ' ') + (add ? ' ' + cls : '');
+}
+
+
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["lead", -1, ""], ["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v)
+                        v = uricom_dec(v)[0]
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map(function (x) { return x; });
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+    }
+    return nodes;
+}
 
 
 function sortTable(table, col, cb) {
@@ -186,7 +273,6 @@ function makeSortable(table, cb) {
 }
 
 
-
 (function () {
     var ops = document.querySelectorAll('#ops>a');
     for (var a = 0; a < ops.length; a++) {
@@ -212,16 +298,16 @@ function opclick(e) {
 function goto(dest) {
     var obj = document.querySelectorAll('.opview.act');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     obj = document.querySelectorAll('#ops>a');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     if (dest) {
         var ui = ebi('op_' + dest);
-        ui.classList.add('act');
+        clmod(ui, 'act', true);
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+        document.querySelector('#ops>a[data-dest=' + dest + ']').className += " act";
 
         var fn = window['goto_' + dest];
         if (fn)
@@ -237,7 +323,10 @@ function goto(dest) {
     goto();
     var op = sread('opmode');
     if (op !== null && op !== '.')
+        try {
             goto(op);
+        }
+        catch (ex) { }
 })();
 
 
@@ -405,8 +494,7 @@ function bcfg_upd_ui(name, val) {
     if (o.getAttribute('type') == 'checkbox')
         o.checked = val;
     else if (o) {
-        var fun = val ? 'add' : 'remove';
+        clmod(o, 'on', val);
-        o.classList[fun]('on');
     }
 }
 

docs/notes.sh

@@ -73,6 +73,13 @@ shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*10
 command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
 
 
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+
 ##
 ## sqlite3 stuff
 
@@ -83,6 +90,9 @@ sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2
 time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
 cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
 
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
 
 ##
 ## media
@@ -126,6 +136,16 @@ pip install virtualenv
 # readme toc
 cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
 
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
 
 ##
 ## http 206

scripts/make-sfx.sh

@@ -45,11 +45,13 @@ pybin=$(command -v python3 || command -v python) || {
 	exit 1
 }
 
+use_gz=
 do_sh=1
 do_py=1
 while [ ! -z "$1" ]; do
 	[ "$1" = clean  ] && clean=1  && shift && continue
 	[ "$1" = re     ] && repack=1 && shift && continue
+	[ "$1" = gz     ] && use_gz=1 && shift && continue
 	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
 	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
 	[ "$1" = no-sh  ] && do_sh=   && shift && continue
@@ -161,7 +163,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -204,16 +206,20 @@ args=(--owner=1000 --group=1000)
 
 tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
 
+pc=bzip2
+pe=bz2
+[ $use_gz ] && pc=gzip && pe=gz
+
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-[ $do_py ] && { for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2; }
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2; }
 [ $do_sh ] && { for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
 rm t.* || true
 exts=()
 
 
 [ $do_sh ] && {
-exts+=(sh)
+exts+=(.sh)
 echo creating unix sfx
 (
 	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
@@ -224,17 +230,30 @@ echo creating unix sfx
 
 
 [ $do_py ] && {
-exts+=(py)
+	echo creating generic sfx
-echo creating generic sfx
+
-$pybin ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
+	py=../scripts/sfx.py
-mv sfx.out $sfx_out.py
+	suf=
-chmod 755 $sfx_out.*
+	[ $use_gz ] && {
+		sed -r 's/"r:bz2"/"r:gz"/' <$py >$py.t
+		py=$py.t
+		suf=-gz
+	}
+
+	$pybin $py --sfx-make tar.bz2 $ver $ts
+	mv sfx.out $sfx_out$suf.py
+	
+	exts+=($suf.py)
+	[ $use_gz ] &&
+		rm $py
 }
 
 
+chmod 755 $sfx_out*
+
 printf "done:\n"
 for ext in ${exts[@]}; do
-	printf "  %s\n" "$(realpath $sfx_out)."$ext
+	printf "  %s\n" "$(realpath $sfx_out)"$ext
 done
 
 # apk add bash python3 tar xz bzip2

scripts/sfx.py

@@ -2,7 +2,7 @@
 # coding: latin-1
 from __future__ import print_function, unicode_literals
 
-import os, sys, time, shutil, runpy, tarfile, hashlib, platform, tempfile, traceback
+import os, sys, time, shutil, threading, tarfile, hashlib, platform, tempfile, traceback
 
 """
 run me with any version of python, i will unpack and run copyparty
@@ -26,6 +26,7 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
+WINDOWS = sys.platform == "win32"
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
 cpp = None
@@ -343,6 +344,21 @@ def get_payload():
                 break
 
 
+def utime(top):
+    i = 0
+    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    while WINDOWS:
+        t = int(time.time())
+        if i:
+            msg("utime {}, {}".format(i, t))
+
+        for f in files:
+            os.utime(f, (t, t))
+
+        i += 1
+        time.sleep(78123)
+
+
 def confirm(rv):
     msg()
     msg(traceback.format_exc())
@@ -362,15 +378,20 @@ def run(tmp, j2ver):
     msg("sfxdir:", tmp)
     msg()
 
-    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    # block systemd-tmpfiles-clean.timer
     try:
         import fcntl
 
         fd = os.open(tmp, os.O_RDONLY)
         fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
         tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
-    except:
+    except Exception as ex:
-        pass
+        if not WINDOWS:
+            msg("\033[31mflock:", repr(ex))
+
+    t = threading.Thread(target=utime, args=(tmp,))
+    t.daemon = True
+    t.start()
 
     ld = [tmp, os.path.join(tmp, "dep-j2")]
     if j2ver:
@@ -380,7 +401,10 @@ def run(tmp, j2ver):
         sys.path.insert(0, x)
 
     try:
-        runpy.run_module(str("copyparty"), run_name=str("__main__"))
+        from copyparty.__main__ import main as copyparty
+
+        copyparty()
+
     except SystemExit as ex:
         if ex.code:
             confirm(ex.code)

tests/test_vfs.py

@@ -18,7 +18,9 @@ from copyparty import util
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)