v1.1.0

* extract codec and format info
* add FFprobe as fallback when mutagen fails
* add option to blacklist FFprobe for tags

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: '9001'
+
+---
+
+NOTE:
+all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md
+
+
+**Describe the bug**
+a description of what the bug is
+
+**To Reproduce**
+List of steps to reproduce the issue, or, if it's hard to reproduce, then at least a detailed explanation of what you did to run into it
+
+**Expected behavior**
+a description of what you expected to happen
+
+**Screenshots**
+if applicable, add screenshots to help explain your problem, such as the kickass crashpage :^)
+
+**Server details**
+if the issue is possibly on the server-side, then mention some of the following:
+* server OS / version: 
+* python version: 
+* copyparty arguments: 
+* filesystem (`lsblk -f` on linux): 
+
+**Client details**
+if the issue is possibly on the client-side, then mention some of the following:
+* the device type and model: 
+* OS version: 
+* browser version: 
+
+**Additional context**
+any other context about the problem here

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: '9001'
+
+---
+
+all of the below are optional, consider them as inspiration, delete and rewrite at will
+
+**is your feature request related to a problem? Please describe.**
+a description of what the problem is, for example, `I'm always frustrated when [...]` or `Why is it not possible to [...]`
+
+**Describe the idea / solution you'd like**
+a description of what you want to happen
+
+**Describe any alternatives you've considered**
+a description of any alternative solutions or features you've considered
+
+**Additional context**
+add any other context or screenshots about the feature request here

.github/ISSUE_TEMPLATE/something-else.md

@@ -0,0 +1,10 @@
+---
+name: Something else
+about: "┐(ﾟ∀ﾟ)┌"
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/branch-rename.md

@@ -0,0 +1,7 @@
+modernize your local checkout of the repo like so,
+```sh
+git branch -m master hovudstraum
+git fetch origin
+git branch -u origin/hovudstraum hovudstraum
+git remote set-head origin -a
+```

.gitignore

@@ -9,6 +9,7 @@ buildenv/
 build/
 dist/
 sfx/
+py2/
 .venv/
 
 # ide
@@ -20,3 +21,7 @@ sfx/
 # derived
 copyparty/web/deps/
 srv/
+
+# state/logs
+up.*.txt
+.hist/

.vscode/launch.json

@@ -16,12 +16,9 @@
                 "-e2ts",
                 "-mtp",
                 ".bpm=f,bin/mtag/audio-bpm.py",
-                "-a",
-                "ed:wark",
-                "-v",
-                "srv::r:aed:cnodupe",
-                "-v",
-                "dist:dist:r"
+                "-aed:wark",
+                "-vsrv::r:rw,ed:c,dupe",
+                "-vdist:dist:r"
             ]
         },
         {
@@ -43,5 +40,13 @@
                 "${file}"
             ]
         },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
     ]
 }

.vscode/launch.py

@@ -3,16 +3,18 @@
 # launches 10x faster than mspython debugpy
 # and is stoppable with ^C
 
+import re
 import os
 import sys
+
+print(sys.executable)
+
 import shlex
-
-sys.path.insert(0, os.getcwd())
-
 import jstyleson
-from copyparty.__main__ import main as copyparty
+import subprocess as sp
 
-with open(".vscode/launch.json", "r") as f:
+
+with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
 
 oj = jstyleson.loads(tj)
@@ -25,11 +27,19 @@ except:
     pass
 
 argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
-try:
-    copyparty(["a"] + argv)
-except SystemExit as ex:
-    if ex.code:
-        raise
+
+if re.search(" -j ?[0-9]", " ".join(argv)):
+    argv = [sys.executable, "-m", "copyparty"] + argv
+    sp.check_call(argv)
+else:
+    sys.path.insert(0, os.getcwd())
+    from copyparty.__main__ import main as copyparty
+
+    try:
+        copyparty(["a"] + argv)
+    except SystemExit as ex:
+        if ex.code:
+            raise
 
 print("\n\033[32mokke\033[0m")
 sys.exit(1)

.vscode/settings.json

@@ -55,4 +55,5 @@
         "py27"
     ],
     "python.linting.enabled": true,
+    "python.pythonPath": "/usr/bin/python3"
 }

.vscode/tasks.json

@@ -9,7 +9,10 @@
         {
             "label": "no_dbg",
             "type": "shell",
-            "command": "${config:python.pythonPath} .vscode/launch.py"
+            "command": "${config:python.pythonPath}",
+            "args": [
+                ".vscode/launch.py"
+            ]
         }
     ]
 }

CODE_OF_CONDUCT.md

@@ -0,0 +1,24 @@
+in the words of Abraham Lincoln:
+
+> Be excellent to each other... and... PARTY ON, DUDES!
+
+more specifically I'll paraphrase some examples from a german automotive corporation as they cover all the bases without being too wordy
+
+## Examples of unacceptable behavior
+* intimidation, harassment, trolling
+* insulting, derogatory, harmful or prejudicial comments
+* posting private information without permission
+* political or personal attacks
+
+## Examples of expected behavior
+* being nice, friendly, welcoming, inclusive, mindful and empathetic
+* acting considerate, modest, respectful
+* using polite and inclusive language
+* criticize constructively and accept constructive criticism
+* respect different points of view
+
+## finally and even more specifically,
+* parse opinions and feedback objectively without prejudice
+  * it's the message that matters, not who said it
+
+aaand that's how you say `be nice` in a way that fills half a floppy w

CONTRIBUTING.md

@@ -0,0 +1,3 @@
+* do something cool
+
+really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight

bin/README.md

@@ -1,3 +1,11 @@
+# [`up2k.py`](up2k.py)
+* command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
+* file uploads, file-search, autoresume of aborted/broken uploads
+* faster than browsers
+* early beta, if something breaks just restart it
+
+
+
 # [`copyparty-fuse.py`](copyparty-fuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
@@ -45,3 +53,26 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 # [`mtag/`](mtag/)
 * standalone programs which perform misc. file analysis
 * copyparty can Popen programs like these during file indexing to collect additional metadata
+
+
+
+# [`dbtool.py`](dbtool.py)
+upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
+
+for that example (upgrading to v0.11.20), first launch the new version of copyparty like usual, let it make a backup of the old db and rebuild the new db until the point where it starts running mtp (colored messages as it adds the mtp tags), that's when you hit CTRL-C and patch in the old mtp tags from the old db instead
+
+so assuming you have `-mtp` parsers to provide the tags `key` and `.bpm`:
+
+```
+cd /mnt/nas/music/.hist
+~/src/copyparty/bin/dbtool.py -ls up2k.db
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -cmp
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy key
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
+```
+
+
+
+# [`prisonparty.sh`](prisonparty.sh)
+* run copyparty in a chroot, preventing any accidental file access
+* creates bindmounts for /bin, /lib, and so on, see `sysdirs=`

bin/copyparty-fuse-streaming.py

@@ -345,7 +345,7 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, headers={}, **kwargs):
+    def sendreq(self, meth, path, headers, **kwargs):
         if self.password:
             headers["Cookie"] = "=".join(["cppwd", self.password])
 
@@ -354,21 +354,21 @@ class Gateway(object):
             if c.rx_path:
                 raise Exception()
 
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             c.rx = c.getresponse()
             return c
         except:
             tid = threading.current_thread().ident
             dbg(
-                "\033[1;37;44mbad conn {:x}\n  {}\n  {}\033[0m".format(
-                    tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
+                "\033[1;37;44mbad conn {:x}\n  {} {}\n  {}\033[0m".format(
+                    tid, meth, path, c.rx_path if c else "(null)"
                 )
             )
 
         self.closeconn(c)
         c = self.getconn()
         try:
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             c.rx = c.getresponse()
             return c
         except:
@@ -386,7 +386,7 @@ class Gateway(object):
             path = dewin(path)
 
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
-        c = self.sendreq("GET", web_path)
+        c = self.sendreq("GET", web_path, {})
         if c.rx.status != 200:
             self.closeconn(c)
             log(
@@ -440,7 +440,7 @@ class Gateway(object):
             )
         )
 
-        c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        c = self.sendreq("GET", web_path, {"Range": hdr_range})
         if c.rx.status != http.client.PARTIAL_CONTENT:
             self.closeconn(c)
             raise Exception(

bin/copyparty-fuse.py

@@ -22,7 +22,7 @@ dependencies:
 
 note:
   you probably want to run this on windows clients:
-  https://github.com/9001/copyparty/blob/master/contrib/explorer-nothumbs-nofoldertypes.reg
+  https://github.com/9001/copyparty/blob/hovudstraum/contrib/explorer-nothumbs-nofoldertypes.reg
 
 get server cert:
   awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
@@ -54,6 +54,15 @@ MACOS = platform.system() == "Darwin"
 info = log = dbg = None
 
 
+print(
+    "{} v{} @ {}".format(
+        platform.python_implementation(),
+        ".".join([str(x) for x in sys.version_info]),
+        sys.executable,
+    )
+)
+
+
 try:
     from fuse import FUSE, FuseOSError, Operations
 except:
@@ -62,7 +71,7 @@ except:
     elif MACOS:
         libfuse = "install https://osxfuse.github.io/"
     else:
-        libfuse = "apt install libfuse\n    modprobe fuse"
+        libfuse = "apt install libfuse3-3\n    modprobe fuse"
 
     print(
         "\n  could not import fuse; these may help:"
@@ -293,14 +302,14 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, headers={}, **kwargs):
+    def sendreq(self, meth, path, headers, **kwargs):
         tid = get_tid()
         if self.password:
             headers["Cookie"] = "=".join(["cppwd", self.password])
 
         try:
             c = self.getconn(tid)
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             return c.getresponse()
         except:
             dbg("bad conn")
@@ -308,7 +317,7 @@ class Gateway(object):
         self.closeconn(tid)
         try:
             c = self.getconn(tid)
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             return c.getresponse()
         except:
             info("http connection failed:\n" + traceback.format_exc())
@@ -325,7 +334,7 @@ class Gateway(object):
             path = dewin(path)
 
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
-        r = self.sendreq("GET", web_path)
+        r = self.sendreq("GET", web_path, {})
         if r.status != 200:
             self.closeconn()
             log(
@@ -362,7 +371,7 @@ class Gateway(object):
             )
         )
 
-        r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        r = self.sendreq("GET", web_path, {"Range": hdr_range})
         if r.status != http.client.PARTIAL_CONTENT:
             self.closeconn()
             raise Exception(
@@ -384,15 +393,16 @@ class Gateway(object):
 
         rsp = json.loads(rsp.decode("utf-8"))
         ret = []
-        for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
+        for statfun, nodes in [
+            [self.stat_dir, rsp["dirs"]],
+            [self.stat_file, rsp["files"]],
+        ]:
             for n in nodes:
-                fname = unquote(n["href"]).rstrip(b"/")
-                fname = fname.decode("wtf-8")
+                fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
                 if bad_good:
                     fname = enwin(fname)
 
-                fun = self.stat_dir if is_dir else self.stat_file
-                ret.append([fname, fun(n["ts"], n["sz"]), 0])
+                ret.append([fname, statfun(n["ts"], n["sz"]), 0])
 
         return ret
 

bin/dbtool.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import shutil
+import sqlite3
+import argparse
+
+DB_VER1 = 3
+DB_VER2 = 4
+
+
+def die(msg):
+    print("\033[31m\n" + msg + "\n\033[0m")
+    sys.exit(1)
+
+
+def read_ver(db):
+    for tab in ["ki", "kv"]:
+        try:
+            c = db.execute(r"select v from {} where k = 'sver'".format(tab))
+        except:
+            continue
+
+        rows = c.fetchall()
+        if rows:
+            return int(rows[0][0])
+
+    return "corrupt"
+
+
+def ls(db):
+    nfiles = next(db.execute("select count(w) from up"))[0]
+    ntags = next(db.execute("select count(w) from mt"))[0]
+    print(f"{nfiles} files")
+    print(f"{ntags} tags\n")
+
+    print("number of occurences for each tag,")
+    print(" 'x' = file has no tags")
+    print(" 't:mtp' = the mtp flag (file not mtp processed yet)")
+    print()
+    for k, nk in db.execute("select k, count(k) from mt group by k order by k"):
+        print(f"{nk:9} {k}")
+
+
+def compare(n1, d1, n2, d2, verbose):
+    nt = next(d1.execute("select count(w) from up"))[0]
+    n = 0
+    miss = 0
+    for w1, rd, fn in d1.execute("select w, rd, fn from up"):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} files in {n1} against {n2}\033[0m"
+            print(m)
+
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select w from up where rd = ? and fn = ?"
+        hit = d2.execute(q, (rd, fn)).fetchone()
+        if not hit:
+            miss += 1
+            if verbose:
+                print(f"file in {n1} missing in {n2}: [{w1}] {rd}/{fn}")
+
+    print(f" {miss} files in {n1} missing in {n2}\n")
+
+    nt = next(d1.execute("select count(w) from mt"))[0]
+    n = 0
+    miss = {}
+    nmiss = 0
+    for w1, k, v in d1.execute("select * from mt"):
+
+        n += 1
+        if n % 100_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
+            print(m)
+
+        q = "select rd, fn from up where substr(w,1,16) = ?"
+        rd, fn = d1.execute(q, (w1,)).fetchone()
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
+        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if w2:
+            w2 = w2[0]
+
+        v2 = None
+        if w2:
+            v2 = d2.execute(
+                "select v from mt where w = ? and +k = ?", (w2, k)
+            ).fetchone()
+            if v2:
+                v2 = v2[0]
+
+        # if v != v2 and v2 and k in [".bpm", "key"] and n2 == "src":
+        #    print(f"{w} [{rd}/{fn}] {k} = [{v}] / [{v2}]")
+
+        if v2 is not None:
+            if k.startswith("."):
+                try:
+                    diff = abs(float(v) - float(v2))
+                    if diff > float(v) / 0.9:
+                        v2 = None
+                    else:
+                        v2 = v
+                except:
+                    pass
+
+            if v != v2:
+                v2 = None
+
+        if v2 is None:
+            nmiss += 1
+            try:
+                miss[k] += 1
+            except:
+                miss[k] = 1
+
+            if verbose:
+                print(f"missing in {n2}: [{w1}] [{rd}/{fn}] {k} = {v}")
+
+    for k, v in sorted(miss.items()):
+        if v:
+            print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
+
+    print(f"in total, {nmiss} missing tags in {n2}\n")
+
+
+def copy_mtp(d1, d2, tag, rm):
+    nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
+    n = 0
+    ndone = 0
+    for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
+            print(m)
+
+        q = "select rd, fn from up where substr(w,1,16) = ?"
+        rd, fn = d1.execute(q, (w1,)).fetchone()
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
+        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if not w2:
+            continue
+
+        w2 = w2[0]
+        hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
+        if hit:
+            hit = hit[0]
+
+        if hit != v:
+            ndone += 1
+            if hit is not None:
+                d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
+
+            d2.execute("insert into mt values (?,?,?)", (w2, k, v))
+            if rm:
+                d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
+
+    d2.commit()
+    print(f"copied {ndone} {tag} tags over")
+
+
+def main():
+    os.system("")
+    print()
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db", help="database to work on")
+    ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
+
+    ap2 = ap.add_argument_group("informational / read-only stuff")
+    ap2.add_argument("-v", action="store_true", help="verbose")
+    ap2.add_argument("-ls", action="store_true", help="list summary for db")
+    ap2.add_argument("-cmp", action="store_true", help="compare databases")
+
+    ap2 = ap.add_argument_group("options which modify target db")
+    ap2.add_argument("-copy", metavar="TAG", type=str, help="mtp tag to copy over")
+    ap2.add_argument(
+        "-rm-mtp-flag",
+        action="store_true",
+        help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
+    )
+    ap2.add_argument("-vac", action="store_true", help="optimize DB")
+
+    ar = ap.parse_args()
+
+    for v in [ar.db, ar.src]:
+        if v and not os.path.exists(v):
+            die("database must exist")
+
+    db = sqlite3.connect(ar.db)
+    ds = sqlite3.connect(ar.src) if ar.src else None
+
+    # revert journals
+    for d, p in [[db, ar.db], [ds, ar.src]]:
+        if not d:
+            continue
+
+        pj = "{}-journal".format(p)
+        if not os.path.exists(pj):
+            continue
+
+        d.execute("create table foo (bar int)")
+        d.execute("drop table foo")
+
+    if ar.copy:
+        db.close()
+        shutil.copy2(ar.db, "{}.bak.dbtool.{:x}".format(ar.db, int(time.time())))
+        db = sqlite3.connect(ar.db)
+
+    for d, n in [[ds, "src"], [db, "dst"]]:
+        if not d:
+            continue
+
+        ver = read_ver(d)
+        if ver == "corrupt":
+            die("{} database appears to be corrupt, sorry")
+
+        if ver < DB_VER1 or ver > DB_VER2:
+            m = f"{n} db is version {ver}, this tool only supports versions between {DB_VER1} and {DB_VER2}, please upgrade it with copyparty first"
+            die(m)
+
+    if ar.ls:
+        ls(db)
+
+    if ar.cmp:
+        if not ds:
+            die("need src db to compare against")
+
+        compare("src", ds, "dst", db, ar.v)
+        compare("dst", db, "src", ds, ar.v)
+
+    if ar.copy:
+        copy_mtp(ds, db, ar.copy, ar.rm_mtp_flag)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/README.md

@@ -1,9 +1,18 @@
 standalone programs which take an audio file as argument
 
+**NOTE:** these all require `-e2ts` to be functional, meaning you need to do at least one of these: `apt install ffmpeg` or `pip3 install mutagen`
+
 some of these rely on libraries which are not MIT-compatible
 
 * [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
 * [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
+* [media-hash.py](./media-hash.py) generates checksums for audio and video streams; uses FFmpeg (LGPL or GPL)
+
+these do not have any problematic dependencies:
+
+* [cksum.py](./cksum.py) computes various checksums
+* [exe.py](./exe.py) grabs metadata from .exe and .dll files (example for retrieving multiple tags with one parser)
+* [wget.py](./wget.py) lets you download files by POSTing URLs to copyparty
 
 
 # dependencies
@@ -18,7 +27,10 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
 
 # usage from copyparty
 
-`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
+`copyparty -e2dsa -e2ts` followed by any combination of these:
+* `-mtp key=f,audio-key.py`
+* `-mtp .bpm=f,audio-bpm.py`
+* `-mtp ahash,vhash=f,media-hash.py`
 
 * `f,` makes the detected value replace any existing values
 * the `.` in `.bpm` indicates numeric value
@@ -29,6 +41,9 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
 ## usage with volume-flags
 
 instead of affecting all volumes, you can set the options for just one volume like so:
-```
-copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
-```
+
+`copyparty -v /mnt/nas/music:/music:r:c,e2dsa:c,e2ts` immediately followed by any combination of these:
+
+* `:c,mtp=key=f,audio-key.py`
+* `:c,mtp=.bpm=f,audio-bpm.py`
+* `:c,mtp=ahash,vhash=f,media-hash.py`

bin/mtag/audio-bpm.py

@@ -25,6 +25,7 @@ def det(tf):
         "-v", "fatal",
         "-ss", "13",
         "-y", "-i", fsenc(sys.argv[1]),
+        "-map", "0:a:0",
         "-ac", "1",
         "-ar", "22050",
         "-t", "300",
@@ -60,7 +61,7 @@ def main():
     try:
         det(tf)
     except:
-        pass
+        pass  # mute
     finally:
         os.unlink(tf)
 

bin/mtag/audio-key-slicing.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+import re
+import os
+import sys
+import tempfile
+import subprocess as sp
+
+import keyfinder
+
+from copyparty.util import fsenc
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+
+note: this is a janky edition of the regular audio-key.py,
+  slicing the files at 20sec intervals and keeping 5sec from each,
+  surprisingly accurate but still garbage (446 ok, 69 bad, 13% miss)
+
+  it is fast tho
+"""
+
+
+def get_duration():
+    # TODO provide ffprobe tags to mtp as json
+
+    # fmt: off
+    dur = sp.check_output([
+        "ffprobe",
+        "-hide_banner",
+        "-v", "fatal",
+        "-show_streams",
+        "-show_format",
+        fsenc(sys.argv[1])
+    ])
+    # fmt: on
+
+    dur = dur.decode("ascii", "replace").split("\n")
+    dur = [x.split("=")[1] for x in dur if x.startswith("duration=")]
+    dur = [float(x) for x in dur if re.match(r"^[0-9\.,]+$", x)]
+    return list(sorted(dur))[-1] if dur else None
+
+
+def get_segs(dur):
+    # keep first 5s of each 20s,
+    # keep entire last segment
+    ofs = 0
+    segs = []
+    while True:
+        seg = [ofs, 5]
+        segs.append(seg)
+        if dur - ofs < 20:
+            seg[-1] = int(dur - seg[0])
+            break
+
+        ofs += 20
+
+    return segs
+
+
+def slice(tf):
+    dur = get_duration()
+    dur = min(dur, 600)  # max 10min
+    segs = get_segs(dur)
+
+    # fmt: off
+    cmd = [
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-y"
+    ]
+
+    for seg in segs:
+        cmd.extend([
+            "-ss", str(seg[0]),
+            "-i", fsenc(sys.argv[1])
+        ])
+    
+    filt = ""
+    for n, seg in enumerate(segs):
+        filt += "[{}:a:0]atrim=duration={}[a{}]; ".format(n, seg[1], n)
+    
+    prev = "a0"
+    for n in range(1, len(segs)):
+        nxt = "b{}".format(n)
+        filt += "[{}][a{}]acrossfade=d=0.5[{}]; ".format(prev, n, nxt)
+        prev = nxt
+
+    cmd.extend([
+        "-filter_complex", filt[:-2],
+        "-map", "[{}]".format(nxt),
+        "-sample_fmt", "s16",
+        tf
+    ])
+    # fmt: on
+
+    # print(cmd)
+    sp.check_call(cmd)
+
+
+def det(tf):
+    slice(tf)
+    print(keyfinder.key(tf).camelot())
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    finally:
+        os.unlink(tf)
+        pass
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key.py

@@ -1,18 +1,55 @@
 #!/usr/bin/env python
 
+import os
 import sys
+import tempfile
+import subprocess as sp
 import keyfinder
 
+from copyparty.util import fsenc
+
 """
 dep: github/mixxxdj/libkeyfinder
 dep: pypi/keyfinder
 dep: ffmpeg
-
-note: cannot fsenc
 """
 
 
-try:
-    print(keyfinder.key(sys.argv[1]).camelot())
-except:
-    pass
+# tried trimming the first/last 5th, bad idea,
+# misdetects 9a law field (Sphere Caliber) as 10b,
+# obvious when mixing 9a ghostly parapara ship
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-map", "0:a:0",
+        "-t", "300",
+        "-sample_fmt", "s16",
+        tf
+    ])
+    # fmt: on
+
+    print(keyfinder.key(tf).camelot())
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass  # mute
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/cksum.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+import zlib
+import struct
+import base64
+import hashlib
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p
+
+
+"""
+calculates various checksums for uploads,
+usage: -mtp crc32,md5,sha1,sha256b=bin/mtag/cksum.py
+"""
+
+
+def main():
+    config = "crc32 md5 md5b sha1 sha1b sha256 sha256b sha512/240 sha512b/240"
+    # b suffix = base64 encoded
+    # slash = truncate to n bits
+
+    known = {
+        "md5": hashlib.md5,
+        "sha1": hashlib.sha1,
+        "sha256": hashlib.sha256,
+        "sha512": hashlib.sha512,
+    }
+    config = config.split()
+    hashers = {
+        k: v()
+        for k, v in known.items()
+        if k in [x.split("/")[0].rstrip("b") for x in known]
+    }
+    crc32 = 0 if "crc32" in config else None
+
+    with open(fsenc(sys.argv[1]), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            for x in hashers.values():
+                x.update(buf)
+
+            if crc32 is not None:
+                crc32 = zlib.crc32(buf, crc32)
+
+    ret = {}
+    for s in config:
+        alg = s.split("/")[0]
+        b64 = alg.endswith("b")
+        alg = alg.rstrip("b")
+        if alg in hashers:
+            v = hashers[alg].digest()
+        elif alg == "crc32":
+            v = crc32
+            if v < 0:
+                v &= 2 ** 32 - 1
+            v = struct.pack(">L", v)
+        else:
+            raise Exception("what is {}".format(s))
+
+        if "/" in s:
+            v = v[: int(int(s.split("/")[1]) / 8)]
+
+        if b64:
+            v = base64.b64encode(v).decode("ascii").rstrip("=")
+        else:
+            try:
+                v = v.hex()
+            except:
+                import binascii
+
+                v = binascii.hexlify(v)
+
+        ret[s] = v
+
+    print(json.dumps(ret, indent=4))
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/exe.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+
+import sys
+import time
+import json
+import pefile
+
+"""
+retrieve exe info,
+example for multivalue providers
+"""
+
+
+def unk(v):
+    return "unk({:04x})".format(v)
+
+
+class PE2(pefile.PE):
+    def __init__(self, *a, **ka):
+        for k in [
+            # -- parse_data_directories:
+            "parse_import_directory",
+            "parse_export_directory",
+            # "parse_resources_directory",
+            "parse_debug_directory",
+            "parse_relocations_directory",
+            "parse_directory_tls",
+            "parse_directory_load_config",
+            "parse_delay_import_directory",
+            "parse_directory_bound_imports",
+            # -- full_load:
+            "parse_rich_header",
+        ]:
+            setattr(self, k, self.noop)
+
+        super(PE2, self).__init__(*a, **ka)
+
+    def noop(*a, **ka):
+        pass
+
+
+try:
+    pe = PE2(sys.argv[1], fast_load=False)
+except:
+    sys.exit(0)
+
+arch = pe.FILE_HEADER.Machine
+if arch == 0x14C:
+    arch = "x86"
+elif arch == 0x8664:
+    arch = "x64"
+else:
+    arch = unk(arch)
+
+try:
+    buildtime = time.gmtime(pe.FILE_HEADER.TimeDateStamp)
+    buildtime = time.strftime("%Y-%m-%d_%H:%M:%S", buildtime)
+except:
+    buildtime = "invalid"
+
+ui = pe.OPTIONAL_HEADER.Subsystem
+if ui == 2:
+    ui = "GUI"
+elif ui == 3:
+    ui = "cmdline"
+else:
+    ui = unk(ui)
+
+extra = {}
+if hasattr(pe, "FileInfo"):
+    for v1 in pe.FileInfo:
+        for v2 in v1:
+            if v2.name != "StringFileInfo":
+                continue
+
+            for v3 in v2.StringTable:
+                for k, v in v3.entries.items():
+                    v = v.decode("utf-8", "replace").strip()
+                    if not v:
+                        continue
+
+                    if k in [b"FileVersion", b"ProductVersion"]:
+                        extra["ver"] = v
+
+                    if k in [b"OriginalFilename", b"InternalName"]:
+                        extra["orig"] = v
+
+r = {
+    "arch": arch,
+    "built": buildtime,
+    "ui": ui,
+    "cksum": "{:08x}".format(pe.OPTIONAL_HEADER.CheckSum),
+}
+r.update(extra)
+
+print(json.dumps(r, indent=4))

bin/mtag/file-ext.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+
+"""
+example that just prints the file extension
+"""
+
+print(sys.argv[1].split(".")[-1])

bin/mtag/install-deps.sh

@@ -4,7 +4,8 @@ set -e
 
 # install dependencies for audio-*.py
 #
-# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
+# linux/alpine: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf cmake
+# linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
 # win64: requires msys2-mingw64 environment
 # macos: requires macports
 #

bin/mtag/media-hash.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import re
+import sys
+import json
+import time
+import base64
+import hashlib
+import subprocess as sp
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p
+
+
+"""
+dep: ffmpeg
+"""
+
+
+def det():
+    # fmt: off
+    cmd = [
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-i", fsenc(sys.argv[1]),
+        "-f", "framemd5",
+        "-"
+    ]
+    # fmt: on
+
+    p = sp.Popen(cmd, stdout=sp.PIPE)
+    # ps = io.TextIOWrapper(p.stdout, encoding="utf-8")
+    ps = p.stdout
+
+    chans = {}
+    for ln in ps:
+        if ln.startswith(b"#stream#"):
+            break
+
+        m = re.match(r"^#media_type ([0-9]): ([a-zA-Z])", ln.decode("utf-8"))
+        if m:
+            chans[m.group(1)] = m.group(2)
+
+    hashers = [hashlib.sha512(), hashlib.sha512()]
+    for ln in ps:
+        n = int(ln[:1])
+        v = ln.rsplit(b",", 1)[-1].strip()
+        hashers[n].update(v)
+
+    r = {}
+    for k, v in chans.items():
+        dg = hashers[int(k)].digest()[:12]
+        dg = base64.urlsafe_b64encode(dg).decode("ascii")
+        r[v[0].lower() + "hash"] = dg
+
+    print(json.dumps(r, indent=4))
+
+
+def main():
+    try:
+        det()
+    except:
+        pass  # mute
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/res/yt-ipr.conf

@@ -0,0 +1,39 @@
+# example config file to use copyparty as a youtube manifest collector,
+# use with copyparty like:  python copyparty.py -c yt-ipr.conf
+#
+# see docs/example.conf for a better explanation of the syntax, but
+# newlines are block separators, so adding blank lines inside a volume definition is bad
+# (use comments as separators instead)
+
+
+# create user ed, password wark
+u ed:wark
+
+
+# create a volume at /ytm which stores files at ./srv/ytm
+./srv/ytm
+/ytm
+# write-only, but read-write for user ed
+w
+rw ed
+# rescan the volume on startup
+c e2dsa
+# collect tags from all new files since last scan
+c e2ts
+# optionally enable compression to make the files 50% smaller
+c pk
+# only allow uploads which are between 16k and 1m large
+c sz=16k-1m
+# allow up to 10 uploads over 5 minutes from each ip
+c maxn=10,300
+# move uploads into subfolders: YEAR-MONTH / DAY-HOUR / <upload>
+c rotf=%Y-%m/%d-%H
+# delete uploads when they are 24 hours old
+c lifetime=86400
+# add the parser and tell copyparty what tags it can expect from it
+c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
+# decide which tags we want to index and in what order
+c mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
+
+
+# create any other volumes you'd like down here, or merge this with an existing config file

bin/mtag/res/yt-ipr.user.js

@@ -0,0 +1,47 @@
+// ==UserScript==
+// @name    youtube-playerdata-hub
+// @match   https://youtube.com/*
+// @match   https://*.youtube.com/*
+// @version 1.0
+// @grant   GM_addStyle
+// ==/UserScript==
+
+function main() {
+    var server = 'https://127.0.0.1:3923/ytm?pw=wark',
+        interval = 60; // sec
+
+    var sent = {};
+    function send(txt, mf_url, desc) {
+        if (sent[mf_url])
+            return;
+
+        fetch(server + '&_=' + Date.now(), { method: "PUT", body: txt });
+        console.log('[yt-pdh] yeet %d bytes, %s', txt.length, desc);
+        sent[mf_url] = 1;
+    }
+
+    function collect() {
+        try {
+            var pd = document.querySelector('ytd-watch-flexy');
+            if (!pd)
+                return console.log('[yt-pdh] no video found');
+
+            pd = pd.playerData;
+            var mu = pd.streamingData.dashManifestUrl || pd.streamingData.hlsManifestUrl;
+            if (!mu || !mu.length)
+                return console.log('[yt-pdh] no manifest found');
+
+            var desc = pd.videoDetails.videoId + ', ' + pd.videoDetails.title;
+            send(JSON.stringify(pd), mu, desc);
+        }
+        catch (ex) {
+            console.log("[yt-pdh]", ex);
+        }
+    }
+    setInterval(collect, interval * 1000);
+}
+
+var scr = document.createElement('script');
+scr.textContent = '(' + main.toString() + ')();';
+(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
+console.log('[yt-pdh] a');

bin/mtag/wget.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+"""
+use copyparty as a file downloader by POSTing URLs as
+application/x-www-form-urlencoded (for example using the
+message/pager function on the website)
+
+example copyparty config to use this:
+  --urlform save,get -vsrv/wget:wget:rwmd,ed:c,e2ts,mtp=title=ebin,t300,ad,bin/mtag/wget.py
+
+explained:
+  for realpath srv/wget (served at /wget) with read-write-modify-delete for ed,
+  enable file analysis on upload (e2ts),
+  use mtp plugin "bin/mtag/wget.py" to provide metadata tag "title",
+  do this on all uploads with the file extension "bin",
+  t300 = 300 seconds timeout for each dwonload,
+  ad = parse file regardless if FFmpeg thinks it is audio or not
+
+PS: this requires e2ts to be functional,
+  meaning you need to do at least one of these:
+   * apt install ffmpeg
+   * pip3 install mutagen
+"""
+
+
+import os
+import sys
+import subprocess as sp
+from urllib.parse import unquote_to_bytes as unquote
+
+
+def main():
+    fp = os.path.abspath(sys.argv[1])
+    fdir = os.path.dirname(fp)
+    fname = os.path.basename(fp)
+    if not fname.startswith("put-") or not fname.endswith(".bin"):
+        raise Exception("not a post file")
+
+    buf = b""
+    with open(fp, "rb") as f:
+        while True:
+            b = f.read(4096)
+            buf += b
+            if len(buf) > 4096:
+                raise Exception("too big")
+
+            if not b:
+                break
+
+    if not buf:
+        raise Exception("file is empty")
+
+    buf = unquote(buf.replace(b"+", b" "))
+    url = buf.decode("utf-8")
+
+    if not url.startswith("msg="):
+        raise Exception("does not start with msg=")
+
+    url = url[4:]
+    if "://" not in url:
+        url = "https://" + url
+
+    os.chdir(fdir)
+
+    name = url.split("?")[0].split("/")[-1]
+    tfn = "-- DOWNLOADING " + name
+    open(tfn, "wb").close()
+
+    cmd = ["wget", "--trust-server-names", "--", url]
+
+    try:
+        sp.check_call(cmd)
+
+        # OPTIONAL:
+        #   on success, delete the .bin file which contains the URL
+        os.unlink(fp)
+    except:
+        open("-- FAILED TO DONWLOAD " + name, "wb").close()
+
+    os.unlink(tfn)
+    print(url)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/yt-ipr.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+
+import re
+import os
+import sys
+import gzip
+import json
+import base64
+import string
+import urllib.request
+from datetime import datetime
+
+"""
+youtube initial player response
+
+it's probably best to use this through a config file; see res/yt-ipr.conf
+
+but if you want to use plain arguments instead then:
+  -v srv/ytm:ytm:w:rw,ed
+       :c,e2ts,e2dsa
+       :c,sz=16k-1m:c,maxn=10,300:c,rotf=%Y-%m/%d-%H
+       :c,mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
+       :c,mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
+
+see res/yt-ipr.user.js for the example userscript to go with this
+"""
+
+
+def main():
+    try:
+        with gzip.open(sys.argv[1], "rt", encoding="utf-8", errors="replace") as f:
+            txt = f.read()
+    except:
+        with open(sys.argv[1], "r", encoding="utf-8", errors="replace") as f:
+            txt = f.read()
+
+    txt = "{" + txt.split("{", 1)[1]
+
+    try:
+        pd = json.loads(txt)
+    except json.decoder.JSONDecodeError as ex:
+        pd = json.loads(txt[: ex.pos])
+
+    # print(json.dumps(pd, indent=2))
+
+    if "videoDetails" in pd:
+        parse_youtube(pd)
+    else:
+        parse_freg(pd)
+
+
+def get_expiration(url):
+    et = re.search(r"[?&]expire=([0-9]+)", url).group(1)
+    et = datetime.utcfromtimestamp(int(et))
+    return et.strftime("%Y-%m-%d, %H:%M")
+
+
+def parse_youtube(pd):
+    vd = pd["videoDetails"]
+    sd = pd["streamingData"]
+
+    et = sd["adaptiveFormats"][0]["url"]
+    et = get_expiration(et)
+
+    mf = []
+    if "dashManifestUrl" in sd:
+        mf.append("dash")
+    if "hlsManifestUrl" in sd:
+        mf.append("hls")
+
+    r = {
+        "yt-id": vd["videoId"],
+        "yt-title": vd["title"],
+        "yt-author": vd["author"],
+        "yt-channel": vd["channelId"],
+        "yt-views": vd["viewCount"],
+        "yt-private": vd["isPrivate"],
+        # "yt-expires": sd["expiresInSeconds"],
+        "yt-manifest": ",".join(mf),
+        "yt-expires": et,
+    }
+    print(json.dumps(r))
+
+    freg_conv(pd)
+
+
+def parse_freg(pd):
+    md = pd["metadata"]
+    r = {
+        "yt-id": md["id"],
+        "yt-title": md["title"],
+        "yt-author": md["channelName"],
+        "yt-channel": md["channelURL"].strip("/").split("/")[-1],
+        "yt-expires": get_expiration(list(pd["video"].values())[0]),
+    }
+    print(json.dumps(r))
+
+
+def freg_conv(pd):
+    # based on getURLs.js v1.5 (2021-08-07)
+    # fmt: off
+    priority = {
+        "video": [
+            337, 315, 266, 138,  # 2160p60
+            313, 336,  # 2160p
+            308,  # 1440p60
+            271, 264,  # 1440p
+            335, 303, 299,  # 1080p60
+            248, 169, 137,  # 1080p
+            334, 302, 298,  # 720p60
+            247, 136  # 720p
+        ],
+        "audio": [
+            251, 141, 171, 140, 250, 249, 139
+        ]
+    }
+
+    vid_id = pd["videoDetails"]["videoId"]
+    chan_id = pd["videoDetails"]["channelId"]
+
+    try:
+        thumb_url = pd["microformat"]["playerMicroformatRenderer"]["thumbnail"]["thumbnails"][0]["url"]
+        start_ts = pd["microformat"]["playerMicroformatRenderer"]["liveBroadcastDetails"]["startTimestamp"]
+    except:
+        thumb_url = f"https://img.youtube.com/vi/{vid_id}/maxresdefault.jpg"
+        start_ts = ""
+
+    # fmt: on
+
+    metadata = {
+        "title": pd["videoDetails"]["title"],
+        "id": vid_id,
+        "channelName": pd["videoDetails"]["author"],
+        "channelURL": "https://www.youtube.com/channel/" + chan_id,
+        "description": pd["videoDetails"]["shortDescription"],
+        "thumbnailUrl": thumb_url,
+        "startTimestamp": start_ts,
+    }
+
+    if [x for x in vid_id if x not in string.ascii_letters + string.digits + "_-"]:
+        print(f"malicious json", file=sys.stderr)
+        return
+
+    basepath = os.path.dirname(sys.argv[1])
+
+    thumb_fn = f"{basepath}/{vid_id}.jpg"
+    tmp_fn = f"{thumb_fn}.{os.getpid()}"
+    if not os.path.exists(thumb_fn) and (
+        thumb_url.startswith("https://img.youtube.com/vi/")
+        or thumb_url.startswith("https://i.ytimg.com/vi/")
+    ):
+        try:
+            with urllib.request.urlopen(thumb_url) as fi:
+                with open(tmp_fn, "wb") as fo:
+                    fo.write(fi.read())
+
+            os.rename(tmp_fn, thumb_fn)
+        except:
+            if os.path.exists(tmp_fn):
+                os.unlink(tmp_fn)
+
+    try:
+        with open(thumb_fn, "rb") as f:
+            thumb = base64.b64encode(f.read()).decode("ascii")
+    except:
+        thumb = "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/yQALCAABAAEBAREA/8wABgAQEAX/2gAIAQEAAD8A0s8g/9k="
+
+    metadata["thumbnail"] = "data:image/jpeg;base64," + thumb
+
+    ret = {
+        "metadata": metadata,
+        "version": "1.5",
+        "createTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
+    }
+
+    for stream, itags in priority.items():
+        for itag in itags:
+            url = None
+            for afmt in pd["streamingData"]["adaptiveFormats"]:
+                if itag == afmt["itag"]:
+                    url = afmt["url"]
+                    break
+
+            if url:
+                ret[stream] = {itag: url}
+                break
+
+    fn = f"{basepath}/{vid_id}.urls.json"
+    with open(fn, "w", encoding="utf-8", errors="replace") as f:
+        f.write(json.dumps(ret, indent=4))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except:
+        # raise
+        pass

bin/prisonparty.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+set -e
+
+# runs copyparty (or any other program really) in a chroot
+#
+# assumption: these directories, and everything within, are owned by root
+sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
+
+
+# error-handler
+help() { cat <<'EOF'
+
+usage:
+  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- copyparty-sfx.py [...]"
+
+example:
+  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- copyparty-sfx.py -v /mnt/nas/music::rwmd"
+
+EOF
+exit 1
+}
+
+
+# read arguments
+trap help EXIT
+jail="$(realpath "$1")"; shift
+uid="$1"; shift
+gid="$1"; shift
+
+vols=()
+while true; do
+	v="$1"; shift
+	[ "$v" = -- ] && break  # end of volumes
+	[ "$#" -eq 0 ] && break  # invalid usage
+	vols+=( "$(realpath "$v")" )
+done
+pybin="$1"; shift
+pybin="$(realpath "$pybin")"
+cpp="$1"; shift
+cpp="$(realpath "$cpp")"
+cppdir="$(dirname "$cpp")"
+trap - EXIT
+
+
+# debug/vis
+echo
+echo "chroot-dir = $jail"
+echo "user:group = $uid:$gid"
+echo " copyparty = $cpp"
+echo
+printf '\033[33m%s\033[0m\n' "copyparty can access these folders and all their subdirectories:"
+for v in "${vols[@]}"; do
+	printf '\033[36m ├─\033[0m %s \033[36m ── added by (You)\033[0m\n' "$v"
+done
+printf '\033[36m ├─\033[0m %s \033[36m ── where the copyparty binary is\033[0m\n' "$cppdir"
+printf '\033[36m ╰─\033[0m %s \033[36m ── the folder you are currently in\033[0m\n' "$PWD"
+vols+=("$cppdir" "$PWD")
+echo
+
+
+# remove any trailing slashes
+jail="${jail%/}"
+cppdir="${cppdir%/}"
+
+
+# bind-mount system directories and volumes
+printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | LC_ALL=C sort |
+while IFS= read -r v; do
+	[ -e "$v" ] || {
+		# printf '\033[1;31mfolder does not exist:\033[0m %s\n' "/$v"
+		continue
+	}
+	i1=$(stat -c%D.%i "$v"      2>/dev/null || echo a)
+	i2=$(stat -c%D.%i "$jail$v" 2>/dev/null || echo b)
+	[ $i1 = $i2 ] && continue
+	
+	mkdir -p "$jail$v"
+	mount --bind "$v" "$jail$v"
+done
+
+
+# create a tmp
+mkdir -p "$jail/tmp"
+chmod 777 "$jail/tmp"
+
+
+# run copyparty
+/sbin/chroot --userspec=$uid:$gid "$jail" "$pybin" "$cpp" "$@" && rv=0 || rv=$?
+
+
+# cleanup if not in use
+lsof "$jail" | grep -qF "$jail" &&
+	echo "chroot is in use, will not cleanup" ||
+{
+	mount | grep -qF " on $jail" |
+	awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
+	LC_ALL=C sort -r  | tee /dev/stderr | tr '\n' '\0' | xargs -r0 umount
+}
+exit $rv

bin/up2k.py

@@ -0,0 +1,800 @@
+#!/usr/bin/env python3
+from __future__ import print_function, unicode_literals
+
+"""
+up2k.py: upload to copyparty
+2021-10-31, v0.11, ed <irc.rizon.net>, MIT-Licensed
+https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
+
+- dependencies: requests
+- supports python 2.6, 2.7, and 3.3 through 3.10
+
+- almost zero error-handling
+- but if something breaks just try again and it'll autoresume
+"""
+
+import os
+import sys
+import stat
+import math
+import time
+import atexit
+import signal
+import base64
+import hashlib
+import argparse
+import platform
+import threading
+import requests
+import datetime
+
+
+# from copyparty/__init__.py
+PY2 = sys.version_info[0] == 2
+if PY2:
+    from Queue import Queue
+    from urllib import unquote
+    from urllib import quote
+
+    sys.dont_write_bytecode = True
+    bytes = str
+else:
+    from queue import Queue
+    from urllib.parse import unquote_to_bytes as unquote
+    from urllib.parse import quote_from_bytes as quote
+
+    unicode = str
+
+VT100 = platform.system() != "Windows"
+
+
+req_ses = requests.Session()
+
+
+class File(object):
+    """an up2k upload task; represents a single file"""
+
+    def __init__(self, top, rel, size, lmod):
+        self.top = top  # type: bytes
+        self.rel = rel.replace(b"\\", b"/")  # type: bytes
+        self.size = size  # type: int
+        self.lmod = lmod  # type: float
+
+        self.abs = os.path.join(top, rel)  # type: bytes
+        self.name = self.rel.split(b"/")[-1].decode("utf-8", "replace")  # type: str
+
+        # set by get_hashlist
+        self.cids = []  # type: list[tuple[str, int, int]]  # [ hash, ofs, sz ]
+        self.kchunks = {}  # type: dict[str, tuple[int, int]]  # hash: [ ofs, sz ]
+
+        # set by handshake
+        self.ucids = []  # type: list[str]  # chunks which need to be uploaded
+        self.wark = None  # type: str
+        self.url = None  # type: str
+
+        # set by upload
+        self.up_b = 0  # type: int
+        self.up_c = 0  # type: int
+
+        # m = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
+        # eprint(m.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
+
+
+class FileSlice(object):
+    """file-like object providing a fixed window into a file"""
+
+    def __init__(self, file, cid):
+        # type: (File, str) -> FileSlice
+
+        self.car, self.len = file.kchunks[cid]
+        self.cdr = self.car + self.len
+        self.ofs = 0  # type: int
+        self.f = open(file.abs, "rb", 512 * 1024)
+        self.f.seek(self.car)
+
+        # https://stackoverflow.com/questions/4359495/what-is-exactly-a-file-like-object-in-python
+        # IOBase, RawIOBase, BufferedIOBase
+        funs = "close closed __enter__ __exit__ __iter__ isatty __next__ readable seekable writable"
+        try:
+            for fun in funs.split():
+                setattr(self, fun, getattr(self.f, fun))
+        except:
+            pass  # py27 probably
+
+    def tell(self):
+        return self.ofs
+
+    def seek(self, ofs, wh=0):
+        if wh == 1:
+            ofs = self.ofs + ofs
+        elif wh == 2:
+            ofs = self.len + ofs  # provided ofs is negative
+
+        if ofs < 0:
+            ofs = 0
+        elif ofs >= self.len:
+            ofs = self.len - 1
+
+        self.ofs = ofs
+        self.f.seek(self.car + ofs)
+
+    def read(self, sz):
+        sz = min(sz, self.len - self.ofs)
+        ret = self.f.read(sz)
+        self.ofs += len(ret)
+        return ret
+
+
+_print = print
+
+
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    ka["end"] = ""
+    if not PY2:
+        ka["flush"] = True
+
+    _print(*a, **ka)
+    if PY2 or not VT100:
+        sys.stderr.flush()
+
+
+def flushing_print(*a, **ka):
+    _print(*a, **ka)
+    if "flush" not in ka:
+        sys.stdout.flush()
+
+
+if not VT100:
+    print = flushing_print
+
+
+def termsize():
+    import os
+
+    env = os.environ
+
+    def ioctl_GWINSZ(fd):
+        try:
+            import fcntl, termios, struct, os
+
+            cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
+        except:
+            return
+        return cr
+
+    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
+    if not cr:
+        try:
+            fd = os.open(os.ctermid(), os.O_RDONLY)
+            cr = ioctl_GWINSZ(fd)
+            os.close(fd)
+        except:
+            pass
+    if not cr:
+        try:
+            cr = (env["LINES"], env["COLUMNS"])
+        except:
+            cr = (25, 80)
+    return int(cr[1]), int(cr[0])
+
+
+class CTermsize(object):
+    def __init__(self):
+        self.ev = False
+        self.margin = None
+        self.g = None
+        self.w, self.h = termsize()
+
+        try:
+            signal.signal(signal.SIGWINCH, self.ev_sig)
+        except:
+            return
+
+        thr = threading.Thread(target=self.worker)
+        thr.daemon = True
+        thr.start()
+
+    def worker(self):
+        while True:
+            time.sleep(0.5)
+            if not self.ev:
+                continue
+
+            self.ev = False
+            self.w, self.h = termsize()
+
+            if self.margin is not None:
+                self.scroll_region(self.margin)
+
+    def ev_sig(self, *a, **ka):
+        self.ev = True
+
+    def scroll_region(self, margin):
+        self.margin = margin
+        if margin is None:
+            self.g = None
+            eprint("\033[s\033[r\033[u")
+        else:
+            self.g = 1 + self.h - margin
+            m = "{0}\033[{1}A".format("\n" * margin, margin)
+            eprint("{0}\033[s\033[1;{1}r\033[u".format(m, self.g - 1))
+
+
+ss = CTermsize()
+
+
+def statdir(top):
+    """non-recursive listing of directory contents, along with stat() info"""
+    if hasattr(os, "scandir"):
+        with os.scandir(top) as dh:
+            for fh in dh:
+                yield [os.path.join(top, fh.name), fh.stat()]
+    else:
+        for name in os.listdir(top):
+            abspath = os.path.join(top, name)
+            yield [abspath, os.stat(abspath)]
+
+
+def walkdir(top):
+    """recursive statdir"""
+    for ap, inf in sorted(statdir(top)):
+        if stat.S_ISDIR(inf.st_mode):
+            for x in walkdir(ap):
+                yield x
+        else:
+            yield ap, inf
+
+
+def walkdirs(tops):
+    """recursive statdir for a list of tops, yields [top, relpath, stat]"""
+    sep = "{0}".format(os.sep).encode("ascii")
+    for top in tops:
+        if top[-1:] == sep:
+            stop = top.rstrip(sep)
+        else:
+            stop = os.path.dirname(top)
+
+        if os.path.isdir(top):
+            for ap, inf in walkdir(top):
+                yield stop, ap[len(stop) :].lstrip(sep), inf
+        else:
+            d, n = top.rsplit(sep, 1)
+            yield d, n, os.stat(top)
+
+
+# mostly from copyparty/util.py
+def quotep(btxt):
+    quot1 = quote(btxt, safe=b"/")
+    if not PY2:
+        quot1 = quot1.encode("ascii")
+
+    return quot1.replace(b" ", b"+")
+
+
+# from copyparty/util.py
+def humansize(sz, terse=False):
+    """picks a sensible unit for the given extent"""
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
+        if sz < 1024:
+            break
+
+        sz /= 1024.0
+
+    ret = " ".join([str(sz)[:4].rstrip("."), unit])
+
+    if not terse:
+        return ret
+
+    return ret.replace("iB", "").replace(" ", "")
+
+
+# from copyparty/up2k.py
+def up2k_chunksize(filesize):
+    """gives The correct chunksize for up2k hashing"""
+    chunksize = 1024 * 1024
+    stepsize = 512 * 1024
+    while True:
+        for mul in [1, 2]:
+            nchunks = math.ceil(filesize * 1.0 / chunksize)
+            if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
+                return chunksize
+
+            chunksize += stepsize
+            stepsize *= mul
+
+
+# mostly from copyparty/up2k.py
+def get_hashlist(file, pcb):
+    # type: (File, any) -> None
+    """generates the up2k hashlist from file contents, inserts it into `file`"""
+
+    chunk_sz = up2k_chunksize(file.size)
+    file_rem = file.size
+    file_ofs = 0
+    ret = []
+    with open(file.abs, "rb", 512 * 1024) as f:
+        while file_rem > 0:
+            hashobj = hashlib.sha512()
+            chunk_sz = chunk_rem = min(chunk_sz, file_rem)
+            while chunk_rem > 0:
+                buf = f.read(min(chunk_rem, 64 * 1024))
+                if not buf:
+                    raise Exception("EOF at " + str(f.tell()))
+
+                hashobj.update(buf)
+                chunk_rem -= len(buf)
+
+            digest = hashobj.digest()[:33]
+            digest = base64.urlsafe_b64encode(digest).decode("utf-8")
+
+            ret.append([digest, file_ofs, chunk_sz])
+            file_ofs += chunk_sz
+            file_rem -= chunk_sz
+
+            if pcb:
+                pcb(file, file_ofs)
+
+    file.cids = ret
+    file.kchunks = {}
+    for k, v1, v2 in ret:
+        file.kchunks[k] = [v1, v2]
+
+
+def handshake(req_ses, url, file, pw, search):
+    # type: (requests.Session, str, File, any, bool) -> List[str]
+    """
+    performs a handshake with the server; reply is:
+      if search, a list of search results
+      otherwise, a list of chunks to upload
+    """
+
+    req = {
+        "hash": [x[0] for x in file.cids],
+        "name": file.name,
+        "lmod": file.lmod,
+        "size": file.size,
+    }
+    if search:
+        req["srch"] = 1
+
+    headers = {"Content-Type": "text/plain"}  # wtf ed
+    if pw:
+        headers["Cookie"] = "=".join(["cppwd", pw])
+
+    if file.url:
+        url = file.url
+    elif b"/" in file.rel:
+        url += quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8", "replace")
+
+    while True:
+        try:
+            r = req_ses.post(url, headers=headers, json=req)
+            break
+        except:
+            eprint("handshake failed, retry...\n")
+            time.sleep(1)
+
+    try:
+        r = r.json()
+    except:
+        raise Exception(r.text)
+
+    if search:
+        return r["hits"]
+
+    try:
+        pre, url = url.split("://")
+        pre += "://"
+    except:
+        pre = ""
+
+    file.url = pre + url.split("/")[0] + r["purl"]
+    file.name = r["name"]
+    file.wark = r["wark"]
+
+    return r["hash"]
+
+
+def upload(req_ses, file, cid, pw):
+    # type: (requests.Session, File, str, any) -> None
+    """upload one specific chunk, `cid` (a chunk-hash)"""
+
+    headers = {
+        "X-Up2k-Hash": cid,
+        "X-Up2k-Wark": file.wark,
+        "Content-Type": "application/octet-stream",
+    }
+    if pw:
+        headers["Cookie"] = "=".join(["cppwd", pw])
+
+    f = FileSlice(file, cid)
+    try:
+        r = req_ses.post(file.url, headers=headers, data=f)
+        if not r:
+            raise Exception(repr(r))
+
+        _ = r.content
+    finally:
+        f.f.close()
+
+
+class Daemon(threading.Thread):
+    def __init__(self, *a, **ka):
+        threading.Thread.__init__(self, *a, **ka)
+        self.daemon = True
+
+
+class Ctl(object):
+    """
+    this will be the coordinator which runs everything in parallel
+    (hashing, handshakes, uploads)  but right now it's p dumb
+    """
+
+    def __init__(self, ar):
+        self.ar = ar
+        ar.files = [
+            os.path.abspath(os.path.realpath(x.encode("utf-8")))
+            + (x[-1:] if x[-1:] == os.sep else "").encode("utf-8")
+            for x in ar.files
+        ]
+        ar.url = ar.url.rstrip("/") + "/"
+        if "://" not in ar.url:
+            ar.url = "http://" + ar.url
+
+        eprint("\nscanning {0} locations\n".format(len(ar.files)))
+
+        nfiles = 0
+        nbytes = 0
+        for _, _, inf in walkdirs(ar.files):
+            nfiles += 1
+            nbytes += inf.st_size
+
+        eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
+        self.nfiles = nfiles
+        self.nbytes = nbytes
+
+        if ar.td:
+            requests.packages.urllib3.disable_warnings()
+            req_ses.verify = False
+        if ar.te:
+            req_ses.verify = ar.te
+
+        self.filegen = walkdirs(ar.files)
+        if ar.safe:
+            self.safe()
+        else:
+            self.fancy()
+
+    def safe(self):
+        """minimal basic slow boring fallback codepath"""
+        search = self.ar.s
+        for nf, (top, rel, inf) in enumerate(self.filegen):
+            file = File(top, rel, inf.st_size, inf.st_mtime)
+            upath = file.abs.decode("utf-8", "replace")
+
+            print("{0} {1}\n  hash...".format(self.nfiles - nf, upath))
+            get_hashlist(file, None)
+
+            burl = self.ar.url[:8] + self.ar.url[8:].split("/")[0] + "/"
+            while True:
+                print("  hs...")
+                hs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+                if search:
+                    if hs:
+                        for hit in hs:
+                            print("  found: {0}{1}".format(burl, hit["rp"]))
+                    else:
+                        print("  NOT found")
+                    break
+
+                file.ucids = hs
+                if not hs:
+                    break
+
+                print("{0} {1}".format(self.nfiles - nf, upath))
+                ncs = len(hs)
+                for nc, cid in enumerate(hs):
+                    print("  {0} up {1}".format(ncs - nc, cid))
+                    upload(req_ses, file, cid, self.ar.a)
+
+            print("  ok!")
+
+    def fancy(self):
+        self.hash_f = 0
+        self.hash_c = 0
+        self.hash_b = 0
+        self.up_f = 0
+        self.up_c = 0
+        self.up_b = 0
+        self.up_br = 0
+        self.hasher_busy = 1
+        self.handshaker_busy = 0
+        self.uploader_busy = 0
+
+        self.t0 = time.time()
+        self.t0_up = None
+        self.spd = None
+
+        self.mutex = threading.Lock()
+        self.q_handshake = Queue()  # type: Queue[File]
+        self.q_recheck = Queue()  # type: Queue[File]  # partial upload exists [...]
+        self.q_upload = Queue()  # type: Queue[tuple[File, str]]
+
+        self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
+        self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
+        if VT100:
+            atexit.register(self.cleanup_vt100)
+            ss.scroll_region(3)
+
+        Daemon(target=self.hasher).start()
+        for _ in range(self.ar.j):
+            Daemon(target=self.handshaker).start()
+            Daemon(target=self.uploader).start()
+
+        idles = 0
+        while idles < 3:
+            time.sleep(0.07)
+            with self.mutex:
+                if (
+                    self.q_handshake.empty()
+                    and self.q_upload.empty()
+                    and not self.hasher_busy
+                    and not self.handshaker_busy
+                    and not self.uploader_busy
+                ):
+                    idles += 1
+                else:
+                    idles = 0
+
+            if VT100:
+                maxlen = ss.w - len(str(self.nfiles)) - 14
+                txt = "\033[s\033[{0}H".format(ss.g)
+                for y, k, st, f in [
+                    [0, "hash", self.st_hash, self.hash_f],
+                    [1, "send", self.st_up, self.up_f],
+                ]:
+                    txt += "\033[{0}H{1}:".format(ss.g + y, k)
+                    file, arg = st
+                    if not file:
+                        txt += " {0}\033[K".format(arg)
+                    else:
+                        if y:
+                            p = 100 * file.up_b / file.size
+                        else:
+                            p = 100 * arg / file.size
+
+                        name = file.abs.decode("utf-8", "replace")[-maxlen:]
+                        if "/" in name:
+                            name = "\033[36m{0}\033[0m/{1}".format(*name.rsplit("/", 1))
+
+                        m = "{0:6.1f}% {1} {2}\033[K"
+                        txt += m.format(p, self.nfiles - f, name)
+
+                txt += "\033[{0}H ".format(ss.g + 2)
+            else:
+                txt = " "
+
+            if not self.up_br:
+                spd = self.hash_b / (time.time() - self.t0)
+                eta = (self.nbytes - self.hash_b) / (spd + 1)
+            else:
+                spd = self.up_br / (time.time() - self.t0_up)
+                spd = self.spd = (self.spd or spd) * 0.9 + spd * 0.1
+                eta = (self.nbytes - self.up_b) / (spd + 1)
+
+            spd = humansize(spd)
+            eta = str(datetime.timedelta(seconds=int(eta)))
+            left = humansize(self.nbytes - self.up_b)
+            tail = "\033[K\033[u" if VT100 else "\r"
+
+            m = "eta: {0} @ {1}/s, {2} left".format(eta, spd, left)
+            eprint(txt + "\033]0;{0}\033\\\r{1}{2}".format(m, m, tail))
+
+    def cleanup_vt100(self):
+        ss.scroll_region(None)
+        eprint("\033[J\033]0;\033\\")
+
+    def cb_hasher(self, file, ofs):
+        self.st_hash = [file, ofs]
+
+    def hasher(self):
+        prd = None
+        ls = {}
+        for top, rel, inf in self.filegen:
+            if self.ar.z:
+                rd = os.path.dirname(rel)
+                if prd != rd:
+                    prd = rd
+                    headers = {}
+                    if self.ar.a:
+                        headers["Cookie"] = "=".join(["cppwd", self.ar.a])
+
+                    ls = {}
+                    try:
+                        print("      ls ~{0}".format(rd.decode("utf-8", "replace")))
+                        r = req_ses.get(
+                            self.ar.url.encode("utf-8") + quotep(rd) + b"?ls",
+                            headers=headers,
+                        )
+                        for f in r.json()["files"]:
+                            rfn = f["href"].split("?")[0].encode("utf-8", "replace")
+                            ls[unquote(rfn)] = f
+                    except:
+                        print("   mkdir ~{0}".format(rd.decode("utf-8", "replace")))
+
+                rf = ls.get(os.path.basename(rel), None)
+                if rf and rf["sz"] == inf.st_size and abs(rf["ts"] - inf.st_mtime) <= 1:
+                    self.nfiles -= 1
+                    self.nbytes -= inf.st_size
+                    continue
+
+            file = File(top, rel, inf.st_size, inf.st_mtime)
+            while True:
+                with self.mutex:
+                    if (
+                        self.hash_b - self.up_b < 1024 * 1024 * 128
+                        and self.hash_c - self.up_c < 64
+                        and (
+                            not self.ar.nh
+                            or (
+                                self.q_upload.empty()
+                                and self.q_handshake.empty()
+                                and not self.uploader_busy
+                            )
+                        )
+                    ):
+                        break
+
+                time.sleep(0.05)
+
+            get_hashlist(file, self.cb_hasher)
+            with self.mutex:
+                self.hash_f += 1
+                self.hash_c += len(file.cids)
+                self.hash_b += file.size
+
+            self.q_handshake.put(file)
+
+        self.hasher_busy = 0
+        self.st_hash = [None, "(finished)"]
+
+    def handshaker(self):
+        search = self.ar.s
+        q = self.q_handshake
+        burl = self.ar.url[:8] + self.ar.url[8:].split("/")[0] + "/"
+        while True:
+            file = q.get()
+            if not file:
+                if q == self.q_handshake:
+                    q = self.q_recheck
+                    q.put(None)
+                    continue
+
+                self.q_upload.put(None)
+                break
+
+            with self.mutex:
+                self.handshaker_busy += 1
+
+            upath = file.abs.decode("utf-8", "replace")
+
+            try:
+                hs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+            except Exception as ex:
+                if q == self.q_handshake and "<pre>partial upload exists" in str(ex):
+                    self.q_recheck.put(file)
+                    hs = []
+                else:
+                    raise
+
+            if search:
+                if hs:
+                    for hit in hs:
+                        m = "found: {0}\n  {1}{2}\n"
+                        print(m.format(upath, burl, hit["rp"]), end="")
+                else:
+                    print("NOT found: {0}\n".format(upath), end="")
+
+                with self.mutex:
+                    self.up_f += 1
+                    self.up_c += len(file.cids)
+                    self.up_b += file.size
+                    self.handshaker_busy -= 1
+
+                continue
+
+            with self.mutex:
+                if not hs:
+                    # all chunks done
+                    self.up_f += 1
+                    self.up_c += len(file.cids) - file.up_c
+                    self.up_b += file.size - file.up_b
+
+                if hs and file.up_c:
+                    # some chunks failed
+                    self.up_c -= len(hs)
+                    file.up_c -= len(hs)
+                    for cid in hs:
+                        sz = file.kchunks[cid][1]
+                        self.up_b -= sz
+                        file.up_b -= sz
+
+                file.ucids = hs
+                self.handshaker_busy -= 1
+
+            if not hs:
+                kw = "uploaded" if file.up_b else "   found"
+                print("{0} {1}".format(kw, upath))
+            for cid in hs:
+                self.q_upload.put([file, cid])
+
+    def uploader(self):
+        while True:
+            task = self.q_upload.get()
+            if not task:
+                self.st_up = [None, "(finished)"]
+                break
+
+            with self.mutex:
+                self.uploader_busy += 1
+                self.t0_up = self.t0_up or time.time()
+
+            file, cid = task
+            try:
+                upload(req_ses, file, cid, self.ar.a)
+            except:
+                eprint("upload failed, retry...\n")
+                pass  # handshake will fix it
+
+            with self.mutex:
+                sz = file.kchunks[cid][1]
+                file.ucids = [x for x in file.ucids if x != cid]
+                if not file.ucids:
+                    self.q_handshake.put(file)
+
+                self.st_up = [file, cid]
+                file.up_b += sz
+                self.up_b += sz
+                self.up_br += sz
+                file.up_c += 1
+                self.up_c += 1
+                self.uploader_busy -= 1
+
+
+class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
+    pass
+
+
+def main():
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if not VT100:
+        os.system("rem")  # enables colors
+
+    # fmt: off
+    ap = app = argparse.ArgumentParser(formatter_class=APF, epilog="""
+NOTE:
+source file/folder selection uses rsync syntax, meaning that:
+  "foo" uploads the entire folder to URL/foo/
+  "foo/" uploads the CONTENTS of the folder into URL/
+""")
+
+    ap.add_argument("url", type=unicode, help="server url, including destination folder")
+    ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
+    ap.add_argument("-a", metavar="PASSWORD", help="password")
+    ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
+    ap = app.add_argument_group("performance tweaks")
+    ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
+    ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
+    ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
+    ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
+    ap = app.add_argument_group("tls")
+    ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
+    ap.add_argument("-td", action="store_true", help="disable certificate check")
+    # fmt: on
+
+    Ctl(app.parse_args())
+
+
+if __name__ == "__main__":
+    main()

bin/up2k.sh

@@ -8,7 +8,7 @@ set -e
 ##
 ## config
 
-datalen=$((2*1024*1024*1024))
+datalen=$((128*1024*1024))
 target=127.0.0.1
 posturl=/inc
 passwd=wark
@@ -37,10 +37,10 @@ gendata() {
 # pipe a chunk, get the base64 checksum
 gethash() {
     printf $(
-        sha512sum | cut -c-64 |
+        sha512sum | cut -c-66 |
         sed -r 's/ .*//;s/(..)/\\x\1/g'
     ) |
-    base64 -w0 | cut -c-43 |
+    base64 -w0 | cut -c-44 |
     tr '+/' '-_'
 }
 
@@ -123,7 +123,7 @@ printf '\033[36m'
 {
     {
         cat <<EOF
-POST $posturl/handshake.php HTTP/1.1
+POST $posturl/ HTTP/1.1
 Connection: Close
 Cookie: cppwd=$passwd
 Content-Type: text/plain;charset=UTF-8
@@ -145,14 +145,16 @@ printf '\033[0m\nwark: %s\n' $wark
 ##
 ## wait for signal to continue
 
-w8=/dev/shm/$salt.w8
-touch $w8
+true || {
+    w8=/dev/shm/$salt.w8
+    touch $w8
 
-echo "ready;  rm -f $w8"
+    echo "ready;  rm -f $w8"
 
-while [ -e $w8 ]; do
-    sleep 0.2
-done
+    while [ -e $w8 ]; do
+        sleep 0.2
+    done
+}
 
 
 ##
@@ -175,7 +177,7 @@ while [ $remains -gt 0 ]; do
     
     {
         cat <<EOF
-POST $posturl/chunkpit.php HTTP/1.1
+POST $posturl/ HTTP/1.1
 Connection: Keep-Alive
 Cookie: cppwd=$passwd
 Content-Type: application/octet-stream

contrib/README.md

@@ -9,6 +9,16 @@
 * assumes the webserver and copyparty is running on the same server/IP
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
+### [`sharex.sxcu`](sharex.sxcu)
+* sharex config file to upload screenshots and grab the URL
+* `RequestURL`: full URL to the target folder
+* `pw`: password (remove the `pw` line if anon-write)
+
+however if your copyparty is behind a reverse-proxy, you may want to use [`sharex-html.sxcu`](sharex-html.sxcu) instead:
+* `RequestURL`: full URL to the target folder
+* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
+* `pw`: password (remove `Parameters` if anon-write)
+
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
 * disables thumbnails and folder-type detection in windows explorer
 * makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
@@ -19,7 +29,9 @@
 
 # OS integration
 init-scripts to start copyparty as a service
-* [`systemd/copyparty.service`](systemd/copyparty.service)
+* [`systemd/copyparty.service`](systemd/copyparty.service) runs the sfx normally
+* [`rc/copyparty`](rc/copyparty) runs sfx normally on freebsd, create a `copyparty` user
+* [`systemd/prisonparty.service`](systemd/prisonparty.service) runs the sfx in a chroot
 * [`openrc/copyparty`](openrc/copyparty)
 
 # Reverse-proxy

contrib/cfssl.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 set -e
 
-# ca-name and server-name
+# ca-name and server-fqdn
 ca_name="$1"
-srv_name="$2"
+srv_fqdn="$2"
 
-[ -z "$srv_name" ] && {
+[ -z "$srv_fqdn" ] && {
 	echo "need arg 1: ca name"
-	echo "need arg 2: server name"
+	echo "need arg 2: server fqdn"
+	echo "optional arg 3: if set, write cert into copyparty cfg"
 	exit 1
 }
 
@@ -31,15 +32,15 @@ EOF
 gen_srv() {
 	(tee /dev/stderr <<EOF
 {"key": {"algo":"rsa", "size":4096},
-"names": [{"O":"$ca_name - $srv_name"}]}
+"names": [{"O":"$ca_name - $srv_fqdn"}]}
 EOF
 	)|
 	cfssl gencert -ca ca.pem -ca-key ca.key \
-		-profile=www -hostname="$srv_name.$ca_name" - |
-	cfssljson -bare "$srv_name"
+		-profile=www -hostname="$srv_fqdn" - |
+	cfssljson -bare "$srv_fqdn"
 
-	mv "$srv_name-key.pem" "$srv_name.key"
-	rm "$srv_name.csr"
+	mv "$srv_fqdn-key.pem" "$srv_fqdn.key"
+	rm "$srv_fqdn.csr"
 }
 
 
@@ -57,13 +58,13 @@ show() {
 	awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
 }
 show ca.pem
-show "$srv_name.pem"
+show "$srv_fqdn.pem"
 
 
 # write cert into copyparty config
 [ -z "$3" ] || {
 	mkdir -p ~/.config/copyparty
-	cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
+	cat "$srv_fqdn".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
 }
 
 

contrib/nginx/copyparty.conf

@@ -1,3 +1,16 @@
+# when running copyparty behind a reverse proxy,
+# the following arguments are recommended:
+#
+#   -nc 512         important, see next paragraph
+#   --http-only     lower latency on initial connection
+#   -i 127.0.0.1    only accept connections from nginx
+#
+# -nc must match or exceed the webserver's max number of concurrent clients;
+# nginx default is 512  (worker_processes 1, worker_connections 512)
+#
+# you may also consider adding -j0 for CPU-intensive configurations
+# (not that i can really think of any good examples)
+
 upstream cpp {
 	server 127.0.0.1:3923;
 	keepalive 120;

contrib/openrc/copyparty

@@ -8,11 +8,11 @@
 #
 # you may want to:
 #   change '/usr/bin/python' to another interpreter
-#   change '/mnt::a' to another location or permission-set
+#   change '/mnt::rw' to another location or permission-set
 
 name="$SVCNAME"
 command_background=true
 pidfile="/var/run/$SVCNAME.pid"
 
 command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
-command_args="-q -v /mnt::a"
+command_args="-q -v /mnt::rw"

contrib/rc/copyparty

@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# PROVIDE: copyparty
+# REQUIRE: networking
+# KEYWORD:
+
+. /etc/rc.subr
+
+name="copyparty"
+rcvar="copyparty_enable"
+copyparty_user="copyparty"
+copyparty_args="-e2dsa -v /storage:/storage:r" # change as you see fit
+copyparty_command="/usr/local/bin/python3.8 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
+pidfile="/var/run/copyparty/${name}.pid"
+command="/usr/sbin/daemon"
+command_args="-P ${pidfile} -r -f ${copyparty_command}"
+
+stop_postcmd="copyparty_shutdown"
+
+copyparty_shutdown()
+{
+        if [ -e "${pidfile}" ]; then
+                echo "Stopping supervising daemon."
+                kill -s TERM `cat ${pidfile}`
+        fi
+}
+
+load_rc_config $name
+: ${copyparty_enable:=no}
+
+run_rc_command "$1"

contrib/sharex-html.sxcu

@@ -0,0 +1,19 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty-html",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark"
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "RegexList": [
+    "bytes // <a href=\"/([^\"]+)\""
+  ],
+  "URL": "http://127.0.0.1:3923/$regex:1|1$"
+}

contrib/sharex.sxcu

@@ -0,0 +1,17 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark",
+    "j": null
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "URL": "$json:files[0].url$"
+}

contrib/systemd/copyparty.service

@@ -3,17 +3,37 @@
 #
 # installation:
 #   cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
+#   restorecon -vr /etc/systemd/system/copyparty.service
+#   firewall-cmd --permanent --add-port={80,443,3923}/tcp
+#   firewall-cmd --reload
 #
 # you may want to:
-#   change '/usr/bin/python' to another interpreter
-#   change '/mnt::a' to another location or permission-set
+#   change '/usr/bin/python3' to another interpreter
+#   change '/mnt::rw' to another location or permission-set
+#   remove '-p 80,443,3923' to only listen on port 3923
+#   add '-i 127.0.0.1' to only allow local connections
+#   add '--use-fpool' if uploading into nfs locations
+#
+# with `Type=notify`, copyparty will signal systemd when it is ready to
+#   accept connections; correctly delaying units depending on copyparty.
+#   But note that journalctl will get the timestamps wrong due to
+#   python disabling line-buffering, so messages are out-of-order:
+#   https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
+#
+# if you remove -q to enable logging, you may also want to remove the
+#   following line to enable buffering (slightly better performance):
+#   Environment=PYTHONUNBUFFERED=x
 
 [Unit]
 Description=copyparty file server
 
 [Service]
-ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+Type=notify
+SyslogIdentifier=copyparty
+Environment=PYTHONUNBUFFERED=x
+ExecReload=/bin/kill -s USR1 $MAINPID
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -p 80,443,3923 -v /mnt::rw
 
 [Install]
 WantedBy=multi-user.target

contrib/systemd/prisonparty.service

@@ -0,0 +1,27 @@
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# in a chroot, preventing accidental access elsewhere
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   1) put copyparty-sfx.py and prisonparty.sh in /usr/local/bin
+#   2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
+#
+# you may want to:
+#   change '/mnt::rw' to another location or permission-set
+#    (remember to change the '/mnt' chroot arg too)
+#
+# enable line-buffering for realtime logging (slight performance cost):
+#   inside the [Service] block, add the following line:
+#   Environment=PYTHONUNBUFFERED=x
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+SyslogIdentifier=prisonparty
+WorkingDirectory=/usr/local/bin
+ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
+  /usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
+
+[Install]
+WantedBy=multi-user.target

copyparty/__init__.py

@@ -2,12 +2,16 @@
 from __future__ import print_function, unicode_literals
 
 import platform
+import time
 import sys
 import os
 
 PY2 = sys.version_info[0] == 2
 if PY2:
     sys.dont_write_bytecode = True
+    unicode = unicode
+else:
+    unicode = str
 
 WINDOWS = False
 if platform.system() == "Windows":
@@ -16,21 +20,46 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 
+def get_unix_home():
+    try:
+        v = os.environ["XDG_CONFIG_HOME"]
+        if not v:
+            raise Exception()
+        ret = os.path.normpath(v)
+        os.listdir(ret)
+        return ret
+    except:
+        pass
+
+    try:
+        v = os.path.expanduser("~/.config")
+        if v.startswith("~"):
+            raise Exception()
+        ret = os.path.normpath(v)
+        os.listdir(ret)
+        return ret
+    except:
+        return "/tmp"
+
+
 class EnvParams(object):
     def __init__(self):
+        self.t0 = time.time()
         self.mod = os.path.dirname(os.path.realpath(__file__))
+        if self.mod.endswith("__init__"):
+            self.mod = os.path.dirname(self.mod)
+
         if sys.platform == "win32":
             self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
         elif sys.platform == "darwin":
             self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
         else:
-            self.cfg = os.path.normpath(
-                os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
-                + "/copyparty"
-            )
+            self.cfg = get_unix_home() + "/copyparty"
 
         self.cfg = self.cfg.replace("\\", "/")
         try:

copyparty/__main__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
@@ -20,10 +20,11 @@ import threading
 import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, VT100, PY2
+from .__init__ import E, WINDOWS, ANYWIN, VT100, PY2, unicode
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab, IMPLICATIONS
+from .util import py_desc, align_tab, IMPLICATIONS, ansi_re
+from .authsrv import re_vol
 
 HAVE_SSL = True
 try:
@@ -31,6 +32,8 @@ try:
 except:
     HAVE_SSL = False
 
+printed = ""
+
 
 class RiceFormatter(argparse.HelpFormatter):
     def _get_help_string(self, action):
@@ -61,8 +64,19 @@ class Dodge11874(RiceFormatter):
         super(Dodge11874, self).__init__(*args, **kwargs)
 
 
+def lprint(*a, **ka):
+    global printed
+
+    txt = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    printed += txt
+    if not VT100:
+        txt = ansi_re.sub("", txt)
+
+    print(txt, **ka)
+
+
 def warn(msg):
-    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+    lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
 
 def ensure_locale():
@@ -73,7 +87,7 @@ def ensure_locale():
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            print("Locale:", x)
+            lprint("Locale:", x)
             break
         except:
             continue
@@ -90,11 +104,11 @@ def ensure_cert():
     cert_insec = os.path.join(E.mod, "res/insecure.pem")
     cert_cfg = os.path.join(E.cfg, "cert.pem")
     if not os.path.exists(cert_cfg):
-        shutil.copy2(cert_insec, cert_cfg)
+        shutil.copy(cert_insec, cert_cfg)
 
     try:
         if filecmp.cmp(cert_cfg, cert_insec):
-            print(
+            lprint(
                 "\033[33m  using default TLS certificate; https will be insecure."
                 + "\033[36m\n  certificate location: {}\033[0m\n".format(cert_cfg)
             )
@@ -123,7 +137,7 @@ def configure_ssl_ver(al):
     if "help" in sslver:
         avail = [terse_sslver(x[6:]) for x in flags]
         avail = " ".join(sorted(avail) + ["all"])
-        print("\navailable ssl/tls versions:\n  " + avail)
+        lprint("\navailable ssl/tls versions:\n  " + avail)
         sys.exit(0)
 
     al.ssl_flags_en = 0
@@ -143,7 +157,7 @@ def configure_ssl_ver(al):
 
     for k in ["ssl_flags_en", "ssl_flags_de"]:
         num = getattr(al, k)
-        print("{}: {:8x} ({})".format(k, num, num))
+        lprint("{}: {:8x} ({})".format(k, num, num))
 
     # think i need that beer now
 
@@ -160,18 +174,44 @@ def configure_ssl_ciphers(al):
         try:
             ctx.set_ciphers(al.ciphers)
         except:
-            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+            lprint("\n\033[1;31mfailed to set ciphers\033[0m\n")
 
     if not hasattr(ctx, "get_ciphers"):
-        print("cannot read cipher list: openssl or python too old")
+        lprint("cannot read cipher list: openssl or python too old")
     else:
         ciphers = [x["description"] for x in ctx.get_ciphers()]
-        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+        lprint("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
 
     if is_help:
         sys.exit(0)
 
 
+def args_from_cfg(cfg_path):
+    ret = []
+    skip = False
+    with open(cfg_path, "rb") as f:
+        for ln in [x.decode("utf-8").strip() for x in f]:
+            if not ln:
+                skip = False
+                continue
+
+            if ln.startswith("#"):
+                continue
+
+            if not ln.startswith("-"):
+                continue
+
+            if skip:
+                continue
+
+            try:
+                ret.extend(ln.split(" ", 1))
+            except:
+                ret.append(ln)
+
+    return ret
+
+
 def sighandler(sig=None, frame=None):
     msg = [""] * 5
     for th in threading.enumerate():
@@ -187,27 +227,43 @@ def run_argparse(argv, formatter):
         formatter_class=formatter,
         prog="copyparty",
         description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
-        epilog=dedent(
-            """
+    )
+
+    try:
+        fk_salt = unicode(os.path.getmtime(os.path.join(E.cfg, "cert.pem")))
+    except:
+        fk_salt = "hunter2"
+
+    cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+
+    sects = [
+        [
+            "accounts",
+            "accounts and volumes",
+            dedent(
+                """
             -a takes username:password,
-            -v takes src:dst:permset:permset:cflag:cflag:...
-               where "permset" is accesslevel followed by username (no separator)
-               and "cflag" is config flags to set on this volume
+            -v takes src:dst:perm1:perm2:permN:volflag1:volflag2:volflagN:...
+               where "perm" is "permissions,username1,username2,..."
+               and "volflag" is config flags to set on this volume
             
-            list of cflags:
-              "cnodupe" rejects existing files (instead of symlinking them)
-              "ce2d" sets -e2d (all -e2* args can be set using ce2* cflags)
-              "cd2t" disables metadata collection, overrides -e2t*
-              "cd2d" disables all database stuff, overrides -e2*
+            list of permissions:
+              "r" (read):   list folder contents, download files
+              "w" (write):  upload files; need "r" to see the uploads
+              "m" (move):   move files and folders; need "w" at destination
+              "d" (delete): permanently delete files and folders
+              "g" (get):    download files, but cannot see folder contents
+
+            too many volflags to list here, see the other sections
 
             example:\033[35m
-              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
+              -a ed:hunter2 -v .::r:rw,ed -v ../inc:dump:w:rw,ed:c,nodupe  \033[36m
               mount current directory at "/" with
                * r (read-only) for everyone
-               * a (read+write) for ed
+               * rw (read+write) for ed
               mount ../inc at "/dump" with
                * w (write-only) for everyone
-               * a (read+write) for ed
+               * rw (read+write) for ed
                * reject duplicate files  \033[0m
             
             if no accounts or volumes are configured,
@@ -215,67 +271,234 @@ def run_argparse(argv, formatter):
 
             consider the config file for more flexible account/volume management,
             including dynamic reload at runtime (and being more readable w)
-
-            values for --urlform:
-              "stash" dumps the data to file and returns length + checksum
-              "save,get" dumps to file and returns the page like a GET
-              "print,get" prints the data in the log and returns GET
-              (leave out the ",get" to return an error instead)
-
-            --ciphers help = available ssl/tls ciphers,
-            --ssl-ver help = available ssl/tls versions,
-              default is what python considers safe, usually >= TLS1
             """
-        ),
-    )
+            ),
+        ],
+        [
+            "flags",
+            "list of volflags",
+            dedent(
+                """
+            volflags are appended to volume definitions, for example,
+            to create a write-only volume with the \033[33mnodupe\033[0m and \033[32mnosub\033[0m flags:
+              \033[35m-v /mnt/inc:/inc:w\033[33m:c,nodupe\033[32m:c,nosub
+
+            \033[0muploads, general:
+              \033[36mnodupe\033[35m rejects existing files (instead of symlinking them)
+              \033[36mnosub\033[35m forces all uploads into the top folder of the vfs
+              \033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz)
+              \033[36mpk\033[35m forces server-side compression, optional arg: xz,9
+            
+            \033[0mupload rules:
+              \033[36mmaxn=250,600\033[35m max 250 uploads over 15min
+              \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
+              \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
+            
+            \033[0mupload rotation:
+            (moves all uploads into the specified folder structure)
+              \033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each
+              \033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing
+              \033[36mlifetime=3600\033[35m uploads are deleted after 1 hour
+            
+            \033[0mdatabase, general:
+              \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags)
+              \033[36md2t\033[35m disables metadata collection, overrides -e2t*
+              \033[36md2d\033[35m disables all database stuff, overrides -e2*
+              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
+              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
+              \033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location
+              \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
+            
+            \033[0mdatabase, audio tags:
+            "mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...
+              \033[36mmtp=.bpm=f,audio-bpm.py\033[35m uses the "audio-bpm.py" program to
+                generate ".bpm" tags from uploads (f = overwrite tags)
+              \033[36mmtp=ahash,vhash=media-hash.py\033[35m collects two tags at once
+            
+            \033[0mothers:
+              \033[36mfk=8\033[35m generates per-file accesskeys,
+                which will then be required at the "g" permission
+            \033[0m"""
+            ),
+        ],
+        [
+            "urlform",
+            "",
+            dedent(
+                """
+            values for --urlform:
+              \033[36mstash\033[35m dumps the data to file and returns length + checksum
+              \033[36msave,get\033[35m dumps to file and returns the page like a GET
+              \033[36mprint,get\033[35m prints the data in the log and returns GET
+              (leave out the ",get" to return an error instead)
+            """
+            ),
+        ],
+        [
+            "ls",
+            "volume inspection",
+            dedent(
+                """
+            \033[35m--ls USR,VOL,FLAGS
+              \033[36mUSR\033[0m is a user to browse as; * is anonymous, ** is all users
+              \033[36mVOL\033[0m is a single volume to scan, default is * (all vols)
+              \033[36mFLAG\033[0m is flags;
+                \033[36mv\033[0m in addition to realpaths, print usernames and vpaths
+                \033[36mln\033[0m only prints symlinks leaving the volume mountpoint
+                \033[36mp\033[0m exits 1 if any such symlinks are found
+                \033[36mr\033[0m resumes startup after the listing
+            examples:
+              --ls '**'          # list all files which are possible to read
+              --ls '**,*,ln'     # check for dangerous symlinks
+              --ls '**,*,ln,p,r' # check, then start normally if safe
+            """
+            ),
+        ],
+    ]
+
     # fmt: off
-    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
-    ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
-    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
-    ap.add_argument("-q", action="store_true", help="quiet")
-    ap.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
-    ap.add_argument("-ed", action="store_true", help="enable ?dots")
-    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
-    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
-    ap.add_argument("-nih", action="store_true", help="no info hostname")
-    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
-    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
-    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
-    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
-    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+    u = unicode
+    ap2 = ap.add_argument_group('general options')
+    ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
+    ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores, 0=all")
+    ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, USER:PASS; example [ed:wark")
+    ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
+    ap2.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
+    ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
 
-    ap2 = ap.add_argument_group('database options')
-    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
-    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
-    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
-    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
-    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
-    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
-    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
-    ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
-    ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
-    ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
-        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
-    ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
-    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
+    ap2 = ap.add_argument_group('upload options')
+    ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
+    ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
+    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without")
+    ap2.add_argument("--no-symlink", action="store_true", help="duplicate file contents instead")
 
+    ap2 = ap.add_argument_group('network options')
+    ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
+    
     ap2 = ap.add_argument_group('SSL/TLS options')
     ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
     ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="ssl/tls versions to allow")
-    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [help] shows available ciphers")
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
-    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+    ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets")
+
+    ap2 = ap.add_argument_group('opt-outs')
+    ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
+    ap2.add_argument("--no-del", action="store_true", help="disable delete operations")
+    ap2.add_argument("--no-mv", action="store_true", help="disable move/rename operations")
+    ap2.add_argument("-nih", action="store_true", help="no info hostname")
+    ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+    ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)")
+
+    ap2 = ap.add_argument_group('safety options')
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
+    ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt")
+    ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt")
+    ap2.add_argument("--no-dot-mv", action="store_true", help="disallow moving dotfiles; makes it impossible to move folders containing dotfiles")
+    ap2.add_argument("--no-dot-ren", action="store_true", help="disallow renaming dotfiles; makes it impossible to make something a dotfile")
+    ap2.add_argument("--no-logues", action="store_true", help="disable rendering .prologue/.epilogue.html into directory listings")
+    ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
+    ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
+
+    ap2 = ap.add_argument_group('yolo options')
+    ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
+    ap2.add_argument("--ign-ebind-all", action="store_true", help="continue running even if it's impossible to receive connections at all")
+
+    ap2 = ap.add_argument_group('logging options')
+    ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
+    ap2.add_argument("--no-voldump", action="store_true", help="do not list volumes and permissions on startup")
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--log-htp", action="store_true", help="print http-server threadpool scaling")
+    ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
+
+    ap2 = ap.add_argument_group('admin panel options')
+    ap2.add_argument("--no-reload", action="store_true", help="disable ?reload=cfg (reload users/volumes/volflags from config file)")
+    ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
+    ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
+
+    ap2 = ap.add_argument_group('thumbnail options')
+    ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
+    ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms)")
+    ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
+    ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
+    ap2.add_argument("--th-mt", metavar="CORES", type=int, default=cores, help="num cpu cores to use for generating thumbnails")
+    ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
+    ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
+    ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
+    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
+    ap2.add_argument("--th-ff-swr", action="store_true", help="use swresample instead of soxr for audio thumbs")
+    ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
+    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval; 0=disabled")
+    ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
+    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
+
+    ap2 = ap.add_argument_group('transcoding options')
+    ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding")
+    ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete transcode output after SEC seconds")
+
+    ap2 = ap.add_argument_group('general db options')
+    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
+    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
+    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
+    ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
+    ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
+    ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
+    ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
     
-    return ap.parse_args(args=argv[1:])
+    ap2 = ap.add_argument_group('metadata db options')
+    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
+    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
+    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
+    ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
+    ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=cores, help="num cpu cores to use for tag scanning")
+    ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
+    ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,res,.fps,ahash,vhash")
+    ap2.add_argument("-mth", metavar="M,M,M", type=u, help="tags to hide by default (comma-sep.)",
+        default=".vq,.aq,vc,ac,res,.fps")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", type=u, action="append", help="read tag M using bin")
+
+    ap2 = ap.add_argument_group('ui options')
+    ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include")
+    ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
+    ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
+
+    ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
+    ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
+    ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
+    ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
+    ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
     # fmt: on
 
+    ap2 = ap.add_argument_group("help sections")
+    for k, h, _ in sects:
+        ap2.add_argument("--help-" + k, action="store_true", help=h)
+
+    ret = ap.parse_args(args=argv[1:])
+    for k, h, t in sects:
+        k2 = "help_" + k.replace("-", "_")
+        if vars(ret)[k2]:
+            lprint("# {} help page".format(k))
+            lprint(t + "\033[0m")
+            sys.exit(0)
+
+    return ret
+
 
 def main(argv=None):
     time.strptime("19970815", "%Y%m%d")  # python#7980
@@ -288,13 +511,18 @@ def main(argv=None):
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
     if HAVE_SSL:
         ensure_cert()
 
-    deprecated = [["-e2s", "-e2ds"]]
+    for k, v in zip(argv, argv[1:]):
+        if k == "-c":
+            supp = args_from_cfg(v)
+            argv.extend(supp)
+
+    deprecated = []
     for dk, nk in deprecated:
         try:
             idx = argv.index(dk)
@@ -302,15 +530,57 @@ def main(argv=None):
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
+        lprint(msg.format(dk, nk))
         argv[idx] = nk
         time.sleep(2)
 
+    try:
+        if len(argv) == 1 and (ANYWIN or not os.geteuid()):
+            argv.extend(["-p80,443,3923", "--ign-ebind"])
+    except:
+        pass
+
     try:
         al = run_argparse(argv, RiceFormatter)
     except AssertionError:
         al = run_argparse(argv, Dodge11874)
 
+    nstrs = []
+    anymod = False
+    for ostr in al.v or []:
+        m = re_vol.match(ostr)
+        if not m:
+            # not our problem
+            nstrs.append(ostr)
+            continue
+
+        src, dst, perms = m.groups()
+        na = [src, dst]
+        mod = False
+        for opt in perms.split(":"):
+            if re.match("c[^,]", opt):
+                mod = True
+                na.append("c," + opt[1:])
+            elif re.sub("^[rwmdg]*", "", opt) and "," not in opt:
+                mod = True
+                perm = opt[0]
+                if perm == "a":
+                    perm = "rw"
+                na.append(perm + "," + opt[1:])
+            else:
+                na.append(opt)
+
+        nstr = ":".join(na)
+        nstrs.append(nstr if mod else ostr)
+        if mod:
+            msg = "\033[1;31mWARNING:\033[0;1m\n  -v {} \033[0;33mwas replaced with\033[0;1m\n  -v {} \n\033[0m"
+            lprint(msg.format(ostr, nstr))
+            anymod = True
+
+    if anymod:
+        al.v = nstrs
+        time.sleep(2)
+
     # propagate implications
     for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):
@@ -341,9 +611,12 @@ def main(argv=None):
             + "  (if you crash with codec errors then that is why)"
         )
 
+    if sys.version_info < (3, 6):
+        al.no_scandir = True
+
     # signal.signal(signal.SIGINT, sighandler)
 
-    SvcHub(al).run()
+    SvcHub(al, argv, printed).run()
 
 
 if __name__ == "__main__":

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 10, 12)
-CODENAME = "zip it"
-BUILD_DT = (2021, 4, 19)
+VERSION = (1, 1, 0)
+CODENAME = "opus"
+BUILD_DT = (2021, 11, 6)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/bos/bos.py

@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+from ..util import fsenc, fsdec
+from . import path
+
+
+# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
+# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
+
+
+def chmod(p, mode):
+    return os.chmod(fsenc(p), mode)
+
+
+def listdir(p="."):
+    return [fsdec(x) for x in os.listdir(fsenc(p))]
+
+
+def lstat(p):
+    return os.lstat(fsenc(p))
+
+
+def makedirs(name, mode=0o755, exist_ok=True):
+    bname = fsenc(name)
+    try:
+        os.makedirs(bname, mode)
+    except:
+        if not exist_ok or not os.path.isdir(bname):
+            raise
+
+
+def mkdir(p, mode=0o755):
+    return os.mkdir(fsenc(p), mode)
+
+
+def rename(src, dst):
+    return os.rename(fsenc(src), fsenc(dst))
+
+
+def replace(src, dst):
+    return os.replace(fsenc(src), fsenc(dst))
+
+
+def rmdir(p):
+    return os.rmdir(fsenc(p))
+
+
+def stat(p):
+    return os.stat(fsenc(p))
+
+
+def unlink(p):
+    return os.unlink(fsenc(p))
+
+
+def utime(p, times=None):
+    return os.utime(fsenc(p), times)

copyparty/bos/path.py

@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+from ..util import fsenc, fsdec
+
+
+def abspath(p):
+    return fsdec(os.path.abspath(fsenc(p)))
+
+
+def exists(p):
+    return os.path.exists(fsenc(p))
+
+
+def getmtime(p):
+    return os.path.getmtime(fsenc(p))
+
+
+def getsize(p):
+    return os.path.getsize(fsenc(p))
+
+
+def isfile(p):
+    return os.path.isfile(fsenc(p))
+
+
+def isdir(p):
+    return os.path.isdir(fsenc(p))
+
+
+def islink(p):
+    return os.path.islink(fsenc(p))
+
+
+def realpath(p):
+    return fsdec(os.path.realpath(fsenc(p)))

copyparty/broker_mp.py

@@ -4,17 +4,11 @@ from __future__ import print_function, unicode_literals
 import time
 import threading
 
-from .__init__ import PY2, WINDOWS, VT100
 from .broker_util import try_exec
 from .broker_mpw import MpWorker
 from .util import mp
 
 
-if PY2 and not WINDOWS:
-    from multiprocessing.reduction import ForkingPickler
-    from StringIO import StringIO as MemesIO  # pylint: disable=import-error
-
-
 class BrokerMp(object):
     """external api; manages MpWorkers"""
 
@@ -28,38 +22,33 @@ class BrokerMp(object):
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
 
-        cores = self.args.j
-        if not cores:
-            cores = mp.cpu_count()
-
-        self.log("broker", "booting {} subprocesses".format(cores))
-        for n in range(cores):
+        self.num_workers = self.args.j or mp.cpu_count()
+        self.log("broker", "booting {} subprocesses".format(self.num_workers))
+        for n in range(1, self.num_workers + 1):
             q_pend = mp.Queue(1)
             q_yield = mp.Queue(64)
 
             proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
             proc.q_pend = q_pend
             proc.q_yield = q_yield
-            proc.nid = n
             proc.clients = {}
-            proc.workload = 0
 
-            thr = threading.Thread(target=self.collector, args=(proc,))
+            thr = threading.Thread(
+                target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
+            )
             thr.daemon = True
             thr.start()
 
             self.procs.append(proc)
             proc.start()
 
-        if not self.args.q:
-            thr = threading.Thread(target=self.debug_load_balancer)
-            thr.daemon = True
-            thr.start()
-
     def shutdown(self):
         self.log("broker", "shutting down")
-        for proc in self.procs:
-            thr = threading.Thread(target=proc.q_pend.put([0, "shutdown", []]))
+        for n, proc in enumerate(self.procs):
+            thr = threading.Thread(
+                target=proc.q_pend.put([0, "shutdown", []]),
+                name="mp-shutdown-{}-{}".format(n, len(self.procs)),
+            )
             thr.start()
 
         with self.mutex:
@@ -73,6 +62,11 @@ class BrokerMp(object):
 
             procs.pop()
 
+    def reload(self):
+        self.log("broker", "reloading")
+        for _, proc in enumerate(self.procs):
+            proc.q_pend.put([0, "reload", []])
+
     def collector(self, proc):
         """receive message from hub in other process"""
         while True:
@@ -82,20 +76,6 @@ class BrokerMp(object):
             if dest == "log":
                 self.log(*args)
 
-            elif dest == "workload":
-                with self.mutex:
-                    proc.workload = args[0]
-
-            elif dest == "httpdrop":
-                addr = args[0]
-
-                with self.mutex:
-                    del proc.clients[addr]
-                    if not proc.clients:
-                        proc.workload = 0
-
-                self.hub.tcpsrv.num_clients.add(-1)
-
             elif dest == "retq":
                 # response from previous ipc call
                 with self.retpend_mutex:
@@ -121,38 +101,12 @@ class BrokerMp(object):
         returns a Queue object which eventually contains the response if want_retval
         (not-impl here since nothing uses it yet)
         """
-        if dest == "httpconn":
-            sck, addr = args
-            sck2 = sck
-            if PY2:
-                buf = MemesIO()
-                ForkingPickler(buf).dump(sck)
-                sck2 = buf.getvalue()
+        if dest == "listen":
+            for p in self.procs:
+                p.q_pend.put([0, dest, [args[0], len(self.procs)]])
 
-            proc = sorted(self.procs, key=lambda x: x.workload)[0]
-            proc.q_pend.put([0, dest, [sck2, addr]])
-
-            with self.mutex:
-                proc.clients[addr] = 50
-                proc.workload += 50
+        elif dest == "cb_httpsrv_up":
+            self.hub.cb_httpsrv_up()
 
         else:
             raise Exception("what is " + str(dest))
-
-    def debug_load_balancer(self):
-        fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
-        if not VT100:
-            fmt = "({}{:4})"
-
-        last = ""
-        while self.procs:
-            msg = ""
-            for proc in self.procs:
-                msg += fmt.format(len(proc.clients), proc.workload)
-
-            if msg != last:
-                last = msg
-                with self.hub.log_mutex:
-                    print(msg)
-
-            time.sleep(0.1)

copyparty/broker_mpw.py

@@ -2,17 +2,13 @@
 from __future__ import print_function, unicode_literals
 
 import sys
-import time
 import signal
 import threading
 
-from .__init__ import PY2, WINDOWS
 from .broker_util import ExceptionalQueue
 from .httpsrv import HttpSrv
 from .util import FAKE_MP
-
-if PY2 and not WINDOWS:
-    import pickle  # nosec
+from copyparty.authsrv import AuthSrv
 
 
 class MpWorker(object):
@@ -24,66 +20,62 @@ class MpWorker(object):
         self.args = args
         self.n = n
 
+        self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
+
         self.retpend = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
-        self.workload_thr_active = False
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
         if not FAKE_MP:
-            signal.signal(signal.SIGINT, self.signal_handler)
+            for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1]:
+                signal.signal(sig, self.signal_handler)
+
+        # starting to look like a good idea
+        self.asrv = AuthSrv(args, None, False)
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
-        self.httpsrv.disconnect_func = self.httpdrop
+        self.httpsrv = HttpSrv(self, n)
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
-        thr = threading.Thread(target=self.main)
+        thr = threading.Thread(target=self.main, name="mpw-main")
         thr.daemon = True
         thr.start()
         thr.join()
 
-    def signal_handler(self, signal, frame):
+    def signal_handler(self, sig, frame):
         # print('k')
         pass
 
-    def log(self, src, msg, c=0):
+    def _log_enabled(self, src, msg, c=0):
         self.q_yield.put([0, "log", [src, msg, c]])
 
+    def _log_disabled(self, src, msg, c=0):
+        pass
+
     def logw(self, msg, c=0):
         self.log("mp{}".format(self.n), msg, c)
 
-    def httpdrop(self, addr):
-        self.q_yield.put([0, "httpdrop", [addr]])
-
     def main(self):
         while True:
             retq_id, dest, args = self.q_pend.get()
 
             # self.logw("work: [{}]".format(d[0]))
             if dest == "shutdown":
+                self.httpsrv.shutdown()
                 self.logw("ok bye")
                 sys.exit(0)
                 return
 
-            elif dest == "httpconn":
-                sck, addr = args
-                if PY2:
-                    sck = pickle.loads(sck)  # nosec
+            elif dest == "reload":
+                self.logw("mpw.asrv reloading")
+                self.asrv.reload()
+                self.logw("mpw.asrv reloaded")
 
-                if self.args.log_conn:
-                    self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
-                
-                self.httpsrv.accept(sck, addr)
-
-                with self.mutex:
-                    if not self.workload_thr_active:
-                        self.workload_thr_alive = True
-                        thr = threading.Thread(target=self.thr_workload)
-                        thr.daemon = True
-                        thr.start()
+            elif dest == "listen":
+                self.httpsrv.listen(args[0], args[1])
 
             elif dest == "retq":
                 # response from previous ipc call
@@ -107,16 +99,3 @@ class MpWorker(object):
 
         self.q_yield.put([retq_id, dest, args])
         return retq
-
-    def thr_workload(self):
-        """announce workloads to MpSrv (the mp controller / loadbalancer)"""
-        # avoid locking in extract_filedata by tracking difference here
-        while True:
-            time.sleep(0.2)
-            with self.mutex:
-                if self.httpsrv.num_clients() == 0:
-                    # no clients rn, termiante thread
-                    self.workload_thr_alive = False
-                    return
-
-            self.q_yield.put([0, "workload", [self.httpsrv.workload]])

copyparty/broker_thr.py

@@ -14,24 +14,25 @@ class BrokerThr(object):
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
+        self.asrv = hub.asrv
 
         self.mutex = threading.Lock()
+        self.num_workers = 1
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
-        self.httpsrv.disconnect_func = self.httpdrop
+        self.httpsrv = HttpSrv(self, None)
+        self.reload = self.noop
 
     def shutdown(self):
         # self.log("broker", "shutting down")
+        self.httpsrv.shutdown()
+
+    def noop(self):
         pass
 
     def put(self, want_retval, dest, *args):
-        if dest == "httpconn":
-            sck, addr = args
-            if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
-
-            self.httpsrv.accept(sck, addr)
+        if dest == "listen":
+            self.httpsrv.listen(args[0], 1)
 
         else:
             # new ipc invoking managed service in hub
@@ -48,6 +49,3 @@ class BrokerThr(object):
             retq = ExceptionalQueue(1)
             retq.put(rv)
             return retq
-
-    def httpdrop(self, addr):
-        self.hub.tcpsrv.num_clients.add(-1)

copyparty/httpconn.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
-import sys
 import time
 import socket
 
@@ -16,6 +16,9 @@ from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
 from .u2idx import U2idx
+from .th_cli import ThumbCli
+from .th_srv import HAVE_PIL
+from .ico import Ico
 
 
 class HttpConn(object):
@@ -29,17 +32,33 @@ class HttpConn(object):
         self.addr = addr
         self.hsrv = hsrv
 
+        self.mutex = hsrv.mutex
         self.args = hsrv.args
-        self.auth = hsrv.auth
+        self.asrv = hsrv.asrv
         self.cert_path = hsrv.cert_path
+        self.u2fh = hsrv.u2fh
+
+        enth = HAVE_PIL and not self.args.no_thumb
+        self.thumbcli = ThumbCli(hsrv) if enth else None
+        self.ico = Ico(self.args)
 
         self.t0 = time.time()
+        self.stopping = False
+        self.nreq = 0
         self.nbyte = 0
-        self.workload = 0
         self.u2idx = None
         self.log_func = hsrv.log
+        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
         self.set_rproxy()
 
+    def shutdown(self):
+        self.stopping = True
+        try:
+            self.s.shutdown(socket.SHUT_RDWR)
+            self.s.close()
+        except:
+            pass
+
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -61,7 +80,7 @@ class HttpConn(object):
 
     def get_u2idx(self):
         if not self.u2idx:
-            self.u2idx = U2idx(self.args, self.log_func)
+            self.u2idx = U2idx(self)
 
         return self.u2idx
 
@@ -153,7 +172,7 @@ class HttpConn(object):
                     self.log("client rejected our certificate (nice)")
 
                 elif "ALERT_CERTIFICATE_UNKNOWN" in em:
-                    # chrome-android keeps doing this
+                    # android-chrome keeps doing this
                     pass
 
                 else:
@@ -164,7 +183,8 @@ class HttpConn(object):
         if not self.sr:
             self.sr = Unrecv(self.s)
 
-        while True:
+        while not self.stopping:
+            self.nreq += 1
             cli = HttpCli(self)
             if not cli.run():
                 return

copyparty/httpsrv.py

@@ -4,6 +4,8 @@ from __future__ import print_function, unicode_literals
 import os
 import sys
 import time
+import math
+import base64
 import socket
 import threading
 
@@ -24,9 +26,15 @@ except ImportError:
     )
     sys.exit(1)
 
-from .__init__ import E, MACOS
+from .__init__ import E, PY2, MACOS
+from .util import FHC, spack, min_ex, start_stackmon, start_log_thrs
+from .bos import bos
 from .httpconn import HttpConn
-from .authsrv import AuthSrv
+
+if PY2:
+    import Queue as queue
+else:
+    import queue
 
 
 class HttpSrv(object):
@@ -35,18 +43,32 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker):
+    def __init__(self, broker, nid):
         self.broker = broker
+        self.nid = nid
         self.args = broker.args
         self.log = broker.log
+        self.asrv = broker.asrv
 
-        self.disconnect_func = None
+        nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
+
+        self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
+        self.stopping = False
 
-        self.clients = {}
-        self.workload = 0
-        self.workload_thr_alive = False
-        self.auth = AuthSrv(self.args, self.log)
+        self.tp_nthr = 0  # actual
+        self.tp_ncli = 0  # fading
+        self.tp_time = None  # latest worker collect
+        self.tp_q = None if self.args.no_htp else queue.LifoQueue()
+        self.t_periodic = None
+
+        self.u2fh = FHC()
+        self.srvs = []
+        self.ncli = 0  # exact
+        self.clients = {}  # laggy
+        self.nclimax = 0
+        self.cb_ts = 0
+        self.cb_v = 0
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
@@ -54,28 +76,196 @@ class HttpSrv(object):
             x: env.get_template(x + ".html")
             for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
         }
+        self.prism = os.path.exists(os.path.join(E.mod, "web", "deps", "prism.js.gz"))
 
         cert_path = os.path.join(E.cfg, "cert.pem")
-        if os.path.exists(cert_path):
+        if bos.path.exists(cert_path):
             self.cert_path = cert_path
         else:
             self.cert_path = None
 
+        if self.tp_q:
+            self.start_threads(4)
+
+        if nid:
+            if self.args.stackmon:
+                start_stackmon(self.args.stackmon, nid)
+
+            if self.args.log_thrs:
+                start_log_thrs(self.log, self.args.log_thrs, nid)
+
+    def start_threads(self, n):
+        self.tp_nthr += n
+        if self.args.log_htp:
+            self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
+
+        for _ in range(n):
+            thr = threading.Thread(
+                target=self.thr_poolw,
+                name=self.name + "-poolw",
+            )
+            thr.daemon = True
+            thr.start()
+
+    def stop_threads(self, n):
+        self.tp_nthr -= n
+        if self.args.log_htp:
+            self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
+
+        for _ in range(n):
+            self.tp_q.put(None)
+
+    def periodic(self):
+        while True:
+            time.sleep(2 if self.tp_ncli or self.ncli else 10)
+            with self.mutex:
+                self.u2fh.clean()
+                if self.tp_q:
+                    self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
+                    if self.tp_nthr > self.tp_ncli + 8:
+                        self.stop_threads(4)
+
+                if not self.ncli and not self.u2fh.cache and self.tp_nthr <= 8:
+                    self.t_periodic = None
+                    return
+
+    def listen(self, sck, nlisteners):
+        ip, port = sck.getsockname()
+        self.srvs.append(sck)
+        self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
+        t = threading.Thread(
+            target=self.thr_listen,
+            args=(sck,),
+            name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
+        )
+        t.daemon = True
+        t.start()
+
+    def thr_listen(self, srv_sck):
+        """listens on a shared tcp server"""
+        ip, port = srv_sck.getsockname()
+        fno = srv_sck.fileno()
+        msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
+        self.log(self.name, msg)
+
+        def fun():
+            self.broker.put(False, "cb_httpsrv_up")
+
+        threading.Thread(target=fun).start()
+
+        while not self.stopping:
+            if self.args.log_conn:
+                self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
+
+            if self.ncli >= self.nclimax:
+                self.log(self.name, "at connection limit; waiting", 3)
+                while self.ncli >= self.nclimax:
+                    time.sleep(0.1)
+
+            if self.args.log_conn:
+                self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
+
+            try:
+                sck, addr = srv_sck.accept()
+            except (OSError, socket.error) as ex:
+                self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
+                time.sleep(0.02)
+                continue
+
+            if self.args.log_conn:
+                m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                    "-" * 3, ip, port % 8, port
+                )
+                self.log("%s %s" % addr, m, c="1;30")
+
+            self.accept(sck, addr)
+
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        if self.args.log_conn:
-            self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+        now = time.time()
 
-        thr = threading.Thread(target=self.thr_client, args=(sck, addr))
+        if now - (self.tp_time or now) > 300:
+            m = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
+            self.log(self.name, m.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
+            self.tp_time = None
+            self.tp_q = None
+
+        with self.mutex:
+            self.ncli += 1
+            if not self.t_periodic:
+                name = "hsrv-pt"
+                if self.nid:
+                    name += "-{}".format(self.nid)
+
+                t = threading.Thread(target=self.periodic, name=name)
+                self.t_periodic = t
+                t.daemon = True
+                t.start()
+
+            if self.tp_q:
+                self.tp_time = self.tp_time or now
+                self.tp_ncli = max(self.tp_ncli, self.ncli)
+                if self.tp_nthr < self.ncli + 4:
+                    self.start_threads(8)
+
+                self.tp_q.put((sck, addr))
+                return
+
+        if not self.args.no_htp:
+            m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
+            self.log(self.name, m, 1)
+
+        thr = threading.Thread(
+            target=self.thr_client,
+            args=(sck, addr),
+            name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+        )
         thr.daemon = True
         thr.start()
 
-    def num_clients(self):
-        with self.mutex:
-            return len(self.clients)
+    def thr_poolw(self):
+        while True:
+            task = self.tp_q.get()
+            if not task:
+                break
+
+            with self.mutex:
+                self.tp_time = None
+
+            try:
+                sck, addr = task
+                me = threading.current_thread()
+                me.name = "httpconn-{}-{}".format(
+                    addr[0].split(".", 2)[-1][-6:], addr[1]
+                )
+                self.thr_client(sck, addr)
+                me.name = self.name + "-poolw"
+            except:
+                self.log(self.name, "thr_client: " + min_ex(), 3)
 
     def shutdown(self):
-        self.log("ok bye")
+        self.stopping = True
+        for srv in self.srvs:
+            try:
+                srv.close()
+            except:
+                pass
+
+        clients = list(self.clients.keys())
+        for cli in clients:
+            try:
+                cli.shutdown()
+            except:
+                pass
+
+        if self.tp_q:
+            self.stop_threads(self.tp_nthr)
+            for _ in range(10):
+                time.sleep(0.05)
+                if self.tp_q.empty():
+                    break
+
+        self.log(self.name, "ok bye")
 
     def thr_client(self, sck, addr):
         """thread managing one tcp client"""
@@ -84,70 +274,69 @@ class HttpSrv(object):
         cli = HttpConn(sck, addr, self)
         with self.mutex:
             self.clients[cli] = 0
-            self.workload += 50
-
-            if not self.workload_thr_alive:
-                self.workload_thr_alive = True
-                thr = threading.Thread(target=self.thr_workload)
-                thr.daemon = True
-                thr.start()
 
+        fno = sck.fileno()
         try:
             if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
 
             cli.run()
 
+        except (OSError, socket.error) as ex:
+            if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
+                self.log(
+                    "%s %s" % addr,
+                    "run({}): {}".format(fno, ex),
+                    c=6,
+                )
+
         finally:
+            sck = cli.s
             if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
 
             try:
+                fno = sck.fileno()
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
             except (OSError, socket.error) as ex:
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "shut({}): {}".format(sck.fileno(), ex),
+                        "shut({}): {}".format(fno, ex),
                         c="1;30",
                     )
-                if ex.errno not in [10038, 10054, 107, 57, 9]:
+                if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
                     # 10038 No longer considered a socket
                     # 10054 Foribly closed by remote
                     #   107 Transport endpoint not connected
                     #    57 Socket is not connected
+                    #    49 Can't assign requested address (wifi down)
                     #     9 Bad file descriptor
                     raise
             finally:
                 with self.mutex:
                     del self.clients[cli]
+                    self.ncli -= 1
 
-                if self.disconnect_func:
-                    self.disconnect_func(addr)  # pylint: disable=not-callable
+    def cachebuster(self):
+        if time.time() - self.cb_ts < 1:
+            return self.cb_v
 
-    def thr_workload(self):
-        """indicates the python interpreter workload caused by this HttpSrv"""
-        # avoid locking in extract_filedata by tracking difference here
-        while True:
-            time.sleep(0.2)
-            with self.mutex:
-                if not self.clients:
-                    # no clients rn, termiante thread
-                    self.workload_thr_alive = False
-                    self.workload = 0
-                    return
+        with self.mutex:
+            if time.time() - self.cb_ts < 1:
+                return self.cb_v
 
-            total = 0
-            with self.mutex:
-                for cli in self.clients.keys():
-                    now = cli.workload
-                    delta = now - self.clients[cli]
-                    if delta < 0:
-                        # was reset in HttpCli to prevent overflow
-                        delta = now
+            v = E.t0
+            try:
+                with os.scandir(os.path.join(E.mod, "web")) as dh:
+                    for fh in dh:
+                        inf = fh.stat()
+                        v = max(v, inf.st_mtime)
+            except:
+                pass
 
-                    total += delta
-                    self.clients[cli] = now
-
-            self.workload = total
+            v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
+            self.cb_v = v.decode("ascii")[-4:]
+            self.cb_ts = time.time()
+            return self.cb_v

copyparty/ico.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import hashlib
+import colorsys
+
+from .__init__ import PY2
+
+
+class Ico(object):
+    def __init__(self, args):
+        self.args = args
+
+    def get(self, ext, as_thumb):
+        """placeholder to make thumbnails not break"""
+
+        h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        if PY2:
+            h = [ord(x) for x in h]
+
+        c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
+        c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
+        c = list(c1) + list(c2)
+        c = [int(x * 255) for x in c]
+        c = "".join(["{:02x}".format(x) for x in c])
+
+        h = 30
+        if not self.args.th_no_crop and as_thumb:
+            w, h = self.args.th_size.split("x")
+            h = int(100 / (float(w) / float(h)))
+
+        svg = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<svg version="1.1" viewBox="0 0 100 {}" xmlns="http://www.w3.org/2000/svg"><g>
+<rect width="100%" height="100%" fill="#{}" />
+<text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" xml:space="preserve"
+  fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
+</g></svg>
+"""
+        svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
+
+        return ["image/svg+xml", svg]

copyparty/mtag.py

@@ -1,59 +1,274 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
 import os
 import sys
+import json
 import shutil
 import subprocess as sp
 
-from .__init__ import PY2, WINDOWS
-from .util import fsenc, fsdec, REKOBO_LKEY
+from .__init__ import PY2, WINDOWS, unicode
+from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
+from .bos import bos
 
-if not PY2:
-    unicode = str
+
+def have_ff(cmd):
+    if PY2:
+        print("# checking {}".format(cmd))
+        cmd = (cmd + " -version").encode("ascii").split(b" ")
+        try:
+            sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
+            return True
+        except:
+            return False
+    else:
+        return bool(shutil.which(cmd))
+
+
+HAVE_FFMPEG = have_ff("ffmpeg")
+HAVE_FFPROBE = have_ff("ffprobe")
+
+
+class MParser(object):
+    def __init__(self, cmdline):
+        self.tag, args = cmdline.split("=", 1)
+        self.tags = self.tag.split(",")
+
+        self.timeout = 30
+        self.force = False
+        self.audio = "y"
+        self.ext = []
+
+        while True:
+            try:
+                bp = os.path.expanduser(args)
+                if WINDOWS:
+                    bp = uncyg(bp)
+
+                if bos.path.exists(bp):
+                    self.bin = bp
+                    return
+            except:
+                pass
+
+            arg, args = args.split(",", 1)
+            arg = arg.lower()
+
+            if arg.startswith("a"):
+                self.audio = arg[1:]  # [r]equire [n]ot [d]ontcare
+                continue
+
+            if arg == "f":
+                self.force = True
+                continue
+
+            if arg.startswith("t"):
+                self.timeout = int(arg[1:])
+                continue
+
+            if arg.startswith("e"):
+                self.ext.append(arg[1:])
+                continue
+
+            raise Exception()
+
+
+def ffprobe(abspath):
+    cmd = [
+        b"ffprobe",
+        b"-hide_banner",
+        b"-show_streams",
+        b"-show_format",
+        b"--",
+        fsenc(abspath),
+    ]
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    r = p.communicate()
+    txt = r[0].decode("utf-8", "replace")
+    return parse_ffprobe(txt)
+
+
+def parse_ffprobe(txt):
+    """ffprobe -show_format -show_streams"""
+    streams = []
+    fmt = {}
+    g = None
+    for ln in [x.rstrip("\r") for x in txt.split("\n")]:
+        try:
+            k, v = ln.split("=", 1)
+            g[k] = v
+            continue
+        except:
+            pass
+
+        if ln == "[STREAM]":
+            g = {}
+            streams.append(g)
+
+        if ln == "[FORMAT]":
+            g = {"codec_type": "format"}  # heh
+            fmt = g
+
+    streams = [fmt] + streams
+    ret = {}  # processed
+    md = {}  # raw tags
+
+    is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
+    if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
+        is_audio = True
+
+    # if audio file, ensure audio stream appears first
+    if (
+        is_audio
+        and len(streams) > 2
+        and streams[1].get("codec_type") != "audio"
+        and streams[2].get("codec_type") == "audio"
+    ):
+        streams = [fmt, streams[2], streams[1]] + streams[3:]
+
+    have = {}
+    for strm in streams:
+        typ = strm.get("codec_type")
+        if typ in have:
+            continue
+
+        have[typ] = True
+        kvm = []
+
+        if typ == "audio":
+            kvm = [
+                ["codec_name", "ac"],
+                ["channel_layout", "chs"],
+                ["sample_rate", ".hz"],
+                ["bit_rate", ".aq"],
+                ["duration", ".dur"],
+            ]
+
+        if typ == "video":
+            if strm.get("DISPOSITION:attached_pic") == "1" or is_audio:
+                continue
+
+            kvm = [
+                ["codec_name", "vc"],
+                ["pix_fmt", "pixfmt"],
+                ["r_frame_rate", ".fps"],
+                ["bit_rate", ".vq"],
+                ["width", ".resw"],
+                ["height", ".resh"],
+                ["duration", ".dur"],
+            ]
+
+        if typ == "format":
+            kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
+
+        for sk, rk in kvm:
+            v = strm.get(sk)
+            if v is None:
+                continue
+
+            if rk.startswith("."):
+                try:
+                    v = float(v)
+                    v2 = ret.get(rk)
+                    if v2 is None or v > v2:
+                        ret[rk] = v
+                except:
+                    # sqlite doesnt care but the code below does
+                    if v not in ["N/A"]:
+                        ret[rk] = v
+            else:
+                ret[rk] = v
+
+    if ret.get("vc") == "ansi":  # shellscript
+        return {}, {}
+
+    for strm in streams:
+        for k, v in strm.items():
+            if not k.startswith("TAG:"):
+                continue
+
+            k = k[4:].strip()
+            v = v.strip()
+            if k and v and k not in md:
+                md[k] = [v]
+
+    for k in [".q", ".vq", ".aq"]:
+        if k in ret:
+            ret[k] /= 1000  # bit_rate=320000
+
+    for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
+        if k in ret:
+            ret[k] = int(ret[k])
+
+    if ".fps" in ret:
+        fps = ret[".fps"]
+        if "/" in fps:
+            fa, fb = fps.split("/")
+            fps = int(fa) * 1.0 / int(fb)
+
+        if fps < 1000 and fmt.get("format_name") not in ["image2", "png_pipe"]:
+            ret[".fps"] = round(fps, 3)
+        else:
+            del ret[".fps"]
+
+    if ".dur" in ret:
+        if ret[".dur"] < 0.1:
+            del ret[".dur"]
+            if ".q" in ret:
+                del ret[".q"]
+
+    if ".resw" in ret and ".resh" in ret:
+        ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
+
+    ret = {k: [0, v] for k, v in ret.items()}
+
+    return ret, md
 
 
 class MTag(object):
     def __init__(self, log_func, args):
         self.log_func = log_func
+        self.args = args
         self.usable = True
-        self.prefer_mt = False
-        mappings = args.mtm
+        self.prefer_mt = not args.no_mtag_ff
         self.backend = "ffprobe" if args.no_mutagen else "mutagen"
-        or_ffprobe = " or ffprobe"
+        self.can_ffprobe = (
+            HAVE_FFPROBE
+            and not args.no_mtag_ff
+            and (not WINDOWS or sys.version_info >= (3, 8))
+        )
+        mappings = args.mtm
+        or_ffprobe = " or FFprobe"
 
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
                 import mutagen
             except:
-                self.log("could not load mutagen, trying ffprobe instead", c=3)
+                self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
         if self.backend == "ffprobe":
+            self.usable = self.can_ffprobe
             self.get = self.get_ffprobe
             self.prefer_mt = True
-            # about 20x slower
-            if PY2:
-                cmd = [b"ffprobe", b"-version"]
-                try:
-                    sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
-                except:
-                    self.usable = False
-            else:
-                if not shutil.which("ffprobe"):
-                    self.usable = False
 
-            if self.usable and WINDOWS and sys.version_info < (3, 8):
-                self.usable = False
+            if not HAVE_FFPROBE:
+                pass
+
+            elif args.no_mtag_ff:
+                msg = "found FFprobe but it was disabled by --no-mtag-ff"
+                self.log(msg, c=3)
+
+            elif WINDOWS and sys.version_info < (3, 8):
                 or_ffprobe = " or python >= 3.8"
-                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                msg = "found FFprobe but your python is too old; need 3.8 or newer"
                 self.log(msg, c=1)
 
         if not self.usable:
-            msg = "need mutagen{} to read media tags so please run this:\n  {} -m pip install --user mutagen"
-            self.log(msg.format(or_ffprobe, os.path.basename(sys.executable)), c=1)
+            msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
+            pybin = os.path.basename(sys.executable)
+            self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
             return
 
         # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -183,7 +398,7 @@ class MTag(object):
             v2 = r2.get(k)
             if v1 == v2:
                 print("  ", k, v1)
-            elif v1 != "0000":  # ffprobe date=0
+            elif v1 != "0000":  # FFprobe date=0
                 diffs.append(k)
                 print(" 1", k, v1)
                 print(" 2", k, v2)
@@ -198,128 +413,64 @@ class MTag(object):
         return r1
 
     def get_mutagen(self, abspath):
+        if not bos.path.isfile(abspath):
+            return {}
+
         import mutagen
 
         try:
-            md = mutagen.File(abspath, easy=True)
+            md = mutagen.File(fsenc(abspath), easy=True)
             x = md.info.length
         except Exception as ex:
-            return {}
+            return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
-        ret = {}
-        try:
-            dur = int(md.info.length)
+        sz = bos.path.getsize(abspath)
+        ret = {".q": [0, int((sz / md.info.length) / 128)]}
+
+        for attr, k, norm in [
+            ["codec", "ac", unicode],
+            ["channels", "chs", int],
+            ["sample_rate", ".hz", int],
+            ["bitrate", ".aq", int],
+            ["length", ".dur", int],
+        ]:
             try:
-                q = int(md.info.bitrate / 1024)
+                v = getattr(md.info, attr)
             except:
-                q = int((os.path.getsize(abspath) / dur) / 128)
+                if k != "ac":
+                    continue
 
-            ret[".dur"] = [0, dur]
-            ret[".q"] = [0, q]
-        except:
-            pass
+                try:
+                    v = str(md.info).split(".")[1]
+                    if v.startswith("ogg"):
+                        v = v[3:]
+                except:
+                    continue
+
+            if not v:
+                continue
+
+            if k == ".aq":
+                v /= 1000
+
+            if k == "ac" and v.startswith("mp4a.40."):
+                v = "aac"
+
+            ret[k] = [0, norm(v)]
 
         return self.normalize_tags(ret, md)
 
     def get_ffprobe(self, abspath):
-        cmd = [b"ffprobe", b"-hide_banner", b"--", fsenc(abspath)]
-        p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
-        r = p.communicate()
-        txt = r[1].decode("utf-8", "replace")
-        txt = [x.rstrip("\r") for x in txt.split("\n")]
-
-        """
-        note:
-          tags which contain newline will be truncated on first \n,
-          ffprobe emits \n and spacepads the : to align visually
-        note:
-          the Stream ln always mentions Audio: if audio
-          the Stream ln usually has kb/s, is more accurate
-          the Duration ln always has kb/s
-          the Metadata: after Chapter may contain BPM info,
-            title : Tempo: 126.0
-
-        Input #0, wav,
-          Metadata:
-            date : <OK>
-          Duration:
-            Chapter #
-            Metadata:
-              title : <NG>
-
-        Input #0, mp3,
-          Metadata:
-            album : <OK>
-          Duration:
-            Stream #0:0: Audio:
-            Stream #0:1: Video:
-            Metadata:
-              comment : <NG>
-        """
-
-        ptn_md_beg = re.compile("^( +)Metadata:$")
-        ptn_md_kv = re.compile("^( +)([^:]+) *: (.*)")
-        ptn_dur = re.compile("^ *Duration: ([^ ]+)(, |$)")
-        ptn_br1 = re.compile("^ *Duration: .*, bitrate: ([0-9]+) kb/s(, |$)")
-        ptn_br2 = re.compile("^ *Stream.*: Audio:.* ([0-9]+) kb/s(, |$)")
-        ptn_audio = re.compile("^ *Stream .*: Audio: ")
-        ptn_au_parent = re.compile("^ *(Input #|Stream .*: Audio: )")
-
-        ret = {}
-        md = {}
-        in_md = False
-        is_audio = False
-        au_parent = False
-        for ln in txt:
-            m = ptn_md_kv.match(ln)
-            if m and in_md and len(m.group(1)) == in_md:
-                _, k, v = [x.strip() for x in m.groups()]
-                if k != "" and v != "":
-                    md[k] = [v]
-                continue
-            else:
-                in_md = False
-
-            m = ptn_md_beg.match(ln)
-            if m and au_parent:
-                in_md = len(m.group(1)) + 2
-                continue
-
-            au_parent = bool(ptn_au_parent.search(ln))
-
-            if ptn_audio.search(ln):
-                is_audio = True
-
-            m = ptn_dur.search(ln)
-            if m:
-                sec = 0
-                tstr = m.group(1)
-                if tstr.lower() != "n/a":
-                    try:
-                        tf = tstr.split(",")[0].split(".")[0].split(":")
-                        for f in tf:
-                            sec *= 60
-                            sec += int(f)
-                    except:
-                        self.log("invalid timestr from ffprobe: [{}]".format(tstr), c=3)
-
-                ret[".dur"] = sec
-                m = ptn_br1.search(ln)
-                if m:
-                    ret[".q"] = m.group(1)
-
-            m = ptn_br2.search(ln)
-            if m:
-                ret[".q"] = m.group(1)
-
-        if not is_audio:
+        if not bos.path.isfile(abspath):
             return {}
 
-        ret = {k: [0, v] for k, v in ret.items()}
-
+        ret, md = ffprobe(abspath)
         return self.normalize_tags(ret, md)
 
     def get_bin(self, parsers, abspath):
+        if not bos.path.isfile(abspath):
+            return {}
+
         pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
         pypath = [str(pypath)] + [str(x) for x in sys.path if x]
         pypath = str(os.pathsep.join(pypath))
@@ -327,10 +478,13 @@ class MTag(object):
         env["PYTHONPATH"] = pypath
 
         ret = {}
-        for tagname, (binpath, timeout) in parsers.items():
+        for tagname, mp in parsers.items():
             try:
-                cmd = [sys.executable, binpath, abspath]
-                args = {"env": env, "timeout": timeout}
+                cmd = [mp.bin, abspath]
+                if mp.bin.endswith(".py"):
+                    cmd = [sys.executable] + cmd
+
+                args = {"env": env, "timeout": mp.timeout}
 
                 if WINDOWS:
                     args["creationflags"] = 0x4000
@@ -339,8 +493,16 @@ class MTag(object):
 
                 cmd = [fsenc(x) for x in cmd]
                 v = sp.check_output(cmd, **args).strip()
-                if v:
+                if not v:
+                    continue
+
+                if "," not in tagname:
                     ret[tagname] = v.decode("utf-8")
+                else:
+                    v = json.loads(v)
+                    for tag in tagname.split(","):
+                        if tag and tag in v:
+                            ret[tag] = v[tag]
             except:
                 pass
 

copyparty/star.py

@@ -1,9 +1,12 @@
-import os
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import tarfile
 import threading
 
 from .sutil import errdesc
 from .util import Queue, fsenc
+from .bos import bos
 
 
 class QFile(object):
@@ -30,10 +33,11 @@ class QFile(object):
 class StreamTar(object):
     """construct in-memory tar file from the given path"""
 
-    def __init__(self, fgen, **kwargs):
+    def __init__(self, log, fgen, **kwargs):
         self.ci = 0
         self.co = 0
         self.qfile = QFile()
+        self.log = log
         self.fgen = fgen
         self.errf = None
 
@@ -42,7 +46,7 @@ class StreamTar(object):
         fmt = tarfile.GNU_FORMAT
         self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
 
-        w = threading.Thread(target=self._gen)
+        w = threading.Thread(target=self._gen, name="star-gen")
         w.daemon = True
         w.start()
 
@@ -57,7 +61,7 @@ class StreamTar(object):
 
         yield None
         if self.errf:
-            os.unlink(self.errf["ap"])
+            bos.unlink(self.errf["ap"])
 
     def ser(self, f):
         name = f["vp"]
@@ -88,7 +92,8 @@ class StreamTar(object):
                 errors.append([f["vp"], repr(ex)])
 
         if errors:
-            self.errf = errdesc(errors)
+            self.errf, txt = errdesc(errors)
+            self.log("\n".join(([repr(self.errf)] + txt[1:])))
             self.ser(self.errf)
 
         self.tar.close()

copyparty/stolen/surrogateescape.py

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 """
 This is Victor Stinner's pure-Python implementation of PEP 383: the "surrogateescape" error
 handler of Python 3.
@@ -171,7 +173,7 @@ FS_ENCODING = sys.getfilesystemencoding()
 
 if WINDOWS and not PY3:
     # py2 thinks win* is mbcs, probably a bug? anyways this works
-    FS_ENCODING = 'utf-8'
+    FS_ENCODING = "utf-8"
 
 
 # normalize the filesystem encoding name.

copyparty/sutil.py

@@ -1,8 +1,12 @@
-import os
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import time
 import tempfile
 from datetime import datetime
 
+from .bos import bos
+
 
 def errdesc(errors):
     report = ["copyparty failed to add the following files to the archive:", ""]
@@ -14,12 +18,11 @@ def errdesc(errors):
         tf_path = tf.name
         tf.write("\r\n".join(report).encode("utf-8", "replace"))
 
-    dt = datetime.utcfromtimestamp(time.time())
-    dt = dt.strftime("%Y-%m%d-%H%M%S")
+    dt = datetime.utcnow().strftime("%Y-%m%d-%H%M%S")
 
-    os.chmod(tf_path, 0o444)
+    bos.chmod(tf_path, 0o444)
     return {
         "vp": "archive-errors-{}.txt".format(dt),
         "ap": tf_path,
-        "st": os.stat(tf_path),
-    }
+        "st": bos.stat(tf_path),
+    }, report

copyparty/svchub.py

@@ -1,17 +1,24 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
+import os
 import sys
 import time
+import shlex
+import string
+import signal
+import socket
 import threading
 from datetime import datetime, timedelta
 import calendar
 
-from .__init__ import PY2, WINDOWS, MACOS, VT100
+from .__init__ import E, PY2, WINDOWS, ANYWIN, MACOS, VT100, unicode
+from .util import mp, start_log_thrs, start_stackmon, min_ex, ansi_re
+from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
-from .util import mp
+from .th_srv import ThumbSrv, HAVE_PIL, HAVE_WEBP
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
 
 
 class SvcHub(object):
@@ -25,48 +32,306 @@ class SvcHub(object):
     put() can return a queue (if want_reply=True) which has a blocking get() with the response.
     """
 
-    def __init__(self, args):
+    def __init__(self, args, argv, printed):
         self.args = args
+        self.argv = argv
+        self.logf = None
+        self.stop_req = False
+        self.reload_req = False
+        self.stopping = False
+        self.reloading = False
+        self.stop_cond = threading.Condition()
+        self.retcode = 0
+        self.httpsrv_up = 0
 
-        self.ansi_re = re.compile("\033\\[[^m]*m")
         self.log_mutex = threading.Lock()
         self.next_day = 0
 
         self.log = self._log_disabled if args.q else self._log_enabled
+        if args.lo:
+            self._setup_logfile(printed)
+
+        if args.stackmon:
+            start_stackmon(args.stackmon, 0)
+
+        if args.log_thrs:
+            start_log_thrs(self.log, args.log_thrs, 0)
+
+        if not ANYWIN and not args.use_fpool:
+            args.no_fpool = True
+
+        if not args.no_fpool and args.j != 1:
+            m = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
+            if ANYWIN:
+                m = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
+                args.no_fpool = True
+
+            self.log("root", m, c=3)
 
         # initiate all services to manage
+        self.asrv = AuthSrv(self.args, self.log)
+        if args.ls:
+            self.asrv.dbg_ls()
+
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
+        self.thumbsrv = None
+        if not args.no_thumb:
+            if HAVE_PIL:
+                if not HAVE_WEBP:
+                    args.th_no_webp = True
+                    msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
+                    self.log("thumb", msg, c=3)
+
+                self.thumbsrv = ThumbSrv(self)
+            else:
+                msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
+                self.log(
+                    "thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3
+                )
+
+        if not args.no_acode and args.no_thumb:
+            msg = "setting --no-acode because --no-thumb (sorry)"
+            self.log("thumb", msg, c=6)
+            args.no_acode = True
+
+        if not args.no_acode and (not HAVE_FFMPEG or not HAVE_FFPROBE):
+            msg = "setting --no-acode because either FFmpeg or FFprobe is not available"
+            self.log("thumb", msg, c=6)
+            args.no_acode = True
+
+        args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
+
         # decide which worker impl to use
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker
         else:
-            self.log("root", "cannot efficiently use multiple CPU cores")
             from .broker_thr import BrokerThr as Broker
 
         self.broker = Broker(self)
 
-    def run(self):
-        thr = threading.Thread(target=self.tcpsrv.run)
+    def thr_httpsrv_up(self):
+        time.sleep(5)
+        expected = self.broker.num_workers * self.tcpsrv.nsrv
+        failed = expected - self.httpsrv_up
+        if not failed:
+            return
+
+        if self.args.ign_ebind_all:
+            return
+
+        if self.args.ign_ebind and self.tcpsrv.srv:
+            return
+
+        m = "{}/{} workers failed to start"
+        m = m.format(failed, expected)
+        self.log("root", m, 1)
+
+        self.retcode = 1
+        os.kill(os.getpid(), signal.SIGTERM)
+
+    def cb_httpsrv_up(self):
+        self.httpsrv_up += 1
+        if self.httpsrv_up != self.broker.num_workers:
+            return
+
+        time.sleep(0.1)  # purely cosmetic dw
+        self.log("root", "workers OK\n")
+        self.up2k.init_vols()
+
+        thr = threading.Thread(target=self.sd_notify, name="sd-notify")
         thr.daemon = True
         thr.start()
 
-        # winxp/py2.7 support: thr.join() kills signals
-        try:
-            while True:
-                time.sleep(9001)
+    def _logname(self):
+        dt = datetime.utcnow()
+        fn = self.args.lo
+        for fs in "YmdHMS":
+            fs = "%" + fs
+            if fs in fn:
+                fn = fn.replace(fs, dt.strftime(fs))
 
-        except KeyboardInterrupt:
+        return fn
+
+    def _setup_logfile(self, printed):
+        base_fn = fn = sel_fn = self._logname()
+        if fn != self.args.lo:
+            ctr = 0
+            # yup this is a race; if started sufficiently concurrently, two
+            # copyparties can grab the same logfile (considered and ignored)
+            while os.path.exists(sel_fn):
+                ctr += 1
+                sel_fn = "{}.{}".format(fn, ctr)
+
+        fn = sel_fn
+
+        try:
+            import lzma
+
+            lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+
+        except:
+            import codecs
+
+            lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
+
+        lh.base_fn = base_fn
+
+        argv = [sys.executable] + self.argv
+        if hasattr(shlex, "quote"):
+            argv = [shlex.quote(x) for x in argv]
+        else:
+            argv = ['"{}"'.format(x) for x in argv]
+
+        msg = "[+] opened logfile [{}]\n".format(fn)
+        printed += msg
+        lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
+        self.logf = lh
+        print(msg, end="")
+
+    def run(self):
+        self.tcpsrv.run()
+
+        thr = threading.Thread(target=self.thr_httpsrv_up)
+        thr.daemon = True
+        thr.start()
+
+        sigs = [signal.SIGINT, signal.SIGTERM]
+        if not ANYWIN:
+            sigs.append(signal.SIGUSR1)
+
+        for sig in sigs:
+            signal.signal(sig, self.signal_handler)
+
+        # macos hangs after shutdown on sigterm with while-sleep,
+        # windows cannot ^c stop_cond (and win10 does the macos thing but winxp is fine??)
+        # linux is fine with both,
+        # never lucky
+        if ANYWIN:
+            # msys-python probably fine but >msys-python
+            thr = threading.Thread(target=self.stop_thr, name="svchub-sig")
+            thr.daemon = True
+            thr.start()
+
+            try:
+                while not self.stop_req:
+                    time.sleep(1)
+            except:
+                pass
+
+            self.shutdown()
+            thr.join()
+        else:
+            self.stop_thr()
+
+    def reload(self):
+        if self.reloading:
+            return "cannot reload; already in progress"
+
+        self.reloading = True
+        t = threading.Thread(target=self._reload)
+        t.daemon = True
+        t.start()
+        return "reload initiated"
+
+    def _reload(self):
+        self.log("root", "reload scheduled")
+        with self.up2k.mutex:
+            self.asrv.reload()
+            self.up2k.reload()
+            self.broker.reload()
+
+        self.reloading = False
+
+    def stop_thr(self):
+        while not self.stop_req:
+            with self.stop_cond:
+                self.stop_cond.wait(9001)
+
+            if self.reload_req:
+                self.reload_req = False
+                self.reload()
+
+        self.shutdown()
+
+    def signal_handler(self, sig, frame):
+        if self.stopping:
+            return
+
+        if sig == signal.SIGUSR1:
+            self.reload_req = True
+        else:
+            self.stop_req = True
+
+        with self.stop_cond:
+            self.stop_cond.notify_all()
+
+    def shutdown(self):
+        if self.stopping:
+            return
+
+        # start_log_thrs(print, 0.1, 1)
+
+        self.stopping = True
+        self.stop_req = True
+        with self.stop_cond:
+            self.stop_cond.notify_all()
+
+        ret = 1
+        try:
             with self.log_mutex:
                 print("OPYTHAT")
 
             self.tcpsrv.shutdown()
             self.broker.shutdown()
-            print("nailed it")
+            self.up2k.shutdown()
+            if self.thumbsrv:
+                self.thumbsrv.shutdown()
+
+                for n in range(200):  # 10s
+                    time.sleep(0.05)
+                    if self.thumbsrv.stopped():
+                        break
+
+                    if n == 3:
+                        print("waiting for thumbsrv (10sec)...")
+
+            print("nailed it", end="")
+            ret = self.retcode
+        finally:
+            print("\033[0m")
+            if self.logf:
+                self.logf.close()
+
+            sys.exit(ret)
 
     def _log_disabled(self, src, msg, c=0):
-        pass
+        if not self.logf:
+            return
+
+        with self.log_mutex:
+            ts = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")[:-3]
+            self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
+
+            now = time.time()
+            if now >= self.next_day:
+                self._set_next_day()
+
+    def _set_next_day(self):
+        if self.next_day and self.logf and self.logf.base_fn != self._logname():
+            self.logf.close()
+            self._setup_logfile("")
+
+        dt = datetime.utcnow()
+
+        # unix timestamp of next 00:00:00 (leap-seconds safe)
+        day_now = dt.day
+        while dt.day == day_now:
+            dt += timedelta(hours=12)
+
+        dt = dt.replace(hour=0, minute=0, second=0)
+        self.next_day = calendar.timegm(dt.utctimetuple())
 
     def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
@@ -75,22 +340,15 @@ class SvcHub(object):
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
                 print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
-
-                # unix timestamp of next 00:00:00 (leap-seconds safe)
-                day_now = dt.day
-                while dt.day == day_now:
-                    dt += timedelta(hours=12)
-
-                dt = dt.replace(hour=0, minute=0, second=0)
-                self.next_day = calendar.timegm(dt.utctimetuple())
+                self._set_next_day()
 
             fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
                 fmt = "{} {:21} {}\n"
                 if "\033" in msg:
-                    msg = self.ansi_re.sub("", msg)
+                    msg = ansi_re.sub("", msg)
                 if "\033" in src:
-                    src = self.ansi_re.sub("", src)
+                    src = ansi_re.sub("", src)
             elif c:
                 if isinstance(c, int):
                     msg = "\033[3{}m{}".format(c, msg)
@@ -109,20 +367,20 @@ class SvcHub(object):
                 except:
                     print(msg.encode("ascii", "replace").decode(), end="")
 
+            if self.logf:
+                self.logf.write(msg)
+
     def check_mp_support(self):
         vmin = sys.version_info[1]
         if WINDOWS:
             msg = "need python 3.3 or newer for multiprocessing;"
-            if PY2:
-                # py2 pickler doesn't support winsock
-                return msg
-            elif vmin < 3:
+            if PY2 or vmin < 3:
                 return msg
         elif MACOS:
             return "multiprocessing is wonky on mac osx;"
         else:
-            msg = "need python 2.7 or 3.3+ for multiprocessing;"
-            if not PY2 and vmin < 3:
+            msg = "need python 3.3+ for multiprocessing;"
+            if PY2 or vmin < 3:
                 return msg
 
         try:
@@ -137,10 +395,11 @@ class SvcHub(object):
 
     def check_mp_enable(self):
         if self.args.j == 1:
-            self.log("root", "multiprocessing disabled by argument -j 1;")
+            self.log("svchub", "multiprocessing disabled by argument -j 1")
             return False
 
         if mp.cpu_count() <= 1:
+            self.log("svchub", "only one CPU detected; multiprocessing disabled")
             return False
 
         try:
@@ -154,5 +413,25 @@ class SvcHub(object):
         if not err:
             return True
         else:
-            self.log("root", err)
+            self.log("svchub", err)
+            self.log("svchub", "cannot efficiently use multiple CPU cores")
             return False
+
+    def sd_notify(self):
+        try:
+            addr = os.getenv("NOTIFY_SOCKET")
+            if not addr:
+                return
+
+            addr = unicode(addr)
+            if addr.startswith("@"):
+                addr = "\0" + addr[1:]
+
+            m = "".join(x for x in addr if x in string.printable)
+            self.log("sd_notify", m)
+
+            sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+            sck.connect(addr)
+            sck.sendall(b"READY=1")
+        except:
+            self.log("sd_notify", min_ex())

copyparty/szip.py

@@ -1,15 +1,18 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import time
 import zlib
-import struct
 from datetime import datetime
 
 from .sutil import errdesc
-from .util import yieldfile, sanitize_fn
+from .util import yieldfile, sanitize_fn, spack, sunpack
+from .bos import bos
 
 
 def dostime2unix(buf):
-    t, d = struct.unpack("<HH", buf)
+    t, d = sunpack(b"<HH", buf)
 
     ts = (t & 0x1F) * 2
     tm = (t >> 5) & 0x3F
@@ -33,13 +36,13 @@ def unixtime2dos(ts):
 
     bd = ((dy - 1980) << 9) + (dm << 5) + dd
     bt = (th << 11) + (tm << 5) + ts // 2
-    return struct.pack("<HH", bt, bd)
+    return spack(b"<HH", bt, bd)
 
 
 def gen_fdesc(sz, crc32, z64):
     ret = b"\x50\x4b\x07\x08"
-    fmt = "<LQQ" if z64 else "<LLL"
-    ret += struct.pack(fmt, crc32, sz, sz)
+    fmt = b"<LQQ" if z64 else b"<LLL"
+    ret += spack(fmt, crc32, sz, sz)
     return ret
 
 
@@ -63,7 +66,7 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
 
     if crc32:
-        crc32 = struct.pack("<L", crc32)
+        crc32 = spack(b"<L", crc32)
     else:
         crc32 = b"\x00" * 4
 
@@ -84,14 +87,14 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     # however infozip does actual sz and it even works on winxp
     # (same reasning for z64 extradata later)
     vsz = 0xFFFFFFFF if z64 else sz
-    ret += struct.pack("<LL", vsz, vsz)
+    ret += spack(b"<LL", vsz, vsz)
 
     # windows support (the "?" replace below too)
-    fn = sanitize_fn(fn, "/")
+    fn = sanitize_fn(fn, "/", [])
     bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
 
     z64_len = len(z64v) * 8 + 4 if z64v else 0
-    ret += struct.pack("<HH", len(bfn), z64_len)
+    ret += spack(b"<HH", len(bfn), z64_len)
 
     if h_pos is not None:
         # 2b comment, 2b diskno
@@ -103,12 +106,12 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
         ret += b"\x01\x00\x00\x00\xa4\x81"
 
         # 4b local-header-ofs
-        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+        ret += spack(b"<L", min(h_pos, 0xFFFFFFFF))
 
     ret += bfn
 
     if z64v:
-        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+        ret += spack(b"<HH" + b"Q" * len(z64v), 1, len(z64v) * 8, *z64v)
 
     return ret
 
@@ -133,7 +136,7 @@ def gen_ecdr(items, cdir_pos, cdir_end):
     need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
 
     # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
-    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+    ret += spack(b"<HHLL", nitems, nitems, csz, cpos)
 
     # 2b comment length
     ret += b"\x00\x00"
@@ -160,7 +163,7 @@ def gen_ecdr64(items, cdir_pos, cdir_end):
 
     # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
     cdir_sz = cdir_end - cdir_pos
-    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+    ret += spack(b"<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
 
     return ret
 
@@ -175,13 +178,14 @@ def gen_ecdr64_loc(ecdr64_pos):
     ret = b"\x50\x4b\x06\x07"
 
     # 4b cdisk, 8b start of ecdr64, 4b ndisks
-    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+    ret += spack(b"<LQL", 0, ecdr64_pos, 1)
 
     return ret
 
 
 class StreamZip(object):
-    def __init__(self, fgen, utf8=False, pre_crc=False):
+    def __init__(self, log, fgen, utf8=False, pre_crc=False):
+        self.log = log
         self.fgen = fgen
         self.utf8 = utf8
         self.pre_crc = pre_crc
@@ -244,8 +248,8 @@ class StreamZip(object):
                 errors.append([f["vp"], repr(ex)])
 
         if errors:
-            errf = errdesc(errors)
-            print(repr(errf))
+            errf, txt = errdesc(errors)
+            self.log("\n".join(([repr(errf)] + txt[1:])))
             for x in self.ser(errf):
                 yield x
 
@@ -268,4 +272,4 @@ class StreamZip(object):
         yield self._ct(ecdr)
 
         if errors:
-            os.unlink(errf["ap"])
+            bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -2,11 +2,10 @@
 from __future__ import print_function, unicode_literals
 
 import re
-import time
 import socket
-import select
 
-from .util import chkcmd, Counter
+from .__init__ import MACOS, ANYWIN
+from .util import chkcmd
 
 
 class TcpSrv(object):
@@ -20,7 +19,30 @@ class TcpSrv(object):
         self.args = hub.args
         self.log = hub.log
 
-        self.num_clients = Counter()
+        self.stopping = False
+
+        self.srv = []
+        self.nsrv = 0
+        ok = {}
+        for ip in self.args.i:
+            ok[ip] = []
+            for port in self.args.p:
+                self.nsrv += 1
+                try:
+                    self._listen(ip, port)
+                    ok[ip].append(port)
+                except Exception as ex:
+                    if self.args.ign_ebind or self.args.ign_ebind_all:
+                        m = "could not listen on {}:{}: {}"
+                        self.log("tcpsrv", m.format(ip, port, ex), c=3)
+                    else:
+                        raise
+
+        if not self.srv and not self.args.ign_ebind_all:
+            raise Exception("could not listen on any of the given interfaces")
+
+        if self.nsrv != len(self.srv):
+            self.log("tcpsrv", "")
 
         ip = "127.0.0.1"
         eps = {ip: "local only"}
@@ -31,19 +53,19 @@ class TcpSrv(object):
                 for x in nonlocals:
                     eps[x] = "external"
 
+        msgs = []
+        m = "available @ http://{}:{}/  (\033[33m{}\033[0m)"
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
             for port in sorted(self.args.p):
-                self.log(
-                    "tcpsrv",
-                    "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
-                        ip, port, desc
-                    ),
-                )
+                if port not in ok.get(ip, ok.get("0.0.0.0", [])):
+                    continue
 
-        self.srv = []
-        for ip in self.args.i:
-            for port in self.args.p:
-                self.srv.append(self._listen(ip, port))
+                msgs.append(m.format(ip, port, desc))
+
+        if msgs:
+            msgs[-1] += "\n"
+            for m in msgs:
+                self.log("tcpsrv", m)
 
     def _listen(self, ip, port):
         srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -51,7 +73,7 @@ class TcpSrv(object):
         srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         try:
             srv.bind((ip, port))
-            return srv
+            self.srv.append(srv)
         except (OSError, socket.error) as ex:
             if ex.errno in [98, 48]:
                 e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
@@ -65,57 +87,118 @@ class TcpSrv(object):
         for srv in self.srv:
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
-            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
+            fno = srv.fileno()
+            msg = "listening @ {}:{}  f{}".format(ip, port, fno)
+            self.log("tcpsrv", msg)
+            if self.args.q:
+                print(msg)
 
-        while True:
-            if self.args.log_conn:
-                self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
-
-            if self.num_clients.v >= self.args.nc:
-                time.sleep(0.1)
-                continue
-
-            if self.args.log_conn:
-                self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
-
-            ready, _, _ = select.select(self.srv, [], [])
-            for srv in ready:
-                sck, addr = srv.accept()
-                sip, sport = srv.getsockname()
-                if self.args.log_conn:
-                    self.log(
-                        "%s %s" % addr,
-                        "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
-                            "-" * 3, sip, sport % 8, sport
-                        ),
-                        c="1;30",
-                    )
-
-                self.num_clients.add()
-                self.hub.broker.put(False, "httpconn", sck, addr)
+            self.hub.broker.put(False, "listen", srv)
 
     def shutdown(self):
+        self.stopping = True
+        try:
+            for srv in self.srv:
+                srv.close()
+        except:
+            pass
+
         self.log("tcpsrv", "ok bye")
 
-    def detect_interfaces(self, listen_ips):
+    def ips_linux(self):
         eps = {}
-
-        # get all ips and their interfaces
         try:
-            ip_addr, _ = chkcmd("ip", "addr")
+            txt, _ = chkcmd(["ip", "addr"])
         except:
-            ip_addr = None
+            return eps
 
-        if ip_addr:
-            r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
-            for ln in ip_addr.split("\n"):
-                try:
-                    ip, dev = r.match(ln.rstrip()).groups()
-                    for lip in listen_ips:
-                        if lip in ["0.0.0.0", ip]:
-                            eps[ip] = dev
-                except:
-                    pass
+        r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
+        for ln in txt.split("\n"):
+            try:
+                ip, dev = r.match(ln.rstrip()).groups()
+                eps[ip] = dev
+            except:
+                pass
+
+        return eps
+
+    def ips_macos(self):
+        eps = {}
+        try:
+            txt, _ = chkcmd(["ifconfig"])
+        except:
+            return eps
+
+        rdev = re.compile(r"^([^ ]+):")
+        rip = re.compile(r"^\tinet ([0-9\.]+) ")
+        dev = None
+        for ln in txt.split("\n"):
+            m = rdev.match(ln)
+            if m:
+                dev = m.group(1)
+
+            m = rip.match(ln)
+            if m:
+                eps[m.group(1)] = dev
+                dev = None
+
+        return eps
+
+    def ips_windows_ipconfig(self):
+        eps = {}
+        try:
+            txt, _ = chkcmd(["ipconfig"])
+        except:
+            return eps
+
+        rdev = re.compile(r"(^[^ ].*):$")
+        rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
+        dev = None
+        for ln in txt.replace("\r", "").split("\n"):
+            m = rdev.match(ln)
+            if m:
+                dev = m.group(1).split(" adapter ", 1)[-1]
+
+            m = rip.match(ln)
+            if m and dev:
+                eps[m.group(1)] = dev
+                dev = None
+
+        return eps
+
+    def ips_windows_netsh(self):
+        eps = {}
+        try:
+            txt, _ = chkcmd("netsh interface ip show address".split())
+        except:
+            return eps
+
+        rdev = re.compile(r'.* "([^"]+)"$')
+        rip = re.compile(r".* IP\b.*: +([0-9\.]{7,15})$")
+        dev = None
+        for ln in txt.replace("\r", "").split("\n"):
+            m = rdev.match(ln)
+            if m:
+                dev = m.group(1)
+
+            m = rip.match(ln)
+            if m and dev:
+                eps[m.group(1)] = dev
+                dev = None
+
+        return eps
+
+    def detect_interfaces(self, listen_ips):
+        if MACOS:
+            eps = self.ips_macos()
+        elif ANYWIN:
+            eps = self.ips_windows_ipconfig()  # sees more interfaces
+            eps.update(self.ips_windows_netsh())  # has better names
+        else:
+            eps = self.ips_linux()
+
+        if "0.0.0.0" not in listen_ips:
+            eps = {k: v for k, v in eps.items() if k in listen_ips}
 
         default_route = None
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

copyparty/th_cli.py

@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+
+from .util import Cooldown
+from .th_srv import thumb_path, THUMBABLE, FMT_FFV, FMT_FFA
+from .bos import bos
+
+
+class ThumbCli(object):
+    def __init__(self, hsrv):
+        self.broker = hsrv.broker
+        self.log_func = hsrv.log
+        self.args = hsrv.args
+        self.asrv = hsrv.asrv
+
+        # cache on both sides for less broker spam
+        self.cooldown = Cooldown(self.args.th_poke)
+
+    def log(self, msg, c=0):
+        self.log_func("thumbcli", msg, c)
+
+    def get(self, ptop, rem, mtime, fmt):
+        ext = rem.rsplit(".")[-1].lower()
+        if ext not in THUMBABLE:
+            return None
+
+        is_vid = ext in FMT_FFV
+        if is_vid and self.args.no_vthumb:
+            return None
+
+        want_opus = fmt == "opus"
+        is_au = ext in FMT_FFA
+        if is_au:
+            if want_opus:
+                if self.args.no_acode:
+                    return None
+            else:
+                if self.args.no_athumb:
+                    return None
+        elif want_opus:
+            return None
+
+        if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg"]:
+            return os.path.join(ptop, rem)
+
+        if fmt == "j" and self.args.th_no_jpg:
+            fmt = "w"
+
+        if fmt == "w":
+            if self.args.th_no_webp or ((is_vid or is_au) and self.args.th_ff_jpg):
+                fmt = "j"
+
+        histpath = self.asrv.vfs.histtab.get(ptop)
+        if not histpath:
+            self.log("no histpath for [{}]".format(ptop))
+            return None
+
+        tpath = thumb_path(histpath, rem, mtime, fmt)
+        ret = None
+        try:
+            st = bos.stat(tpath)
+            if st.st_size:
+                ret = tpath
+            else:
+                return None
+        except:
+            pass
+
+        if ret:
+            tdir = os.path.dirname(tpath)
+            if self.cooldown.poke(tdir):
+                self.broker.put(False, "thumbsrv.poke", tdir)
+
+            if want_opus:
+                # audio files expire individually
+                if self.cooldown.poke(tpath):
+                    self.broker.put(False, "thumbsrv.poke", tpath)
+
+            return ret
+
+        x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
+        return x.get()

copyparty/th_srv.py

@@ -0,0 +1,516 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import time
+import shutil
+import base64
+import hashlib
+import threading
+import subprocess as sp
+
+from .__init__ import PY2, unicode
+from .util import fsenc, vsplit, statdir, runcmd, Queue, Cooldown, BytesIO, min_ex
+from .bos import bos
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+
+
+HAVE_PIL = False
+HAVE_HEIF = False
+HAVE_AVIF = False
+HAVE_WEBP = False
+
+try:
+    from PIL import Image, ImageOps, ExifTags
+
+    HAVE_PIL = True
+    try:
+        Image.new("RGB", (2, 2)).save(BytesIO(), format="webp")
+        HAVE_WEBP = True
+    except:
+        pass
+
+    try:
+        from pyheif_pillow_opener import register_heif_opener
+
+        register_heif_opener()
+        HAVE_HEIF = True
+    except:
+        pass
+
+    try:
+        import pillow_avif
+
+        HAVE_AVIF = True
+    except:
+        pass
+except:
+    pass
+
+# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
+# ffmpeg -formats
+FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
+FMT_FFV = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc mts h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
+FMT_FFA = "aac m4a ogg opus flac alac mp3 mp2 ac3 dts wma ra wav aif aiff au alaw ulaw mulaw amr gsm ape tak tta wv"
+
+if HAVE_HEIF:
+    FMT_PIL += " heif heifs heic heics"
+
+if HAVE_AVIF:
+    FMT_PIL += " avif avifs"
+
+FMT_PIL, FMT_FFV, FMT_FFA = [
+    {x: True for x in y.split(" ") if x} for y in [FMT_PIL, FMT_FFV, FMT_FFA]
+]
+
+
+THUMBABLE = {}
+
+if HAVE_PIL:
+    THUMBABLE.update(FMT_PIL)
+
+if HAVE_FFMPEG and HAVE_FFPROBE:
+    THUMBABLE.update(FMT_FFV)
+    THUMBABLE.update(FMT_FFA)
+
+
+def thumb_path(histpath, rem, mtime, fmt):
+    # base16 = 16 = 256
+    # b64-lc = 38 = 1444
+    # base64 = 64 = 4096
+    rd, fn = vsplit(rem)
+    if rd:
+        h = hashlib.sha512(fsenc(rd)).digest()
+        b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+        rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
+    else:
+        rd = "top"
+
+    # could keep original filenames but this is safer re pathlen
+    h = hashlib.sha512(fsenc(fn)).digest()
+    fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+
+    if fmt == "opus":
+        cat = "ac"
+    else:
+        fmt = "webp" if fmt == "w" else "jpg"
+        cat = "th"
+
+    return "{}/{}/{}/{}.{:x}.{}".format(histpath, cat, rd, fn, int(mtime), fmt)
+
+
+class ThumbSrv(object):
+    def __init__(self, hub):
+        self.hub = hub
+        self.asrv = hub.asrv
+        self.args = hub.args
+        self.log_func = hub.log
+
+        res = hub.args.th_size.split("x")
+        self.res = tuple([int(x) for x in res])
+        self.poke_cd = Cooldown(self.args.th_poke)
+
+        self.mutex = threading.Lock()
+        self.busy = {}
+        self.stopping = False
+        self.nthr = max(1, self.args.th_mt)
+
+        self.q = Queue(self.nthr * 4)
+        for n in range(self.nthr):
+            t = threading.Thread(
+                target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
+            )
+            t.daemon = True
+            t.start()
+
+        want_ff = not self.args.no_vthumb or not self.args.no_athumb
+        if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
+            missing = []
+            if not HAVE_FFMPEG:
+                missing.append("FFmpeg")
+
+            if not HAVE_FFPROBE:
+                missing.append("FFprobe")
+
+            msg = "cannot create audio/video thumbnails because some of the required programs are not available: "
+            msg += ", ".join(missing)
+            self.log(msg, c=3)
+
+        if self.args.th_clean:
+            t = threading.Thread(target=self.cleaner, name="thumb.cln")
+            t.daemon = True
+            t.start()
+
+    def log(self, msg, c=0):
+        self.log_func("thumb", msg, c)
+
+    def shutdown(self):
+        self.stopping = True
+        for _ in range(self.nthr):
+            self.q.put(None)
+
+    def stopped(self):
+        with self.mutex:
+            return not self.nthr
+
+    def get(self, ptop, rem, mtime, fmt):
+        histpath = self.asrv.vfs.histtab.get(ptop)
+        if not histpath:
+            self.log("no histpath for [{}]".format(ptop))
+            return None
+
+        tpath = thumb_path(histpath, rem, mtime, fmt)
+        abspath = os.path.join(ptop, rem)
+        cond = threading.Condition(self.mutex)
+        do_conv = False
+        with self.mutex:
+            try:
+                self.busy[tpath].append(cond)
+                self.log("wait {}".format(tpath))
+            except:
+                thdir = os.path.dirname(tpath)
+                bos.makedirs(thdir)
+
+                inf_path = os.path.join(thdir, "dir.txt")
+                if not bos.path.exists(inf_path):
+                    with open(inf_path, "wb") as f:
+                        f.write(fsenc(os.path.dirname(abspath)))
+
+                self.busy[tpath] = [cond]
+                do_conv = True
+
+        if do_conv:
+            self.q.put([abspath, tpath])
+            self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
+
+        while not self.stopping:
+            with self.mutex:
+                if tpath not in self.busy:
+                    break
+
+            with cond:
+                cond.wait(3)
+
+        try:
+            st = bos.stat(tpath)
+            if st.st_size:
+                self.poke(tpath)
+                return tpath
+        except:
+            pass
+
+        return None
+
+    def worker(self):
+        while not self.stopping:
+            task = self.q.get()
+            if not task:
+                break
+
+            abspath, tpath = task
+            ext = abspath.split(".")[-1].lower()
+            fun = None
+            if not bos.path.exists(tpath):
+                if ext in FMT_PIL:
+                    fun = self.conv_pil
+                elif ext in FMT_FFV:
+                    fun = self.conv_ffmpeg
+                elif ext in FMT_FFA:
+                    if tpath.endswith(".opus"):
+                        fun = self.conv_opus
+                    else:
+                        fun = self.conv_spec
+
+            if fun:
+                try:
+                    fun(abspath, tpath)
+                except:
+                    msg = "{} could not create thumbnail of {}\n{}"
+                    self.log(msg.format(fun.__name__, abspath, min_ex()), "1;30")
+                    with open(tpath, "wb") as _:
+                        pass
+
+            with self.mutex:
+                subs = self.busy[tpath]
+                del self.busy[tpath]
+
+            for x in subs:
+                with x:
+                    x.notify_all()
+
+        with self.mutex:
+            self.nthr -= 1
+
+    def fancy_pillow(self, im):
+        # exif_transpose is expensive (loads full image + unconditional copy)
+        r = max(*self.res) * 2
+        im.thumbnail((r, r), resample=Image.LANCZOS)
+        try:
+            k = next(k for k, v in ExifTags.TAGS.items() if v == "Orientation")
+            exif = im.getexif()
+            rot = int(exif[k])
+            del exif[k]
+        except:
+            rot = 1
+
+        rots = {8: Image.ROTATE_90, 3: Image.ROTATE_180, 6: Image.ROTATE_270}
+        if rot in rots:
+            im = im.transpose(rots[rot])
+
+        if self.args.th_no_crop:
+            im.thumbnail(self.res, resample=Image.LANCZOS)
+        else:
+            iw, ih = im.size
+            dw, dh = self.res
+            res = (min(iw, dw), min(ih, dh))
+            im = ImageOps.fit(im, res, method=Image.LANCZOS)
+
+        return im
+
+    def conv_pil(self, abspath, tpath):
+        with Image.open(fsenc(abspath)) as im:
+            try:
+                im = self.fancy_pillow(im)
+            except Exception as ex:
+                self.log("fancy_pillow {}".format(ex), "1;30")
+                im.thumbnail(self.res)
+
+            fmts = ["RGB", "L"]
+            args = {"quality": 40}
+
+            if tpath.endswith(".webp"):
+                # quality 80 = pillow-default
+                # quality 75 = ffmpeg-default
+                # method 0 = pillow-default, fast
+                # method 4 = ffmpeg-default
+                # method 6 = max, slow
+                fmts += ["RGBA", "LA"]
+                args["method"] = 6
+            else:
+                # default q = 75
+                args["progressive"] = True
+
+            if im.mode not in fmts:
+                # print("conv {}".format(im.mode))
+                im = im.convert("RGB")
+
+            im.save(tpath, **args)
+
+    def conv_ffmpeg(self, abspath, tpath):
+        ret, _ = ffprobe(abspath)
+
+        ext = abspath.rsplit(".")[-1]
+        if ext in ["h264", "h265"]:
+            seek = []
+        else:
+            dur = ret[".dur"][1] if ".dur" in ret else 4
+            seek = "{:.0f}".format(dur / 3)
+            seek = [b"-ss", seek.encode("utf-8")]
+
+        scale = "scale={0}:{1}:force_original_aspect_ratio="
+        if self.args.th_no_crop:
+            scale += "decrease,setsar=1:1"
+        else:
+            scale += "increase,crop={0}:{1},setsar=1:1"
+
+        scale = scale.format(*list(self.res)).encode("utf-8")
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner"
+        ]
+        cmd += seek
+        cmd += [
+            b"-i", fsenc(abspath),
+            b"-map", b"0:v:0",
+            b"-vf", scale,
+            b"-frames:v", b"1",
+            b"-metadata:s:v:0", b"rotate=0",
+        ]
+        # fmt: on
+
+        if tpath.endswith(".jpg"):
+            cmd += [
+                b"-q:v",
+                b"6",  # default=??
+            ]
+        else:
+            cmd += [
+                b"-q:v",
+                b"50",  # default=75
+                b"-compression_level:v",
+                b"6",  # default=4, 0=fast, 6=max
+            ]
+
+        cmd += [fsenc(tpath)]
+        self._run_ff(cmd)
+
+    def _run_ff(self, cmd):
+        # self.log((b" ".join(cmd)).decode("utf-8"))
+        ret, sout, serr = runcmd(cmd)
+        if ret != 0:
+            m = "FFmpeg failed (probably a corrupt video file):\n"
+            m += "\n".join(["ff: {}".format(x) for x in serr.split("\n")])
+            self.log(m, c="1;30")
+            raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
+
+    def conv_spec(self, abspath, tpath):
+        ret, _ = ffprobe(abspath)
+        if "ac" not in ret:
+            raise Exception("not audio")
+
+        fc = "[0:a:0]aresample=48000{},showspectrumpic=s=640x512,crop=780:544:70:50[o]"
+        fc = fc.format("" if self.args.th_ff_swr else ":resampler=soxr")
+
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner",
+            b"-i", fsenc(abspath),
+            b"-filter_complex", fc.encode("utf-8"),
+            b"-map", b"[o]"
+        ]
+        # fmt: on
+
+        if tpath.endswith(".jpg"):
+            cmd += [
+                b"-q:v",
+                b"6",  # default=??
+            ]
+        else:
+            cmd += [
+                b"-q:v",
+                b"50",  # default=75
+                b"-compression_level:v",
+                b"6",  # default=4, 0=fast, 6=max
+            ]
+
+        cmd += [fsenc(tpath)]
+        self._run_ff(cmd)
+
+    def conv_opus(self, abspath, tpath):
+        if self.args.no_acode:
+            raise Exception("disabled in server config")
+
+        ret, _ = ffprobe(abspath)
+        if "ac" not in ret:
+            raise Exception("not audio")
+
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner",
+            b"-i", fsenc(abspath),
+            b"-map", b"0:a:0",
+            b"-c:a", b"libopus",
+            b"-b:a", b"128k",
+            fsenc(tpath)
+        ]
+        # fmt: on
+
+        self._run_ff(cmd)
+
+    def poke(self, tdir):
+        if not self.poke_cd.poke(tdir):
+            return
+
+        ts = int(time.time())
+        try:
+            for _ in range(4):
+                bos.utime(tdir, (ts, ts))
+                tdir = os.path.dirname(tdir)
+        except:
+            pass
+
+    def cleaner(self):
+        interval = self.args.th_clean
+        while True:
+            time.sleep(interval)
+            ndirs = 0
+            for vol, histpath in self.asrv.vfs.histtab.items():
+                if histpath.startswith(vol):
+                    self.log("\033[Jcln {}/\033[A".format(histpath))
+                else:
+                    self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
+
+                ndirs += self.clean(histpath)
+
+            self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
+
+    def clean(self, histpath):
+        ret = 0
+        for cat in ["th", "ac"]:
+            ret += self._clean(histpath, cat, None)
+
+        return ret
+
+    def _clean(self, histpath, cat, thumbpath):
+        if not thumbpath:
+            thumbpath = os.path.join(histpath, cat)
+
+        # self.log("cln {}".format(thumbpath))
+        exts = ["jpg", "webp"] if cat == "th" else ["opus"]
+        maxage = getattr(self.args, cat + "_maxage")
+        now = time.time()
+        prev_b64 = None
+        prev_fp = None
+        try:
+            ents = statdir(self.log, not self.args.no_scandir, False, thumbpath)
+            ents = sorted(list(ents))
+        except:
+            return 0
+
+        ndirs = 0
+        for f, inf in ents:
+            fp = os.path.join(thumbpath, f)
+            cmp = fp.lower().replace("\\", "/")
+
+            # "top" or b64 prefix/full (a folder)
+            if len(f) <= 3 or len(f) == 24:
+                age = now - inf.st_mtime
+                if age > maxage:
+                    with self.mutex:
+                        safe = True
+                        for k in self.busy.keys():
+                            if k.lower().replace("\\", "/").startswith(cmp):
+                                safe = False
+                                break
+
+                        if safe:
+                            ndirs += 1
+                            self.log("rm -rf [{}]".format(fp))
+                            shutil.rmtree(fp, ignore_errors=True)
+                else:
+                    self._clean(histpath, cat, fp)
+
+                continue
+
+            # thumb file
+            try:
+                b64, ts, ext = f.split(".")
+                if len(b64) != 24 or len(ts) != 8 or ext not in exts:
+                    raise Exception()
+            except:
+                if f != "dir.txt":
+                    self.log("foreign file in thumbs dir: [{}]".format(fp), 1)
+
+                continue
+
+            if b64 == prev_b64:
+                self.log("rm replaced [{}]".format(fp))
+                bos.unlink(prev_fp)
+
+            if cat != "th" and inf.st_mtime + maxage < now:
+                self.log("rm expired [{}]".format(fp))
+                bos.unlink(fp)
+
+            prev_b64 = b64
+            prev_fp = fp
+
+        return ndirs

copyparty/u2idx.py

@@ -6,8 +6,11 @@ import os
 import time
 import threading
 from datetime import datetime
+from operator import itemgetter
 
-from .util import u8safe, s3dec, html_escape, Pebkac
+from .__init__ import ANYWIN, unicode
+from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey, quotep
+from .bos import bos
 from .up2k import up2k_wark_from_hashlist
 
 
@@ -19,13 +22,14 @@ except:
 
 
 class U2idx(object):
-    def __init__(self, args, log_func):
-        self.args = args
-        self.log_func = log_func
-        self.timeout = args.srch_time
+    def __init__(self, conn):
+        self.log_func = conn.log_func
+        self.asrv = conn.asrv
+        self.args = conn.args
+        self.timeout = self.args.srch_time
 
         if not HAVE_SQLITE3:
-            self.log("could not load sqlite3; searchign wqill be disabled")
+            self.log("your python does not have sqlite3; searching will be disabled")
             return
 
         self.cur = {}
@@ -47,57 +51,167 @@ class U2idx(object):
         fhash = body["hash"]
         wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
 
-        uq = "substr(w,1,16) = ? and w = ?"
+        uq = "where substr(w,1,16) = ? and w = ?"
         uv = [wark[:16], wark]
 
         try:
-            return self.run_query(vols, uq, uv, {})[0]
-        except Exception as ex:
-            raise Pebkac(500, repr(ex))
+            return self.run_query(vols, uq, uv)[0]
+        except:
+            raise Pebkac(500, min_ex())
 
     def get_cur(self, ptop):
+        if not HAVE_SQLITE3:
+            return None
+
         cur = self.cur.get(ptop)
         if cur:
             return cur
 
-        cur = _open(ptop)
-        if not cur:
+        histpath = self.asrv.vfs.histtab.get(ptop)
+        if not histpath:
+            self.log("no histpath for [{}]".format(ptop))
             return None
 
+        db_path = os.path.join(histpath, "up2k.db")
+        if not bos.path.exists(db_path):
+            return None
+
+        cur = sqlite3.connect(db_path, 2).cursor()
         self.cur[ptop] = cur
         return cur
 
-    def search(self, vols, body):
+    def search(self, vols, uq):
         """search by query params"""
         if not HAVE_SQLITE3:
             return []
 
-        qobj = {}
-        _conv_sz(qobj, body, "sz_min", "up.sz >= ?")
-        _conv_sz(qobj, body, "sz_max", "up.sz <= ?")
-        _conv_dt(qobj, body, "dt_min", "up.mt >= ?")
-        _conv_dt(qobj, body, "dt_max", "up.mt <= ?")
-        for seg, dk in [["path", "up.rd"], ["name", "up.fn"]]:
-            if seg in body:
-                _conv_txt(qobj, body, seg, dk)
+        q = ""
+        va = []
+        joins = ""
+        is_key = True
+        is_size = False
+        is_date = False
+        kw_key = ["(", ")", "and ", "or ", "not "]
+        kw_val = ["==", "=", "!=", ">", ">=", "<", "<=", "like "]
+        ptn_mt = re.compile(r"^\.?[a-z_-]+$")
+        mt_ctr = 0
+        mt_keycmp = "substr(up.w,1,16)"
+        mt_keycmp2 = None
+        ptn_lc = re.compile(r" (mt[0-9]+\.v) ([=<!>]+) \? $")
+        ptn_lcv = re.compile(r"[a-zA-Z]")
 
-        uq, uv = _sqlize(qobj)
+        while True:
+            uq = uq.strip()
+            if not uq:
+                break
 
-        qobj = {}
-        if "tags" in body:
-            _conv_txt(qobj, body, "tags", "mt.v")
+            ok = False
+            for kw in kw_key + kw_val:
+                if uq.startswith(kw):
+                    is_key = kw in kw_key
+                    uq = uq[len(kw) :]
+                    ok = True
+                    q += kw
+                    break
 
-        if "adv" in body:
-            _conv_adv(qobj, body, "adv")
+            if ok:
+                continue
+
+            v, uq = (uq + " ").split(" ", 1)
+            if is_key:
+                is_key = False
+
+                if v == "size":
+                    v = "up.sz"
+                    is_size = True
+
+                elif v == "date":
+                    v = "up.mt"
+                    is_date = True
+
+                elif v == "path":
+                    v = "up.rd"
+
+                elif v == "name":
+                    v = "up.fn"
+
+                elif v == "tags" or ptn_mt.match(v):
+                    mt_ctr += 1
+                    mt_keycmp2 = "mt{}.w".format(mt_ctr)
+                    joins += "inner join mt mt{} on {} = {} ".format(
+                        mt_ctr, mt_keycmp, mt_keycmp2
+                    )
+                    mt_keycmp = mt_keycmp2
+                    if v == "tags":
+                        v = "mt{0}.v".format(mt_ctr)
+                    else:
+                        v = "+mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
+
+                else:
+                    raise Pebkac(400, "invalid key [" + v + "]")
+
+                q += v + " "
+                continue
+
+            head = ""
+            tail = ""
+
+            if is_date:
+                is_date = False
+                v = v.upper().rstrip("Z").replace(",", " ").replace("T", " ")
+                while "  " in v:
+                    v = v.replace("  ", " ")
+
+                for fmt in [
+                    "%Y-%m-%d %H:%M:%S",
+                    "%Y-%m-%d %H:%M",
+                    "%Y-%m-%d %H",
+                    "%Y-%m-%d",
+                ]:
+                    try:
+                        v = datetime.strptime(v, fmt).timestamp()
+                        break
+                    except:
+                        pass
+
+            elif is_size:
+                is_size = False
+                v = int(float(v) * 1024 * 1024)
+
+            else:
+                if v.startswith("*"):
+                    head = "'%'||"
+                    v = v[1:]
+
+                if v.endswith("*"):
+                    tail = "||'%'"
+                    v = v[:-1]
+
+            q += " {}?{} ".format(head, tail)
+            va.append(v)
+            is_key = True
+
+            # lowercase tag searches
+            m = ptn_lc.search(q)
+            if not m or not ptn_lcv.search(unicode(v)):
+                continue
+
+            va.pop()
+            va.append(v.lower())
+            q = q[: m.start()]
+
+            field, oper = m.groups()
+            if oper in ["=", "=="]:
+                q += " {} like ? ".format(field)
+            else:
+                q += " lower({}) {} ? ".format(field, oper)
 
         try:
-            return self.run_query(vols, uq, uv, qobj)
+            return self.run_query(vols, joins + "where " + q, va)
         except Exception as ex:
             raise Pebkac(500, repr(ex))
 
-    def run_query(self, vols, uq, uv, targs):
-        self.log("qs: {} {} ,  {}".format(uq, repr(uv), repr(targs)))
-
+    def run_query(self, vols, uq, uv):
         done_flag = []
         self.active_id = "{:.6f}_{}".format(
             time.time(), threading.current_thread().ident
@@ -108,39 +222,19 @@ class U2idx(object):
                 self.active_id,
                 done_flag,
             ),
+            name="u2idx-terminator",
         )
         thr.daemon = True
         thr.start()
 
-        if not targs:
-            if not uq:
-                q = "select * from up"
-                v = ()
-            else:
-                q = "select * from up where " + uq
-                v = tuple(uv)
+        if not uq or not uv:
+            q = "select * from up"
+            v = ()
         else:
-            q = "select up.* from up"
-            keycmp = "substr(up.w,1,16)"
-            where = []
-            v = []
-            ctr = 0
-            for tq, tv in sorted(targs.items()):
-                ctr += 1
-                tq = tq.split("\n")[0]
-                keycmp2 = "mt{}.w".format(ctr)
-                q += " inner join mt mt{} on {} = {}".format(ctr, keycmp, keycmp2)
-                keycmp = keycmp2
-                where.append(tq.replace("mt.", keycmp[:-1]))
-                v.append(tv)
+            q = "select up.* from up " + uq
+            v = tuple(uv)
 
-            if uq:
-                where.append(uq)
-                v.extend(uv)
-
-            q += " where " + (" and ".join(where))
-
-        # self.log("q2: {} {}".format(q, repr(v)))
+        self.log("qs: {!r} {!r}".format(q, v))
 
         ret = []
         lim = 1000
@@ -153,9 +247,10 @@ class U2idx(object):
             self.active_cur = cur
 
             sret = []
+            fk = flags.get("fk")
             c = cur.execute(q, v)
             for hit in c:
-                w, ts, sz, rd, fn = hit
+                w, ts, sz, rd, fn, ip, at = hit
                 lim -= 1
                 if lim <= 0:
                     break
@@ -163,7 +258,23 @@ class U2idx(object):
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
 
-                rp = os.path.join(vtop, rd, fn).replace("\\", "/")
+                if not fk:
+                    suf = ""
+                else:
+                    try:
+                        ap = absreal(os.path.join(ptop, rd, fn))
+                        inf = bos.stat(ap)
+                    except:
+                        continue
+
+                    suf = (
+                        "?k="
+                        + gen_filekey(
+                            self.args.fk_salt, ap, sz, 0 if ANYWIN else inf.st_ino
+                        )[:fk]
+                    )
+
+                rp = quotep("/".join([x for x in [vtop, rd, fn] if x])) + suf
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
 
             for hit in sret:
@@ -178,6 +289,7 @@ class U2idx(object):
                 hit["tags"] = tags
 
             ret.extend(sret)
+            # print("[{}] {}".format(ptop, sret))
 
         done_flag.append(True)
         self.active_id = None
@@ -185,9 +297,13 @@ class U2idx(object):
         # undupe hits from multiple metadata keys
         if len(ret) > 1:
             ret = [ret[0]] + [
-                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+                y
+                for x, y in zip(ret[:-1], ret[1:])
+                if x["rp"].split("?")[0] != y["rp"].split("?")[0]
             ]
 
+        ret.sort(key=itemgetter("rp"))
+
         return ret, list(taglist.keys())
 
     def terminator(self, identifier, done_flag):
@@ -198,84 +314,3 @@ class U2idx(object):
 
         if identifier == self.active_id:
             self.active_cur.connection.interrupt()
-
-
-def _open(ptop):
-    db_path = os.path.join(ptop, ".hist", "up2k.db")
-    if os.path.exists(db_path):
-        return sqlite3.connect(db_path).cursor()
-
-
-def _conv_sz(q, body, k, sql):
-    if k in body:
-        q[sql] = int(float(body[k]) * 1024 * 1024)
-
-
-def _conv_dt(q, body, k, sql):
-    if k not in body:
-        return
-
-    v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ")
-    while "  " in v:
-        v = v.replace("  ", " ")
-
-    for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]:
-        try:
-            ts = datetime.strptime(v, fmt).timestamp()
-            break
-        except:
-            ts = None
-
-    if ts:
-        q[sql] = ts
-
-
-def _conv_txt(q, body, k, sql):
-    for v in body[k].split(" "):
-        inv = ""
-        if v.startswith("-"):
-            inv = "not"
-            v = v[1:]
-
-        if not v:
-            continue
-
-        head = "'%'||"
-        if v.startswith("^"):
-            head = ""
-            v = v[1:]
-
-        tail = "||'%'"
-        if v.endswith("$"):
-            tail = ""
-            v = v[:-1]
-
-        qk = "{} {} like {}?{}".format(sql, inv, head, tail)
-        q[qk + "\n" + v] = u8safe(v)
-
-
-def _conv_adv(q, body, k):
-    ptn = re.compile(r"^(\.?[a-z]+) *(==?|!=|<=?|>=?) *(.*)$")
-
-    parts = body[k].split(" ")
-    parts = [x.strip() for x in parts if x.strip()]
-
-    for part in parts:
-        m = ptn.match(part)
-        if not m:
-            p = html_escape(part)
-            raise Pebkac(400, "invalid argument [" + p + "]")
-
-        k, op, v = m.groups()
-        qk = "mt.k = '{}' and mt.v {} ?".format(k, op)
-        q[qk + "\n" + v] = u8safe(v)
-
-
-def _sqlize(qobj):
-    keys = []
-    values = []
-    for k, v in sorted(qobj.items()):
-        keys.append(k.split("\n")[0])
-        values.append(v)
-
-    return " and ".join(keys), values

copyparty/util.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import os
 import sys
+import stat
 import time
 import base64
 import select
@@ -15,8 +16,10 @@ import threading
 import mimetypes
 import contextlib
 import subprocess as sp  # nosec
+from datetime import datetime
+from collections import Counter
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN, VT100, unicode
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -34,10 +37,29 @@ if not PY2:
     from urllib.parse import unquote_to_bytes as unquote
     from urllib.parse import quote_from_bytes as quote
     from queue import Queue
+    from io import BytesIO
 else:
     from urllib import unquote  # pylint: disable=no-name-in-module
     from urllib import quote  # pylint: disable=no-name-in-module
     from Queue import Queue  # pylint: disable=import-error,no-name-in-module
+    from StringIO import StringIO as BytesIO
+
+
+try:
+    struct.unpack(b">i", b"idgi")
+    spack = struct.pack
+    sunpack = struct.unpack
+except:
+
+    def spack(f, *a, **ka):
+        return struct.pack(f.decode("ascii"), *a, **ka)
+
+    def sunpack(f, *a, **ka):
+        return struct.unpack(f.decode("ascii"), *a, **ka)
+
+
+ansi_re = re.compile("\033\\[[^mK]*[mK]")
+
 
 surrogateescape.register_surrogateescape()
 FS_ENCODING = sys.getfilesystemencoding()
@@ -45,15 +67,20 @@ if WINDOWS and PY2:
     FS_ENCODING = "utf-8"
 
 
+HTTP_TS_FMT = "%a, %d %b %Y %H:%M:%S GMT"
+
+
 HTTPCODE = {
     200: "OK",
     204: "No Content",
     206: "Partial Content",
+    302: "Found",
     304: "Not Modified",
     400: "Bad Request",
     403: "Forbidden",
     404: "Not Found",
     405: "Method Not Allowed",
+    411: "Length Required",
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
@@ -72,6 +99,27 @@ IMPLICATIONS = [
 ]
 
 
+MIMES = {
+    "md": "text/plain",
+    "txt": "text/plain",
+    "js": "text/javascript",
+    "opus": "audio/ogg; codecs=opus",
+    "mp3": "audio/mpeg",
+    "m4a": "audio/mp4",
+    "jpg": "image/jpeg",
+}
+for ln in """text css html csv
+application json wasm xml pdf rtf zip
+image webp jpeg png gif bmp
+audio aac ogg wav
+video webm mp4 mpeg
+font woff woff2 otf ttf
+""".splitlines():
+    k, vs = ln.split(" ", 1)
+    for v in vs.strip().split():
+        MIMES[v] = "{}/{}".format(k, v)
+
+
 REKOBO_KEY = {
     v: ln.split(" ", 1)[0]
     for ln in """
@@ -109,21 +157,33 @@ REKOBO_KEY = {
 REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
 
 
-class Counter(object):
-    def __init__(self, v=0):
-        self.v = v
+class Cooldown(object):
+    def __init__(self, maxage):
+        self.maxage = maxage
         self.mutex = threading.Lock()
+        self.hist = {}
+        self.oldest = 0
 
-    def add(self, delta=1):
+    def poke(self, key):
         with self.mutex:
-            self.v += delta
+            now = time.time()
 
-    def set(self, absval):
-        with self.mutex:
-            self.v = absval
+            ret = False
+            v = self.hist.get(key, 0)
+            if now - v > self.maxage:
+                self.hist[key] = now
+                ret = True
+
+            if self.oldest - now > self.maxage * 2:
+                self.hist = {
+                    k: v for k, v in self.hist.items() if now - v < self.maxage
+                }
+                self.oldest = sorted(self.hist.values())[0]
+
+            return ret
 
 
-class Unrecv(object):
+class _Unrecv(object):
     """
     undo any number of socket recv ops
     """
@@ -143,17 +203,124 @@ class Unrecv(object):
         except:
             return b""
 
+    def recv_ex(self, nbytes):
+        """read an exact number of bytes"""
+        ret = self.recv(nbytes)
+        while ret and len(ret) < nbytes:
+            buf = self.recv(nbytes - len(ret))
+            if not buf:
+                break
+
+            ret += buf
+
+        return ret
+
     def unrecv(self, buf):
         self.buf = buf + self.buf
 
 
+class _LUnrecv(object):
+    """
+    with expensive debug logging
+    """
+
+    def __init__(self, s):
+        self.s = s
+        self.buf = b""
+
+    def recv(self, nbytes):
+        if self.buf:
+            ret = self.buf[:nbytes]
+            self.buf = self.buf[nbytes:]
+            m = "\033[0;7mur:pop:\033[0;1;32m {}\n\033[0;7mur:rem:\033[0;1;35m {}\033[0m\n"
+            print(m.format(ret, self.buf), end="")
+            return ret
+
+        try:
+            ret = self.s.recv(nbytes)
+            m = "\033[0;7mur:recv\033[0;1;33m {}\033[0m\n"
+            print(m.format(ret), end="")
+            return ret
+        except:
+            return b""
+
+    def recv_ex(self, nbytes):
+        """read an exact number of bytes"""
+        ret = self.recv(nbytes)
+        while ret and len(ret) < nbytes:
+            buf = self.recv(nbytes - len(ret))
+            if not buf:
+                break
+
+            ret += buf
+
+        return ret
+
+    def unrecv(self, buf):
+        self.buf = buf + self.buf
+        m = "\033[0;7mur:push\033[0;1;31m {}\n\033[0;7mur:rem:\033[0;1;35m {}\033[0m\n"
+        print(m.format(buf, self.buf), end="")
+
+
+Unrecv = _Unrecv
+
+
+class FHC(object):
+    class CE(object):
+        def __init__(self, fh):
+            self.ts = 0
+            self.fhs = [fh]
+
+    def __init__(self):
+        self.cache = {}
+
+    def close(self, path):
+        try:
+            ce = self.cache[path]
+        except:
+            return
+
+        for fh in ce.fhs:
+            fh.close()
+
+        del self.cache[path]
+
+    def clean(self):
+        if not self.cache:
+            return
+
+        keep = {}
+        now = time.time()
+        for path, ce in self.cache.items():
+            if now < ce.ts + 5:
+                keep[path] = ce
+            else:
+                for fh in ce.fhs:
+                    fh.close()
+
+        self.cache = keep
+
+    def pop(self, path):
+        return self.cache[path].fhs.pop()
+
+    def put(self, path, fh):
+        try:
+            ce = self.cache[path]
+            ce.fhs.append(fh)
+        except:
+            ce = self.CE(fh)
+            self.cache[path] = ce
+
+        ce.ts = time.time()
+
+
 class ProgressPrinter(threading.Thread):
     """
     periodically print progress info without linefeeds
     """
 
     def __init__(self):
-        threading.Thread.__init__(self)
+        threading.Thread.__init__(self, name="pp")
         self.daemon = True
         self.msg = None
         self.end = False
@@ -161,15 +328,22 @@ class ProgressPrinter(threading.Thread):
 
     def run(self):
         msg = None
+        fmt = " {}\033[K\r" if VT100 else " {} $\r"
         while not self.end:
             time.sleep(0.1)
             if msg == self.msg or self.end:
                 continue
 
             msg = self.msg
-            uprint(" {}\033[K\r".format(msg))
+            uprint(fmt.format(msg))
+            if PY2:
+                sys.stdout.flush()
+
+        if VT100:
+            print("\033[K", end="")
+        elif msg:
+            print("------------------------")
 
-        print("\033[K", end="")
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
 
 
@@ -189,7 +363,7 @@ def nuprint(msg):
 
 def rice_tid():
     tid = threading.current_thread().ident
-    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    c = sunpack(b"B" * 5, spack(b">Q", tid)[-5:])
     return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
 
 
@@ -212,16 +386,126 @@ def trace(*args, **kwargs):
     nuprint(msg)
 
 
+def alltrace():
+    threads = {}
+    names = dict([(t.ident, t.name) for t in threading.enumerate()])
+    for tid, stack in sys._current_frames().items():
+        name = "{} ({:x})".format(names.get(tid), tid)
+        threads[name] = stack
+
+    rret = []
+    bret = []
+    for name, stack in sorted(threads.items()):
+        ret = ["\n\n# {}".format(name)]
+        pad = None
+        for fn, lno, name, line in traceback.extract_stack(stack):
+            fn = os.sep.join(fn.split(os.sep)[-3:])
+            ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
+            if line:
+                ret.append("  " + str(line.strip()))
+                if "self.not_empty.wait()" in line:
+                    pad = " " * 4
+
+        if pad:
+            bret += [ret[0]] + [pad + x for x in ret[1:]]
+        else:
+            rret += ret
+
+    return "\n".join(rret + bret)
+
+
+def start_stackmon(arg_str, nid):
+    suffix = "-{}".format(nid) if nid else ""
+    fp, f = arg_str.rsplit(",", 1)
+    f = int(f)
+    t = threading.Thread(
+        target=stackmon,
+        args=(fp, f, suffix),
+        name="stackmon" + suffix,
+    )
+    t.daemon = True
+    t.start()
+
+
+def stackmon(fp, ival, suffix):
+    ctr = 0
+    while True:
+        ctr += 1
+        time.sleep(ival)
+        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
+        with open(fp + suffix, "wb") as f:
+            f.write(st.encode("utf-8", "replace"))
+
+
+def start_log_thrs(logger, ival, nid):
+    ival = float(ival)
+    tname = lname = "log-thrs"
+    if nid:
+        tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
+        lname = tname[3:]
+
+    t = threading.Thread(
+        target=log_thrs,
+        args=(logger, ival, lname),
+        name=tname,
+    )
+    t.daemon = True
+    t.start()
+
+
+def log_thrs(log, ival, name):
+    while True:
+        time.sleep(ival)
+        tv = [x.name for x in threading.enumerate()]
+        tv = [
+            x.split("-")[0]
+            if x.split("-")[0] in ["httpconn", "thumb", "tagger"]
+            else "listen"
+            if "-listen-" in x
+            else x
+            for x in tv
+            if not x.startswith("pydevd.")
+        ]
+        tv = ["{}\033[36m{}".format(v, k) for k, v in sorted(Counter(tv).items())]
+        log(name, "\033[0m \033[33m".join(tv), 3)
+
+
+def vol_san(vols, txt):
+    for vol in vols:
+        txt = txt.replace(vol.realpath.encode("utf-8"), vol.vpath.encode("utf-8"))
+        txt = txt.replace(
+            vol.realpath.encode("utf-8").replace(b"\\", b"\\\\"),
+            vol.vpath.encode("utf-8"),
+        )
+
+    return txt
+
+
+def min_ex():
+    et, ev, tb = sys.exc_info()
+    tb = traceback.extract_tb(tb)
+    fmt = "{} @ {} <{}>: {}"
+    ex = [fmt.format(fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in tb]
+    ex.append("[{}] {}".format(et.__name__, ev))
+    return "\n".join(ex[-8:])
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
+    fun = kwargs.pop("fun", open)
     fdir = kwargs.pop("fdir", None)
     suffix = kwargs.pop("suffix", None)
 
     if fname == os.devnull:
-        with open(fname, *args, **kwargs) as f:
+        with fun(fname, *args, **kwargs) as f:
             yield {"orz": [f, fname]}
             return
 
+    if suffix:
+        ext = fname.split(".")[-1]
+        if len(ext) < 7:
+            suffix += "." + ext
+
     orig_name = fname
     bname = fname
     ext = ""
@@ -242,12 +526,12 @@ def ren_open(fname, *args, **kwargs):
             else:
                 fpath = fname
 
-            if suffix and os.path.exists(fpath):
+            if suffix and os.path.exists(fsenc(fpath)):
                 fpath += suffix
                 fname += suffix
                 ext += suffix
 
-            with open(fsenc(fpath), *args, **kwargs) as f:
+            with fun(fsenc(fpath), *args, **kwargs) as f:
                 if b64:
                     fp2 = "fn-trunc.{}.txt".format(b64)
                     fp2 = os.path.join(fdir, fp2)
@@ -265,7 +549,7 @@ def ren_open(fname, *args, **kwargs):
         if not b64:
             b64 = (bname + ext).encode("utf-8", "replace")
             b64 = hashlib.sha512(b64).digest()[:12]
-            b64 = base64.urlsafe_b64encode(b64).decode("utf-8").rstrip("=")
+            b64 = base64.urlsafe_b64encode(b64).decode("utf-8")
 
         badlen = len(fname)
         while len(fname) >= badlen:
@@ -292,8 +576,8 @@ class MultipartParser(object):
         self.log = log_func
         self.headers = http_headers
 
-        self.re_ctype = re.compile(r"^content-type: *([^;]+)", re.IGNORECASE)
-        self.re_cdisp = re.compile(r"^content-disposition: *([^;]+)", re.IGNORECASE)
+        self.re_ctype = re.compile(r"^content-type: *([^; ]+)", re.IGNORECASE)
+        self.re_cdisp = re.compile(r"^content-disposition: *([^; ]+)", re.IGNORECASE)
         self.re_cdisp_field = re.compile(
             r'^content-disposition:(?: *|.*; *)name="([^"]+)"', re.IGNORECASE
         )
@@ -429,19 +713,21 @@ class MultipartParser(object):
         yields [fieldname, unsanitized_filename, fieldvalue]
         where fieldvalue yields chunks of data
         """
-        while True:
+        run = True
+        while run:
             fieldname, filename = self._read_header()
             yield [fieldname, filename, self._read_data()]
 
-            tail = self.sr.recv(2)
+            tail = self.sr.recv_ex(2)
 
             if tail == b"--":
                 # EOF indicated by this immediately after final boundary
-                self.sr.recv(2)
-                return
+                tail = self.sr.recv_ex(2)
+                run = False
 
             if tail != b"\r\n":
-                raise Pebkac(400, "protocol error after field value")
+                m = "protocol error after field value: want b'\\r\\n', got {!r}"
+                raise Pebkac(400, m.format(tail))
 
     def _read_value(self, iterator, max_len):
         ret = b""
@@ -490,7 +776,7 @@ class MultipartParser(object):
 def get_boundary(headers):
     # boundaries contain a-z A-Z 0-9 ' ( ) + _ , - . / : = ?
     # (whitespace allowed except as the last char)
-    ptn = r"^multipart/form-data; *(.*; *)?boundary=([^;]+)"
+    ptn = r"^multipart/form-data *; *(.*; *)?boundary=([^;]+)"
     ct = headers["content-type"]
     m = re.match(ptn, ct, re.IGNORECASE)
     if not m:
@@ -521,8 +807,18 @@ def read_header(sr):
             else:
                 continue
 
-        sr.unrecv(ret[ofs + 4 :])
-        return ret[:ofs].decode("utf-8", "surrogateescape").split("\r\n")
+        if len(ret) > ofs + 4:
+            sr.unrecv(ret[ofs + 4 :])
+
+        return ret[:ofs].decode("utf-8", "surrogateescape").lstrip("\r\n").split("\r\n")
+
+
+def gen_filekey(salt, fspath, fsize, inode):
+    return base64.urlsafe_b64encode(
+        hashlib.sha512(
+            "{} {} {} {}".format(salt, fspath, fsize, inode).encode("utf-8", "replace")
+        ).digest()
+    ).decode("ascii")
 
 
 def humansize(sz, terse=False):
@@ -540,6 +836,17 @@ def humansize(sz, terse=False):
     return ret.replace("iB", "").replace(" ", "")
 
 
+def unhumanize(sz):
+    try:
+        return float(sz)
+    except:
+        pass
+
+    mul = sz[-1:].lower()
+    mul = {"k": 1024, "m": 1024 * 1024, "g": 1024 * 1024 * 1024}.get(mul, 1)
+    return float(sz[:-1]) * mul
+
+
 def get_spd(nbyte, t0, t=None):
     if t is None:
         t = time.time()
@@ -560,6 +867,16 @@ def s2hms(s, optional_h=False):
     return "{}:{:02}:{:02}".format(h, m, s)
 
 
+def uncyg(path):
+    if len(path) < 2 or not path.startswith("/"):
+        return path
+
+    if len(path) > 2 and path[2] != "/":
+        return path
+
+    return "{}:\\{}".format(path[1], path[3:])
+
+
 def undot(path):
     ret = []
     for node in path.split("/"):
@@ -576,12 +893,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn, ok=""):
+def sanitize_fn(fn, ok, bad):
     if "/" not in ok:
         fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [x for x in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -591,19 +908,33 @@ def sanitize_fn(fn, ok=""):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ] if x[0] not in ok]:
-            fn = fn.replace(bad, good)
+        ]
+        for a, b in [x for x in remap if x[0] not in ok]:
+            fn = fn.replace(a, b)
 
-        bad = ["con", "prn", "aux", "nul"]
+        bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):
             bad += "com{0} lpt{0}".format(n).split(" ")
 
-        if fn.lower() in bad:
-            fn = "_" + fn
+    if fn.lower() in bad:
+        fn = "_" + fn
 
     return fn.strip()
 
 
+def absreal(fpath):
+    try:
+        return fsdec(os.path.abspath(os.path.realpath(fsenc(fpath))))
+    except:
+        if not WINDOWS:
+            raise
+
+        # cpython bug introduced in 3.8, still exists in 3.9.1,
+        # some win7sp1 and win10:20H2 boxes cannot realpath a
+        # networked drive letter such as b"n:" or b"n:\\"
+        return os.path.abspath(os.path.realpath(fpath))
+
+
 def u8safe(txt):
     try:
         return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
@@ -615,17 +946,29 @@ def exclude_dotfiles(filepaths):
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def http_ts(ts):
+    file_dt = datetime.utcfromtimestamp(ts)
+    return file_dt.strftime(HTTP_TS_FMT)
+
+
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 
@@ -649,6 +992,13 @@ def unquotep(txt):
     return w8dec(unq2)
 
 
+def vsplit(vpath):
+    if "/" not in vpath:
+        return "", vpath
+
+    return vpath.rsplit("/", 1)
+
+
 def w8dec(txt):
     """decodes filesystem-bytes to wtf8"""
     if PY2:
@@ -714,6 +1064,8 @@ def s3dec(rd, fn):
 
 
 def atomic_move(src, dst):
+    src = fsenc(src)
+    dst = fsenc(dst)
     if not PY2:
         os.replace(src, dst)
     else:
@@ -769,8 +1121,12 @@ def read_socket_chunked(sr, log=None):
             raise Pebkac(400, err)
 
         if chunklen == 0:
-            sr.recv(2)  # \r\n after final chunk
-            return
+            x = sr.recv_ex(2)
+            if x == b"\r\n":
+                return
+
+            m = "protocol error after final chunk: want b'\\r\\n', got {!r}"
+            raise Pebkac(400, m.format(x))
 
         if log:
             log("receiving {} byte chunk".format(chunklen))
@@ -778,7 +1134,10 @@ def read_socket_chunked(sr, log=None):
         for chunk in read_socket(sr, chunklen):
             yield chunk
 
-        sr.recv(2)  # \r\n after each chunk too
+        x = sr.recv_ex(2)
+        if x != b"\r\n":
+            m = "protocol error in chunk separator: want b'\\r\\n', got {!r}"
+            raise Pebkac(400, m.format(x))
 
 
 def yieldfile(fn):
@@ -791,21 +1150,16 @@ def yieldfile(fn):
             yield buf
 
 
-def hashcopy(actor, fin, fout):
-    u32_lim = int((2 ** 31) * 0.9)
+def hashcopy(fin, fout):
     hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
-        actor.workload += 1
-        if actor.workload > u32_lim:
-            actor.workload = 100  # prevent overflow
-
         tlen += len(buf)
         hashobj.update(buf)
         fout.write(buf)
 
-    digest32 = hashobj.digest()[:32]
-    digest_b64 = base64.urlsafe_b64encode(digest32).decode("utf-8").rstrip("=")
+    digest = hashobj.digest()[:33]
+    digest_b64 = base64.urlsafe_b64encode(digest).decode("utf-8")
 
     return tlen, hashobj.hexdigest(), digest_b64
 
@@ -815,7 +1169,7 @@ def sendfile_py(lower, upper, f, s):
     f.seek(lower)
     while remains > 0:
         # time.sleep(0.01)
-        buf = f.read(min(4096, remains))
+        buf = f.read(min(1024 * 32, remains))
         if not buf:
             return remains
 
@@ -851,6 +1205,12 @@ def sendfile_kern(lower, upper, f, s):
 
 
 def statdir(logger, scandir, lstat, top):
+    if lstat and ANYWIN:
+        lstat = False
+
+    if lstat and not os.supports_follow_symlinks:
+        scandir = False
+
     try:
         btop = fsenc(top)
         if scandir and hasattr(os, "scandir"):
@@ -860,8 +1220,7 @@ def statdir(logger, scandir, lstat, top):
                     try:
                         yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
                     except Exception as ex:
-                        msg = "scan-stat: \033[36m{} @ {}"
-                        logger(msg.format(repr(ex), fsdec(fh.path)))
+                        logger(src, "[s] {} @ {}".format(repr(ex), fsdec(fh.path)), 6)
         else:
             src = "listdir"
             fun = os.lstat if lstat else os.stat
@@ -870,11 +1229,35 @@ def statdir(logger, scandir, lstat, top):
                 try:
                     yield [fsdec(name), fun(abspath)]
                 except Exception as ex:
-                    msg = "list-stat: \033[36m{} @ {}"
-                    logger(msg.format(repr(ex), fsdec(abspath)))
+                    logger(src, "[s] {} @ {}".format(repr(ex), fsdec(abspath)), 6)
 
     except Exception as ex:
-        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
+        logger(src, "{} @ {}".format(repr(ex), top), 1)
+
+
+def rmdirs(logger, scandir, lstat, top, depth):
+    if not os.path.exists(fsenc(top)) or not os.path.isdir(fsenc(top)):
+        top = os.path.dirname(top)
+        depth -= 1
+
+    dirs = statdir(logger, scandir, lstat, top)
+    dirs = [x[0] for x in dirs if stat.S_ISDIR(x[1].st_mode)]
+    dirs = [os.path.join(top, x) for x in dirs]
+    ok = []
+    ng = []
+    for d in dirs[::-1]:
+        a, b = rmdirs(logger, scandir, lstat, d, depth + 1)
+        ok += a
+        ng += b
+
+    if depth:
+        try:
+            os.rmdir(fsenc(top))
+            ok.append(top)
+        except:
+            ng.append(top)
+
+    return ok, ng
 
 
 def unescape_cookie(orig):
@@ -905,37 +1288,60 @@ def unescape_cookie(orig):
     return ret
 
 
-def guess_mime(url):
-    if url.endswith(".md"):
-        return ["text/plain; charset=UTF-8"]
+def guess_mime(url, fallback="application/octet-stream"):
+    try:
+        _, ext = url.rsplit(".", 1)
+    except:
+        return fallback
 
-    return mimetypes.guess_type(url)
+    ret = MIMES.get(ext)
+
+    if not ret:
+        x = mimetypes.guess_type(url)
+        ret = "application/{}".format(x[1]) if x[1] else x[0]
+
+    if not ret:
+        ret = fallback
+
+    if ";" not in ret:
+        if ret.startswith("text/") or ret.endswith("/javascript"):
+            ret += "; charset=utf-8"
+
+    return ret
 
 
-def runcmd(*argv):
+def runcmd(argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
-    stdout = stdout.decode("utf-8")
-    stderr = stderr.decode("utf-8")
+    stdout = stdout.decode("utf-8", "replace")
+    stderr = stderr.decode("utf-8", "replace")
     return [p.returncode, stdout, stderr]
 
 
-def chkcmd(*argv):
-    ok, sout, serr = runcmd(*argv)
+def chkcmd(argv):
+    ok, sout, serr = runcmd(argv)
     if ok != 0:
         raise Exception(serr)
 
     return sout, serr
 
 
+def mchkcmd(argv, timeout=10):
+    if PY2:
+        with open(os.devnull, "wb") as f:
+            rv = sp.call(argv, stdout=f, stderr=f)
+    else:
+        rv = sp.call(argv, stdout=sp.DEVNULL, stderr=sp.DEVNULL, timeout=timeout)
+
+    if rv:
+        raise sp.CalledProcessError(rv, (argv[0], b"...", argv[-1]))
+
+
 def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
         rv = f.read(4)
-        try:
-            return struct.unpack(b"I", rv)[0]
-        except:
-            return struct.unpack("I", rv)[0]
+        return sunpack(b"I", rv)[0]
 
 
 def py_desc():

copyparty/web/baguettebox.js

@@ -0,0 +1,886 @@
+/*!
+ * baguetteBox.js
+ * @author  feimosi
+ * @version 1.11.1-mod
+ * @url https://github.com/feimosi/baguetteBox.js
+ */
+
+window.baguetteBox = (function () {
+    'use strict';
+
+    var options = {},
+        defaults = {
+            captions: true,
+            buttons: 'auto',
+            noScrollbars: false,
+            bodyClass: 'bbox-open',
+            titleTag: false,
+            async: false,
+            preload: 2,
+            animation: 'slideIn',
+            afterShow: null,
+            afterHide: null,
+            onChange: null,
+        },
+        overlay, slider, btnPrev, btnNext, btnHelp, btnRotL, btnRotR, btnSel, btnVmode, btnClose,
+        currentGallery = [],
+        currentIndex = 0,
+        isOverlayVisible = false,
+        touch = {},  // start-pos
+        touchFlag = false,  // busy
+        re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
+        re_v = /.+\.(webm|mp4)(\?|$)/i,
+        data = {},  // all galleries
+        imagesElements = [],
+        documentLastFocus = null,
+        isFullscreen = false,
+        vmute = false,
+        vloop = false,
+        vnext = false,
+        resume_mp = false;
+
+    var onFSC = function (e) {
+        isFullscreen = !!document.fullscreenElement;
+    };
+
+    var overlayClickHandler = function (e) {
+        if (e.target.id.indexOf('baguette-img') !== -1)
+            hideOverlay();
+    };
+
+    var touchstartHandler = function (e) {
+        touch.count = e.touches.length;
+        if (touch.count > 1)
+            touch.multitouch = true;
+
+        touch.startX = e.changedTouches[0].pageX;
+        touch.startY = e.changedTouches[0].pageY;
+    };
+    var touchmoveHandler = function (e) {
+        if (touchFlag || touch.multitouch)
+            return;
+
+        e.preventDefault ? e.preventDefault() : e.returnValue = false;
+        var touchEvent = e.touches[0] || e.changedTouches[0];
+        if (touchEvent.pageX - touch.startX > 40) {
+            touchFlag = true;
+            showPreviousImage();
+        } else if (touchEvent.pageX - touch.startX < -40) {
+            touchFlag = true;
+            showNextImage();
+        } else if (touch.startY - touchEvent.pageY > 100) {
+            hideOverlay();
+        }
+    };
+    var touchendHandler = function (e) {
+        touch.count--;
+        if (e && e.touches)
+            touch.count = e.touches.length;
+
+        if (touch.count <= 0)
+            touch.multitouch = false;
+
+        touchFlag = false;
+    };
+    var contextmenuHandler = function () {
+        touchendHandler();
+    };
+
+    var trapFocusInsideOverlay = function (e) {
+        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(e.target))) {
+            e.stopPropagation();
+            btnClose.focus();
+        }
+    };
+
+    function run(selector, userOptions) {
+        buildOverlay();
+        removeFromCache(selector);
+        return bindImageClickListeners(selector, userOptions);
+    }
+
+    function bindImageClickListeners(selector, userOptions) {
+        var galleryNodeList = QSA(selector);
+        var selectorData = {
+            galleries: [],
+            nodeList: galleryNodeList
+        };
+        data[selector] = selectorData;
+
+        [].forEach.call(galleryNodeList, function (galleryElement) {
+            var tagsNodeList = [];
+            if (galleryElement.tagName === 'A')
+                tagsNodeList = [galleryElement];
+            else
+                tagsNodeList = galleryElement.getElementsByTagName('a');
+
+            tagsNodeList = [].filter.call(tagsNodeList, function (element) {
+                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1)
+                    return re_i.test(element.href) || re_v.test(element.href);
+            });
+            if (!tagsNodeList.length)
+                return;
+
+            var gallery = [];
+            [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
+                var imageElementClickHandler = function (e) {
+                    if (ctrl(e))
+                        return true;
+
+                    e.preventDefault ? e.preventDefault() : e.returnValue = false;
+                    prepareOverlay(gallery, userOptions);
+                    showOverlay(imageIndex);
+                };
+                var imageItem = {
+                    eventHandler: imageElementClickHandler,
+                    imageElement: imageElement
+                };
+                bind(imageElement, 'click', imageElementClickHandler);
+                gallery.push(imageItem);
+            });
+            selectorData.galleries.push(gallery);
+        });
+
+        return selectorData.galleries;
+    }
+
+    function clearCachedData() {
+        for (var selector in data)
+            if (data.hasOwnProperty(selector))
+                removeFromCache(selector);
+    }
+
+    function removeFromCache(selector) {
+        if (!data.hasOwnProperty(selector))
+            return;
+
+        var galleries = data[selector].galleries;
+        [].forEach.call(galleries, function (gallery) {
+            [].forEach.call(gallery, function (imageItem) {
+                unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
+            });
+
+            if (currentGallery === gallery)
+                currentGallery = [];
+        });
+
+        delete data[selector];
+    }
+
+    function buildOverlay() {
+        overlay = ebi('bbox-overlay');
+        if (!overlay) {
+            var ctr = mknod('div');
+            ctr.innerHTML = (
+                '<div id="bbox-overlay" role="dialog">' +
+                '<div id="bbox-slider"></div>' +
+                '<button id="bbox-prev" class="bbox-btn" type="button" aria-label="Previous">&lt;</button>' +
+                '<button id="bbox-next" class="bbox-btn" type="button" aria-label="Next">&gt;</button>' +
+                '<div id="bbox-btns">' +
+                '<button id="bbox-help" type="button">?</button>' +
+                '<button id="bbox-rotl" type="button">↶</button>' +
+                '<button id="bbox-rotr" type="button">↷</button>' +
+                '<button id="bbox-tsel" type="button">sel</button>' +
+                '<button id="bbox-vmode" type="button" tt="a"></button>' +
+                '<button id="bbox-close" type="button" aria-label="Close">X</button>' +
+                '</div></div>'
+            );
+            overlay = ctr.firstChild;
+            QS('body').appendChild(overlay);
+            tt.att(overlay);
+        }
+        slider = ebi('bbox-slider');
+        btnPrev = ebi('bbox-prev');
+        btnNext = ebi('bbox-next');
+        btnHelp = ebi('bbox-help');
+        btnRotL = ebi('bbox-rotl');
+        btnRotR = ebi('bbox-rotr');
+        btnSel = ebi('bbox-tsel');
+        btnVmode = ebi('bbox-vmode');
+        btnClose = ebi('bbox-close');
+        bindEvents();
+    }
+
+    function halp() {
+        if (ebi('bbox-halp'))
+            return;
+
+        var list = [
+            ['<b># hotkey</b>', '<b># operation</b>'],
+            ['escape', 'close'],
+            ['left, J', 'previous file'],
+            ['right, L', 'next file'],
+            ['home', 'first file'],
+            ['end', 'last file'],
+            ['R', 'rotate (shift=ccw)'],
+            ['S', 'toggle file selection'],
+            ['space, P, K', 'video: play / pause'],
+            ['U', 'video: seek 10sec back'],
+            ['P', 'video: seek 10sec ahead'],
+            ['M', 'video: toggle mute'],
+            ['V', 'video: toggle loop'],
+            ['C', 'video: toggle auto-next'],
+            ['F', 'video: toggle fullscreen'],
+        ],
+            d = mknod('table'),
+            html = ['<tbody>'];
+
+        for (var a = 0; a < list.length; a++)
+            html.push('<tr><td>' + list[a][0] + '</td><td>' + list[a][1] + '</td></tr>');
+
+        d.innerHTML = html.join('\n') + '</tbody>';
+        d.setAttribute('id', 'bbox-halp');
+        d.onclick = function () {
+            overlay.removeChild(d);
+        };
+        overlay.appendChild(d);
+    }
+
+    function keyDownHandler(e) {
+        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
+            return;
+
+        var k = e.code + '', v = vid();
+
+        if (k == "ArrowLeft" || k == "KeyJ")
+            showPreviousImage();
+        else if (k == "ArrowRight" || k == "KeyL")
+            showNextImage();
+        else if (k == "Escape")
+            hideOverlay();
+        else if (k == "Home")
+            showFirstImage(e);
+        else if (k == "End")
+            showLastImage(e);
+        else if (k == "Space" || k == "KeyP" || k == "KeyK")
+            playpause();
+        else if (k == "KeyU" || k == "KeyO")
+            relseek(k == "KeyU" ? -10 : 10);
+        else if (k == "KeyM" && v) {
+            v.muted = vmute = !vmute;
+            mp_ctl();
+        }
+        else if (k == "KeyV" && v) {
+            vloop = !vloop;
+            vnext = vnext && !vloop;
+            setVmode();
+        }
+        else if (k == "KeyC" && v) {
+            vnext = !vnext;
+            vloop = vloop && !vnext;
+            setVmode();
+        }
+        else if (k == "KeyF")
+            try {
+                if (isFullscreen)
+                    document.exitFullscreen();
+                else
+                    v.requestFullscreen();
+            }
+            catch (ex) { }
+        else if (k == "KeyS")
+            tglsel();
+        else if (k == "KeyR")
+            rotn(e.shiftKey ? -1 : 1);
+    }
+
+    function setVmode() {
+        var v = vid();
+        ebi('bbox-vmode').style.display = v ? '' : 'none';
+        if (!v)
+            return;
+
+        var msg = 'When video ends, ', tts = '', lbl;
+        if (vloop) {
+            lbl = 'Loop';
+            msg += 'repeat it';
+            tts = '$NHotkey: V';
+        }
+        else if (vnext) {
+            lbl = 'Cont';
+            msg += 'continue to next';
+            tts = '$NHotkey: C';
+        }
+        else {
+            lbl = 'Stop';
+            msg += 'just stop'
+        }
+        btnVmode.setAttribute('aria-label', msg);
+        btnVmode.setAttribute('tt', msg + tts);
+        btnVmode.textContent = lbl;
+
+        v.loop = vloop
+        if (vloop && v.paused)
+            v.play();
+    }
+
+    function tglVmode() {
+        if (vloop) {
+            vnext = true;
+            vloop = false;
+        }
+        else if (vnext)
+            vnext = false;
+        else
+            vloop = true;
+
+        setVmode();
+        if (tt.en)
+            tt.show.bind(this)();
+    }
+
+    function tglsel() {
+        var thumb = currentGallery[currentIndex].imageElement,
+            name = vsplit(thumb.href)[1].split('?')[0],
+            files = msel.getall();
+
+        for (var a = 0; a < files.length; a++)
+            if (vsplit(files[a].vp)[1] == name)
+                clmod(ebi(files[a].id).closest('tr'), 'sel', 't');
+
+        msel.selui();
+        selbg();
+    }
+
+    function selbg() {
+        var img = vidimg(),
+            thumb = currentGallery[currentIndex].imageElement,
+            name = vsplit(thumb.href)[1].split('?')[0],
+            files = msel.getsel(),
+            sel = false;
+
+        for (var a = 0; a < files.length; a++)
+            if (vsplit(files[a].vp)[1] == name)
+                sel = true;
+
+        ebi('bbox-overlay').style.background = sel ?
+            'rgba(153,34,85,0.7)' : '';
+
+        img.style.borderRadius = sel ? '1em' : '';
+        btnSel.style.color = sel ? '#fff' : '';
+        btnSel.style.background = sel ? '#d48' : '';
+        btnSel.style.textShadow = sel ? '1px 1px 0 #b38' : '';
+        btnSel.style.boxShadow = sel ? '.15em .15em 0 #502' : '';
+    }
+
+    function keyUpHandler(e) {
+        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
+            return;
+
+        var k = e.code + '';
+
+        if (k == "Space")
+            ev(e);
+    }
+
+    var passiveSupp = false;
+    try {
+        var opts = {
+            get passive() {
+                passiveSupp = true;
+                return false;
+            }
+        };
+        window.addEventListener('test', null, opts);
+        window.removeEventListener('test', null, opts);
+    }
+    catch (ex) {
+        passiveSupp = false;
+    }
+    var passiveEvent = passiveSupp ? { passive: false } : null;
+    var nonPassiveEvent = passiveSupp ? { passive: true } : null;
+
+    function bindEvents() {
+        bind(overlay, 'click', overlayClickHandler);
+        bind(btnPrev, 'click', showPreviousImage);
+        bind(btnNext, 'click', showNextImage);
+        bind(btnClose, 'click', hideOverlay);
+        bind(btnVmode, 'click', tglVmode);
+        bind(btnHelp, 'click', halp);
+        bind(btnRotL, 'click', rotl);
+        bind(btnRotR, 'click', rotr);
+        bind(btnSel, 'click', tglsel);
+        bind(slider, 'contextmenu', contextmenuHandler);
+        bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        bind(overlay, 'touchend', touchendHandler);
+        bind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function unbindEvents() {
+        unbind(overlay, 'click', overlayClickHandler);
+        unbind(btnPrev, 'click', showPreviousImage);
+        unbind(btnNext, 'click', showNextImage);
+        unbind(btnClose, 'click', hideOverlay);
+        unbind(btnVmode, 'click', tglVmode);
+        unbind(btnHelp, 'click', halp);
+        unbind(btnRotL, 'click', rotl);
+        unbind(btnRotR, 'click', rotr);
+        unbind(btnSel, 'click', tglsel);
+        unbind(slider, 'contextmenu', contextmenuHandler);
+        unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        unbind(overlay, 'touchend', touchendHandler);
+        unbind(document, 'focus', trapFocusInsideOverlay, true);
+        timer.rm(rotn);
+    }
+
+    function prepareOverlay(gallery, userOptions) {
+        if (currentGallery === gallery)
+            return;
+
+        currentGallery = gallery;
+        setOptions(userOptions);
+        slider.innerHTML = '';
+        imagesElements.length = 0;
+
+        var imagesFiguresIds = [];
+        var imagesCaptionsIds = [];
+        for (var i = 0, fullImage; i < gallery.length; i++) {
+            fullImage = mknod('div');
+            fullImage.className = 'full-image';
+            fullImage.id = 'baguette-img-' + i;
+            imagesElements.push(fullImage);
+
+            imagesFiguresIds.push('bbox-figure-' + i);
+            imagesCaptionsIds.push('bbox-figcaption-' + i);
+            slider.appendChild(imagesElements[i]);
+        }
+        overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
+        overlay.setAttribute('aria-describedby', imagesCaptionsIds.join(' '));
+    }
+
+    function setOptions(newOptions) {
+        if (!newOptions)
+            newOptions = {};
+
+        for (var item in defaults) {
+            options[item] = defaults[item];
+            if (typeof newOptions[item] !== 'undefined')
+                options[item] = newOptions[item];
+        }
+        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
+            options.animation === 'slideIn' ? '' : 'none');
+
+        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1))
+            options.buttons = false;
+
+        btnPrev.style.display = btnNext.style.display = (options.buttons ? '' : 'none');
+    }
+
+    function showOverlay(chosenImageIndex) {
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'hidden';
+            document.body.style.overflowY = 'scroll';
+        }
+        if (overlay.style.display === 'block')
+            return;
+
+        bind(document, 'keydown', keyDownHandler);
+        bind(document, 'keyup', keyUpHandler);
+        bind(document, 'fullscreenchange', onFSC);
+        currentIndex = chosenImageIndex;
+        touch = {
+            count: 0,
+            startX: null,
+            startY: null
+        };
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+
+        updateOffset();
+        overlay.style.display = 'block';
+        // Fade in overlay
+        setTimeout(function () {
+            overlay.className = 'visible';
+            if (options.bodyClass && document.body.classList)
+                document.body.classList.add(options.bodyClass);
+
+            if (options.afterShow)
+                options.afterShow();
+        }, 50);
+
+        if (options.onChange)
+            options.onChange(currentIndex, imagesElements.length);
+
+        documentLastFocus = document.activeElement;
+        btnClose.focus();
+        isOverlayVisible = true;
+    }
+
+    function hideOverlay(e) {
+        ev(e);
+        playvid(false);
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'auto';
+            document.body.style.overflowY = 'auto';
+        }
+        if (overlay.style.display === 'none')
+            return;
+
+        unbind(document, 'keydown', keyDownHandler);
+        unbind(document, 'keyup', keyUpHandler);
+        unbind(document, 'fullscreenchange', onFSC);
+        // Fade out and hide the overlay
+        overlay.className = '';
+        setTimeout(function () {
+            overlay.style.display = 'none';
+            if (options.bodyClass && document.body.classList)
+                document.body.classList.remove(options.bodyClass);
+
+            qsr('#bbox-halp');
+
+            if (options.afterHide)
+                options.afterHide();
+
+            documentLastFocus && documentLastFocus.focus();
+            isOverlayVisible = false;
+        }, 500);
+    }
+
+    function loadImage(index, callback) {
+        var imageContainer = imagesElements[index];
+        var galleryItem = currentGallery[index];
+
+        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined')
+            return;  // out-of-bounds or gallery dirty
+
+        if (imageContainer.querySelector('img, video'))
+            // was loaded, cb and bail
+            return callback ? callback() : null;
+
+        // maybe unloaded video
+        while (imageContainer.firstChild)
+            imageContainer.removeChild(imageContainer.firstChild);
+
+        var imageElement = galleryItem.imageElement,
+            imageSrc = imageElement.href,
+            is_vid = re_v.test(imageSrc),
+            thumbnailElement = imageElement.querySelector('img, video'),
+            imageCaption = typeof options.captions === 'function' ?
+                options.captions.call(currentGallery, imageElement) :
+                imageElement.getAttribute('data-caption') || imageElement.title;
+
+        imageSrc += imageSrc.indexOf('?') < 0 ? '?cache' : '&cache';
+
+        if (is_vid && index != currentIndex)
+            return;  // no preload
+
+        var figure = mknod('figure');
+        figure.id = 'bbox-figure-' + index;
+        figure.innerHTML = '<div class="bbox-spinner">' +
+            '<div class="bbox-double-bounce1"></div>' +
+            '<div class="bbox-double-bounce2"></div>' +
+            '</div>';
+
+        if (options.captions && imageCaption) {
+            var figcaption = mknod('figcaption');
+            figcaption.id = 'bbox-figcaption-' + index;
+            figcaption.innerHTML = imageCaption;
+            figure.appendChild(figcaption);
+        }
+        imageContainer.appendChild(figure);
+
+        var image = mknod(is_vid ? 'video' : 'img');
+        clmod(imageContainer, 'vid', is_vid);
+
+        image.addEventListener(is_vid ? 'loadedmetadata' : 'load', function () {
+            // Remove loader element
+            qsr('#baguette-img-' + index + ' .bbox-spinner');
+            if (!options.async && callback)
+                callback();
+        });
+        image.setAttribute('src', imageSrc);
+        if (is_vid) {
+            image.setAttribute('controls', 'controls');
+            image.onended = vidEnd;
+        }
+        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (options.titleTag && imageCaption)
+            image.title = imageCaption;
+
+        figure.appendChild(image);
+
+        if (options.async && callback)
+            callback();
+    }
+
+    function showNextImage(e) {
+        ev(e);
+        return show(currentIndex + 1);
+    }
+
+    function showPreviousImage(e) {
+        ev(e);
+        return show(currentIndex - 1);
+    }
+
+    function showFirstImage(e) {
+        if (e)
+            e.preventDefault();
+
+        return show(0);
+    }
+
+    function showLastImage(e) {
+        if (e)
+            e.preventDefault();
+
+        return show(currentGallery.length - 1);
+    }
+
+    function show(index, gallery) {
+        if (!isOverlayVisible && index >= 0 && index < gallery.length) {
+            prepareOverlay(gallery, options);
+            showOverlay(index);
+            return true;
+        }
+        if (index < 0) {
+            if (options.animation)
+                bounceAnimation('left');
+
+            return false;
+        }
+        if (index >= imagesElements.length) {
+            if (options.animation)
+                bounceAnimation('right');
+
+            return false;
+        }
+
+        var v = vid();
+        if (v) {
+            v.src = '';
+            v.load();
+            v.parentNode.removeChild(v);
+        }
+
+        currentIndex = index;
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+        updateOffset();
+
+        if (options.onChange)
+            options.onChange(currentIndex, imagesElements.length);
+
+        return true;
+    }
+
+    var prev_cw = 0, prev_ch = 0, unrot_timer = null;
+    function rotn(n) {
+        var el = vidimg(),
+            orot = parseInt(el.getAttribute('rot') || 0),
+            frot = orot + (n || 0) * 90;
+
+        if (!frot && !orot)
+            return;  // reflow noop
+
+        var co = ebi('bbox-overlay'),
+            cw = co.clientWidth,
+            ch = co.clientHeight;
+
+        if (!n && prev_cw === cw && prev_ch === ch)
+            return;  // reflow noop
+
+        prev_cw = cw;
+        prev_ch = ch;
+        var rot = frot,
+            iw = el.naturalWidth || el.videoWidth,
+            ih = el.naturalHeight || el.videoHeight,
+            magic = 4,  // idk, works in enough browsers
+            dl = el.closest('div').querySelector('figcaption a'),
+            vw = cw,
+            vh = ch - dl.offsetHeight + magic,
+            pmag = Math.min(1, Math.min(vw / ih, vh / iw)),
+            wmag = Math.min(1, Math.min(vw / iw, vh / ih));
+
+        while (rot < 0) rot += 360;
+        while (rot >= 360) rot -= 360;
+        var q = rot == 90 || rot == 270 ? 1 : 0,
+            mag = q ? pmag : wmag;
+
+        el.style.cssText = 'max-width:none; max-height:none; position:absolute; display:block; margin:0';
+        if (!orot) {
+            el.style.width = iw * wmag + 'px';
+            el.style.height = ih * wmag + 'px';
+            el.style.left = (vw - iw * wmag) / 2 + 'px';
+            el.style.top = (vh - ih * wmag) / 2 - magic + 'px';
+            q = el.offsetHeight;
+        }
+        el.style.width = iw * mag + 'px';
+        el.style.height = ih * mag + 'px';
+        el.style.left = (vw - iw * mag) / 2 + 'px';
+        el.style.top = (vh - ih * mag) / 2 - magic + 'px';
+        el.style.transform = 'rotate(' + frot + 'deg)';
+        el.setAttribute('rot', frot);
+        timer.add(rotn);
+        if (!rot) {
+            clearTimeout(unrot_timer);
+            unrot_timer = setTimeout(unrot, 300);
+        }
+    }
+    function rotl() {
+        rotn(-1);
+    }
+    function rotr() {
+        rotn(1);
+    }
+    function unrot() {
+        var el = vidimg(),
+            orot = el.getAttribute('rot'),
+            rot = parseInt(orot || 0);
+
+        while (rot < 0) rot += 360;
+        while (rot >= 360) rot -= 360;
+        if (rot || orot === null)
+            return;
+
+        clmod(el, 'nt', 1);
+        el.removeAttribute('rot');
+        el.removeAttribute("style");
+        rot = el.offsetHeight;
+        clmod(el, 'nt');
+        timer.rm(rotn);
+    }
+
+    function vid() {
+        return imagesElements[currentIndex].querySelector('video');
+    }
+
+    function vidimg() {
+        return imagesElements[currentIndex].querySelector('img, video');
+    }
+
+    function playvid(play) {
+        if (vid())
+            vid()[play ? 'play' : 'pause']();
+    }
+
+    function playpause() {
+        var v = vid();
+        if (v)
+            v[v.paused ? "play" : "pause"]();
+    }
+
+    function relseek(sec) {
+        if (vid())
+            vid().currentTime += sec;
+    }
+
+    function vidEnd() {
+        if (this == vid() && vnext)
+            showNextImage();
+    }
+
+    function mp_ctl() {
+        var v = vid();
+        if (!vmute && v && mp.au && !mp.au.paused) {
+            mp.fade_out();
+            resume_mp = true;
+        }
+        else if (resume_mp && (vmute || !v) && mp.au && mp.au.paused) {
+            mp.fade_in();
+            resume_mp = false;
+        }
+    }
+
+    function bounceAnimation(direction) {
+        slider.className = 'bounce-from-' + direction;
+        setTimeout(function () {
+            slider.className = '';
+        }, 400);
+    }
+
+    function updateOffset() {
+        var offset = -currentIndex * 100 + '%',
+            xform = slider.style.perspective !== undefined;
+
+        if (options.animation === 'fadeIn') {
+            slider.style.opacity = 0;
+            setTimeout(function () {
+                xform ?
+                    slider.style.transform = 'translate3d(' + offset + ',0,0)' :
+                    slider.style.left = offset;
+                slider.style.opacity = 1;
+            }, 400);
+        } else {
+            xform ?
+                slider.style.transform = 'translate3d(' + offset + ',0,0)' :
+                slider.style.left = offset;
+        }
+        playvid(false);
+        var v = vid();
+        if (v) {
+            playvid(true);
+            v.muted = vmute;
+            v.loop = vloop;
+        }
+        selbg();
+        mp_ctl();
+        setVmode();
+
+        var el = vidimg();
+        if (el.getAttribute('rot'))
+            timer.add(rotn);
+        else
+            timer.rm(rotn);
+
+        var prev = QS('.full-image.vis');
+        if (prev)
+            clmod(prev, 'vis');
+
+        clmod(el.closest('div'), 'vis', 1);
+    }
+
+    function preloadNext(index) {
+        if (index - currentIndex >= options.preload)
+            return;
+
+        loadImage(index + 1, function () {
+            preloadNext(index + 1);
+        });
+    }
+
+    function preloadPrev(index) {
+        if (currentIndex - index >= options.preload)
+            return;
+
+        loadImage(index - 1, function () {
+            preloadPrev(index - 1);
+        });
+    }
+
+    function bind(element, event, callback, options) {
+        element.addEventListener(event, callback, options);
+    }
+
+    function unbind(element, event, callback, options) {
+        element.removeEventListener(event, callback, options);
+    }
+
+    function destroyPlugin() {
+        unbindEvents();
+        clearCachedData();
+        unbind(document, 'keydown', keyDownHandler);
+        unbind(document, 'keyup', keyUpHandler);
+        document.getElementsByTagName('body')[0].removeChild(ebi('bbox-overlay'));
+        data = {};
+        currentGallery = [];
+        currentIndex = 0;
+    }
+
+    return {
+        run: run,
+        show: show,
+        showNext: showNextImage,
+        showPrevious: showPreviousImage,
+        relseek: relseek,
+        playpause: playpause,
+        hide: hideOverlay,
+        destroy: destroyPlugin
+    };
+})();

copyparty/web/browser.html

@@ -2,144 +2,157 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <title>⇆🎉 {{ title }}</title>
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
+	<meta charset="utf-8">
+	<title>⇆🎉 {{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
+	{%- if css %}
+	<link rel="stylesheet" media="screen" href="{{ css }}?_={{ ts }}">
+	{%- endif %}
 </head>
 
 <body>
-    <div id="ops">
-        <a href="#" data-dest="" data-desc="close submenu">---</a>
-        {%- if have_up2k_idx %}
-        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
-        <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
-        {%- else %}
-        <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
-        {%- endif %}
-        <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
-        <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
-        <a href="#" data-perm="write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
-        <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
-        <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
-        <div id="opdesc"></div>
-    </div>
+	<div id="ops"></div>
 
-    <div id="op_search" class="opview">
-        {%- if have_tags_idx %}
-        <div id="srch_form" class="tags"></div>
-        {%- else %}
-        <div id="srch_form"></div>
-        {%- endif %}
-        <div id="srch_q"></div>
-    </div>
+	<div id="op_search" class="opview">
+		{%- if have_tags_idx %}
+		<div id="srch_form" class="tags opbox"></div>
+		{%- else %}
+		<div id="srch_form" class="opbox"></div>
+		{%- endif %}
+		<div id="srch_q"></div>
+	</div>
 
-    {%- include 'upload.html' %}
+	<div id="op_player" class="opview opbox opwide"></div>
 
-    <div id="op_cfg" class="opview opbox">
-        <h3>switches</h3>
-        <div>
-            <a id="tooltips" class="tglbtn" href="#">tooltips</a>
-            <a id="lightmode" class="tglbtn" href="#">lightmode</a>
-        </div>
-        {%- if have_zip %}
-        <h3>folder download</h3>
-        <div id="arc_fmt"></div>
-        {%- endif %}
-        <h3>key notation</h3>
-        <div id="key_notation"></div>
-    </div>
-    
-    <h1 id="path">
-        <a href="#" id="entree">🌲</a>
-        {%- for n in vpnodes %}
-        <a href="/{{ n[0] }}">{{ n[1] }}</a>
-        {%- endfor %}
-    </h1>
-    
-    <div id="tree">
-        <a href="#" id="detree">🍞...</a>
-        <a href="#" step="2" id="twobytwo">+</a>
-        <a href="#" step="-2" id="twig">&ndash;</a>
-        <a href="#" class="tglbtn" id="dyntree">a</a>
-        <ul id="treeul"></ul>
-        <div id="thx_ff">&nbsp;</div>
-    </div>
+	<div id="op_bup" class="opview opbox act">
+		<div id="u2err"></div>
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="bput" />
+			<input type="file" name="f" multiple /><br />
+			<input type="submit" value="start upload">
+		</form>
+	</div>
+
+	<div id="op_mkdir" class="opview opbox act">
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="mkdir" />
+			📂<input type="text" name="name" class="i">
+			<input type="submit" value="make directory">
+		</form>
+	</div>
+
+	<div id="op_new_md" class="opview opbox">
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="new_md" />
+			📝<input type="text" name="name" class="i">
+			<input type="submit" value="new markdown doc">
+		</form>
+	</div>
+
+	<div id="op_msg" class="opview opbox act">
+		<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
+			📟<input type="text" name="msg" class="i">
+			<input type="submit" value="send msg to srv log">
+		</form>
+	</div>
+
+	<div id="op_unpost" class="opview opbox"></div>
+
+	<div id="op_up2k" class="opview"></div>
+
+	<div id="op_cfg" class="opview opbox opwide"></div>
+	
+	<h1 id="path">
+		<a href="#" id="entree" tt="show navpane (directory tree sidebar)$NHotkey: B">🌲</a>
+		{%- for n in vpnodes %}
+		<a href="/{{ n[0] }}">{{ n[1] }}</a>
+		{%- endfor %}
+	</h1>
+	
+	<div id="tree"></div>
 
 <div id="wrap">
 
-    <div id="pro" class="logue">{{ logues[0] }}</div>
+	{%- if doc %}
+	<div id="bdoc"><pre>{{ doc|e }}</pre></div>
+	{%- else %}
+	<div id="bdoc"></div>
+	{%- endif %}
 
-    <table id="files">
-        <thead>
-            <tr>
-                <th name="lead"><span>c</span></th>
-                <th name="href"><span>File Name</span></th>
-                <th name="sz" sort="int"><span>Size</span></th>
-                {%- for k in taglist %}
-                    {%- if k.startswith('.') %}
-                        <th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
-                    {%- else %}
-                        <th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
-                    {%- endif %}
-                {%- endfor %}
-                <th name="ext"><span>T</span></th>
-                <th name="ts"><span>Date</span></th>
-            </tr>
-        </thead>
-        <tbody>
+	<div id="pro" class="logue">{{ logues[0] }}</div>
+
+	<table id="files">
+		<thead>
+			<tr>
+				<th name="lead"><span>c</span></th>
+				<th name="href"><span>File Name</span></th>
+				<th name="sz" sort="int"><span>Size</span></th>
+				{%- for k in taglist %}
+					{%- if k.startswith('.') %}
+				<th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
+					{%- else %}
+				<th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
+					{%- endif %}
+				{%- endfor %}
+				<th name="ext"><span>T</span></th>
+				<th name="ts"><span>Date</span></th>
+			</tr>
+		</thead>
+<tbody>
 
 {%- for f in files %}
-    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td>
-    {%- if f.tags is defined %}
-        {%- for k in taglist %}
-            <td>{{ f.tags[k] }}</td>
-        {%- endfor %}
-    {%- endif %}
-    <td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td>
+	{%- if f.tags is defined %}
+		{%- for k in taglist %}
+<td>{{ f.tags[k] }}</td>
+		{%- endfor %}
+	{%- endif %}
+<td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
-        </tbody>
-    </table>
-    
-    <div id="epi" class="logue">{{ logues[1] }}</div>
+		</tbody>
+	</table>
+	
+	<div id="epi" class="logue">{{ logues[1] }}</div>
 
-    <h2><a href="?h">control-panel</a></h2>
+	<h2><a href="/?h">control-panel</a></h2>
+	
+	<a href="#" id="repl">π</a>
 
 </div>
 
-    {%- if srv_info %}
-    <div id="srv_info"><span>{{ srv_info }}</span></div>
-    {%- endif %}
+	{%- if srv_info %}
+	<div id="srv_info"><span>{{ srv_info }}</span></div>
+	{%- endif %}
 
-    <div id="widget">
-        <div id="wtoggle">
-            <span id="wzip">
-                <a href="#" id="selall">sel.<br />all</a>
-                <a href="#" id="selinv">sel.<br />inv.</a>
-                <a href="#" id="selzip">zip</a>
-            </span><a
-                href="#" id="wtico">♫</a>
-        </div>
-        <div id="widgeti">
-            <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
-            <canvas id="pvol" width="288" height="38"></canvas>
-            <canvas id="barpos"></canvas>
-            <canvas id="barbuf"></canvas>
-        </div>
-    </div>
+	<div id="widget"></div>
 
-    <script>
-        var tag_order_cfg = {{ tag_order }};
-    </script>
-    <script src="/.cpr/util.js{{ ts }}"></script>
-    <script src="/.cpr/browser.js{{ ts }}"></script>
-    <script src="/.cpr/up2k.js{{ ts }}"></script>
-    <script>
-        apply_perms({{ perms }});
-    </script>
+	<script>
+		var acct = "{{ acct }}",
+			perms = {{ perms }},
+			def_hcols = {{ def_hcols|tojson }},
+			have_up2k_idx = {{ have_up2k_idx|tojson }},
+			have_tags_idx = {{ have_tags_idx|tojson }},
+			have_acode = {{ have_acode|tojson }},
+			have_mv = {{ have_mv|tojson }},
+			have_del = {{ have_del|tojson }},
+			have_unpost = {{ have_unpost|tojson }},
+			have_zip = {{ have_zip|tojson }},
+			txt_ext = "{{ txt_ext }}",
+			{% if no_prism %}no_prism = 1,{% endif %}
+			readme = {{ readme|tojson }};
+
+		document.documentElement.setAttribute("class", localStorage.lightmode == 1 ? "light" : "dark");
+	</script>
+	<script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/browser.js?_={{ ts }}"></script>
+	<script src="/.cpr/up2k.js?_={{ ts }}"></script>
+	{%- if js %}
+	<script src="{{ js }}?_={{ ts }}"></script>
+	{%- endif %}
 </body>
 
 </html>

copyparty/web/browser2.html

@@ -2,54 +2,59 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <title>{{ title }}</title>
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<meta charset="utf-8">
+	<title>{{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<style>
+		html{font-family:sans-serif}
+		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
+		a{display:block}
+	</style>
 </head>
 
 <body>
-    {%- if srv_info %}
-    <p><span>{{ srv_info }}</span></p>
-    {%- endif %}
+	{%- if srv_info %}
+	<p><span>{{ srv_info }}</span></p>
+	{%- endif %}
 
-    {%- if have_b_u %}
-    <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
-        <input type="hidden" name="act" value="bput" />
-        <input type="file" name="f" multiple /><br />
-        <input type="submit" value="start upload" />
-    </form>
-    <br />
-    {%- endif %}
+	{%- if have_b_u %}
+	<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+		<input type="hidden" name="act" value="bput" />
+		<input type="file" name="f" multiple /><br />
+		<input type="submit" value="start upload" />
+	</form>
+	<br />
+	{%- endif %}
 
-    {%- if logues[0] %}
-    <div>{{ logues[0] }}</div><br />
-    {%- endif %}
+	{%- if logues[0] %}
+	<div>{{ logues[0] }}</div><br />
+	{%- endif %}
 
-    <table id="files">
-        <thead>
-            <tr>
-                <th name="lead"><span>c</span></th>
-                <th name="href"><span>File Name</span></th>
-                <th name="sz" sort="int"><span>Size</span></th>
-                <th name="ts"><span>Date</span></th>
-            </tr>
-        </thead>
-        <tbody>
-            <tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+	<table id="files">
+		<thead>
+			<tr>
+				<th name="lead"><span>c</span></th>
+				<th name="href"><span>File Name</span></th>
+				<th name="sz" sort="int"><span>Size</span></th>
+				<th name="ts"><span>Date</span></th>
+			</tr>
+		</thead>
+		<tbody>
+<tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
 
 {%- for f in files %}
-    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
-        </tbody>
-    </table>
-    
-    {%- if logues[1] %}
-    <div>{{ logues[1] }}</div><br />
-    {%- endif %}
-    
-    <h2><a href="{{ url_suf }}&amp;h">control-panel</a></h2>
+		</tbody>
+	</table>
+	
+	{%- if logues[1] %}
+	<div>{{ logues[1] }}</div><br />
+	{%- endif %}
+	
+	<h2><a href="/{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
 
 </body>
 </html>

copyparty/web/dbg-audio.js

@@ -0,0 +1,61 @@
+var ofun = audio_eq.apply.bind(audio_eq);
+audio_eq.apply = function () {
+    var ac1 = mp.ac;
+    ofun();
+    var ac = mp.ac,
+        w = 2048,
+        h = 256;
+
+    if (!audio_eq.filters.length) {
+        audio_eq.ana = null;
+        return;
+    }
+
+    var can = ebi('fft_can');
+    if (!can) {
+        can = mknod('canvas');
+        can.setAttribute('id', 'fft_can');
+        can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
+        document.body.appendChild(can);
+        can.width = w;
+        can.height = h;
+    }
+    var cc = can.getContext('2d');
+    if (!ac)
+        return;
+
+    var ana = ac.createAnalyser();
+    ana.smoothingTimeConstant = 0;
+    ana.fftSize = 8192;
+
+    audio_eq.filters[0].connect(ana);
+    audio_eq.ana = ana;
+
+    var buf = new Uint8Array(ana.frequencyBinCount),
+        colw = can.width / buf.length;
+
+    cc.fillStyle = '#fc0';
+    function draw() {
+        if (ana == audio_eq.ana)
+            requestAnimationFrame(draw);
+
+        ana.getByteFrequencyData(buf);
+
+        cc.clearRect(0, 0, can.width, can.height);
+
+        /*var x = 0, w = 1;
+        for (var a = 0; a < buf.length; a++) {
+            cc.fillRect(x, h - buf[a], w, h);
+            x += w;
+        }*/
+        var mul = Math.pow(w, 4) / buf.length;
+        for (var x = 0; x < w; x++) {
+            var a = Math.floor(Math.pow(x, 4) / mul),
+                v = buf[a];
+
+            cc.fillRect(x, h - v, 1, v);
+        }
+    }
+    draw();
+};
+audio_eq.apply();

copyparty/web/md.css

@@ -1,13 +1,17 @@
-@font-face {
-	font-family: 'scp';
-	src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(/.cpr/deps/scp.woff2) format('woff2');
-}
 html, body {
 	color: #333;
 	background: #eee;
 	font-family: sans-serif;
 	line-height: 1.5em;
 }
+#repl {
+	position: absolute;
+	top: 0;
+	right: .5em;
+	border: none;
+	color: inherit;
+	background: none;
+}
 #mtw {
 	display: none;
 }
@@ -15,119 +19,12 @@ html, body {
 	margin: 0 auto;
 	padding: 0 1.5em;
 }
-pre, code, a {
-	color: #480;
-	background: #f7f7f7;
-	border: .07em solid #ddd;
-	border-radius: .2em;
-	padding: .1em .3em;
-	margin: 0 .1em;
+#toast {
+	bottom: auto;
+	top: 1.4em;
 }
-code {
-	font-size: .96em;
-}
-pre, code {
-	font-family: 'scp', monospace, monospace;
-	white-space: pre-wrap;
-	word-break: break-all;
-}
-pre {
-	counter-reset: precode;
-}
-pre code {
-	counter-increment: precode;
-	display: inline-block;
-	margin: 0 -.3em;
-	padding: .4em .5em;
-	border: none;
-	border-bottom: 1px solid #cdc;
-	min-width: calc(100% - .6em);
-	line-height: 1.1em;
-}
-pre code:last-child {
-	border-bottom: none;
-}
-pre code::before {
-	content: counter(precode);
-	-webkit-user-select: none;
-	display: inline-block;
-	text-align: right;
-	font-size: .75em;
-	color: #48a;
-	width: 4em;
-	padding-right: 1.5em;
-	margin-left: -5.5em;
-}
-pre code:hover {
-	background: #fec;
-	color: #360;
-}
-h1, h2 {
-	line-height: 1.5em;
-}
-h1 {
-	font-size: 1.7em;
-	text-align: center;
-	border: 1em solid #777;
-	border-width: .05em 0;
-	margin: 3em 0;
-}
-h2 {
-	font-size: 1.5em;
-	font-weight: normal;
-	background: #f7f7f7;
-	border-top: .07em solid #fff;
-	border-bottom: .07em solid #bbb;
-	border-radius: .5em .5em 0 0;
-	padding-left: .4em;
-	margin-top: 3em;
-}
-h3 {
-	border-bottom: .1em solid #999;
-}
-h1 a, h3 a, h5 a,
-h2 a, h4 a, h6 a {
-	color: inherit;
-	display: block;
-	background: none;
-	border: none;
-	padding: 0;
-	margin: 0;
-}
-#mp ul,
-#mp ol {
-	border-left: .3em solid #ddd;
-}
-#m>ul,
-#m>ol {
-	border-color: #bbb;
-}
-#mp ul>li {
-	list-style-type: disc;
-}
-#mp ul>li,
-#mp ol>li {
-	margin: .7em 0;
-}
-strong {
-	color: #000;
-}
-p>em,
-li>em,
-td>em {
-	color: #c50;
-	padding: .1em;
-	border-bottom: .1em solid #bbb;
-}
-blockquote {
-	font-family: serif;
-	background: #f7f7f7;
-	border: .07em dashed #ccc;
-	padding: 0 2em;
-	margin: 1em 0;
-}
-small {
-	opacity: .8;
+a {
+	text-decoration: none;
 }
 #toc {
 	margin: 0 1em;
@@ -163,7 +60,7 @@ small {
 	z-index: 99;
 	position: relative;
 	display: inline-block;
-	font-family: monospace, monospace;
+	font-family: 'scp', monospace, monospace;
 	font-weight: bold;
 	font-size: 1.3em;
 	line-height: .1em;
@@ -175,14 +72,6 @@ small {
 	color: #6b3;
 	text-shadow: .02em 0 0 #6b3;
 }
-table {
-	border-collapse: collapse;
-	margin: 1em 0;
-}
-th, td {
-	padding: .2em .5em;
-	border: .12em solid #aaa;
-}
 blink {
 	animation: blinker .7s cubic-bezier(.9, 0, .1, 1) infinite;
 }
@@ -195,6 +84,36 @@ blink {
 	}
 }
 
+
+.mdo pre {
+	counter-reset: precode;
+}
+.mdo pre code {
+	counter-increment: precode;
+	display: inline-block;
+	border: none;
+	border-bottom: 1px solid #cdc;
+	min-width: calc(100% - .6em);
+}
+.mdo pre code:last-child {
+	border-bottom: none;
+}
+.mdo pre code::before {
+	content: counter(precode);
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+	display: inline-block;
+	text-align: right;
+	font-size: .75em;
+	color: #48a;
+	width: 4em;
+	padding-right: 1.5em;
+	margin-left: -5.5em;
+}
+
+
 @media screen {
 	html, body {
 		margin: 0;
@@ -211,34 +130,6 @@ blink {
 	#mp {
 		max-width: 52em;
 		margin-bottom: 6em;
-		word-break: break-word;
-		overflow-wrap: break-word;
-		word-wrap: break-word; /*ie*/
-	}
-	a {
-		color: #fff;
-		background: #39b;
-		text-decoration: none;
-		padding: 0 .3em;
-		border: none;
-		border-bottom: .07em solid #079;
-	}
-	h2 {
-		color: #fff;
-		background: #555;
-		margin-top: 2em;
-		border-bottom: .22em solid #999;
-		border-top: none;
-	}
-	h1 {
-		color: #fff;
-		background: #444;
-		font-weight: normal;
-		border-top: .4em solid #fb0;
-		border-bottom: .4em solid #777;
-		border-radius: 0 1em 0 1em;
-		margin: 3em 0 1em 0;
-		padding: .5em 0;
 	}
 	#mn {
 		padding: 1.3em 0 .7em 1em;
@@ -291,6 +182,8 @@ blink {
 		color: #444;
 		background: none;
 		text-decoration: underline;
+		margin: 0 .1em;
+		padding: 0 .3em;
 		border: none;
 	}
 	#mh a:hover {
@@ -319,6 +212,10 @@ blink {
 	#toolsbox a+a {
 		text-decoration: none;
 	}
+	#lno {
+		position: absolute;
+		right: 0;
+	}
 
 
 
@@ -339,55 +236,6 @@ blink {
 	html.dark #toc li {
 		border-width: 0;
 	}
-	html.dark #mp a {
-		background: #057;
-	}
-	html.dark #mp h1 a, html.dark #mp h4 a,
-	html.dark #mp h2 a, html.dark #mp h5 a,
-	html.dark #mp h3 a, html.dark #mp h6 a {
-		color: inherit;
-		background: none;
-	}
-	html.dark pre,
-	html.dark code {
-		color: #8c0;
-		background: #1a1a1a;
-		border: .07em solid #333;
-	}
-	html.dark #mp ul,
-	html.dark #mp ol {
-		border-color: #444;
-	}
-	html.dark #m>ul,
-	html.dark #m>ol {
-		border-color: #555;
-	}
-	html.dark strong {
-		color: #fff;
-	}
-	html.dark p>em,
-	html.dark li>em,
-	html.dark td>em {
-		color: #f94;
-		border-color: #666;
-	}
-	html.dark h1 {
-		background: #383838;
-		border-top: .4em solid #b80;
-		border-bottom: .4em solid #4c4c4c;
-	}
-	html.dark h2 {
-		background: #444;
-		border-bottom: .22em solid #555;
-	}
-	html.dark td,
-	html.dark th {
-		border-color: #444;
-	}
-	html.dark blockquote {
-		background: #282828;
-		border: .07em dashed #444;
-	}
 	html.dark #mn a:not(:last-child)::after {
 		border-color: rgba(255,255,255,0.3);
 	}
@@ -493,12 +341,15 @@ blink {
 		mso-footer-margin: .6in;
 		mso-paper-source: 0;
 	}
-	a {
+	.mdo a {
 		color: #079;
 		text-decoration: none;
 		border-bottom: .07em solid #4ac;
 		padding: 0 .3em;
 	}
+	#repl {
+		display: none;
+	}
 	#toc>ul {
 		border-left: .1em solid #84c4dd;
 	}
@@ -523,18 +374,20 @@ blink {
 	a[ctr]::before {
 		content: attr(ctr) '. ';
 	}
-	h1 {
+	.mdo h1 {
 		margin: 2em 0;
 	}
-	h2 {
+	.mdo h2 {
 		margin: 2em 0 0 0;
 	}
-	h1, h2, h3 {
+	.mdo h1,
+	.mdo h2,
+	.mdo h3 {
 		page-break-inside: avoid;
 	}
-	h1::after,
-	h2::after,
-	h3::after {
+	.mdo h1::after,
+	.mdo h2::after,
+	.mdo h3::after {
 		content: 'orz';
 		color: transparent;
 		display: block;
@@ -542,20 +395,20 @@ blink {
 		padding: 4em 0 0 0;
 		margin: 0 0 -5em 0;
 	}
-	p {
+	.mdo p {
 		page-break-inside: avoid;
 	}
-	table {
+	.mdo table {
 		page-break-inside: auto;
 	}
-	tr {
+	.mdo tr {
 		page-break-inside: avoid;
 		page-break-after: auto;
 	}
-	thead {
+	.mdo thead {
 		display: table-header-group;
 	}
-	tfoot {
+	.mdo tfoot {
 		display: table-footer-group;
 	}
 	#mp a.vis::after {
@@ -563,40 +416,32 @@ blink {
 		border-bottom: 1px solid #bbb;
 		color: #444;
 	}
-	blockquote {
+	.mdo blockquote {
 		border-color: #555;
 	}
-	code {
+	.mdo code {
 		border-color: #bbb;
 	}
-	pre, pre code {
+	.mdo pre,
+	.mdo pre code {
 		border-color: #999;
 	}
-	pre code::before {
+	.mdo pre code::before {
 		color: #058;
 	}
 
 
 	
-	html.dark a {
+	html.dark .mdo a {
 		color: #000;
 	}
-	html.dark pre,
-	html.dark code {
+	html.dark .mdo pre,
+	html.dark .mdo code {
 		color: #240;
 	}
-	html.dark p>em,
-	html.dark li>em,
-	html.dark td>em {
+	html.dark .mdo p>em,
+	html.dark .mdo li>em,
+	html.dark .mdo td>em {
 		color: #940;
 	}
 }
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-*/

copyparty/web/md.html

@@ -1,11 +1,12 @@
 <!DOCTYPE html><html><head>
 	<meta charset="utf-8">
-	<title>📝🎉 {{ title }}</title> <!-- 📜 -->
+	<title>📝🎉 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
-	<link href="/.cpr/md.css" rel="stylesheet">
+	<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/md.css?_={{ ts }}">
 	{%- if edit %}
-	<link href="/.cpr/md2.css" rel="stylesheet">
+	<link rel="stylesheet" href="/.cpr/md2.css?_={{ ts }}">
 	{%- endif %}
 </head>
 <body>
@@ -14,9 +15,9 @@
 		<a id="lightswitch" href="#">go dark</a>
 		<a id="navtoggle" href="#">hide nav</a>
 		{%- if edit %}
-			<a id="save" href="?edit">save</a>
-			<a id="sbs" href="#">sbs</a>
-			<a id="nsbs" href="#">editor</a>
+			<a id="save" href="{{ arg_base }}edit" tt="Hotkey: ctrl-s">save</a>
+			<a id="sbs" href="#" tt="editor and preview side by side">sbs</a>
+			<a id="nsbs" href="#" tt="switch between editor and preview$NHotkey: ctrl-e">editor</a>
 			<div id="toolsbox">
 				<a id="tools" href="#">tools</a>
 				<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
@@ -25,10 +26,11 @@
 				<a id="cfg_uni" href="#">non-ascii: whitelist</a>
 				<a id="help" href="#">help</a>
 			</div>
+			<span id="lno">L#</span>
 		{%- else %}
-			<a href="?edit">edit (basic)</a>
-			<a href="?edit2">edit (fancy)</a>
-			<a href="?raw">view raw</a>
+			<a href="{{ arg_base }}edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
+			<a href="{{ arg_base }}edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
+			<a href="{{ arg_base }}raw">view raw</a>
 		{%- endif %}
 	</div>
 	<div id="toc"></div>
@@ -42,8 +44,9 @@
 				if you're still reading this, check that javascript is allowed
 			</div>
 		</div>
-		<div id="mp"></div>
+		<div id="mp" class="mdo"></div>
 	</div>
+	<a href="#" id="repl">π</a>
 	
 	{%- if edit %}
 	<div id="helpbox">
@@ -131,25 +134,25 @@ var md_opt = {
 };
 
 (function () {
-    var btn = document.getElementById("lightswitch");
-    var toggle = function (e) {
-		if (e) e.preventDefault();
-        var dark = !document.documentElement.getAttribute("class");
-        document.documentElement.setAttribute("class", dark ? "dark" : "");
-        btn.innerHTML = "go " + (dark ? "light" : "dark");
-        if (window.localStorage)
-            localStorage.setItem('lightmode', dark ? 0 : 1);
-    };
-    btn.onclick = toggle;
-    if (window.localStorage && localStorage.getItem('lightmode') != 1)
-		toggle();
+    var l = localStorage,
+		drk = l.lightmode != 1,
+		btn = document.getElementById("lightswitch"),
+		f = function (e) {
+if (e) { e.preventDefault(); drk = !drk; }
+document.documentElement.setAttribute("class", drk? "dark":"light");
+btn.innerHTML = "go " + (drk ? "light":"dark");
+l.lightmode = drk? 0:1;
+    	};
+	
+	btn.onclick = f;
+	f();
 })();
 
 	</script>
-    <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.js"></script>
-	<script src="/.cpr/md.js"></script>
+    <script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
+	<script src="/.cpr/md.js?_={{ ts }}"></script>
 	{%- if edit %}
-	<script src="/.cpr/md2.js"></script>
+	<script src="/.cpr/md2.js?_={{ ts }}"></script>
 	{%- endif %}
 </body></html>

copyparty/web/md.js

@@ -24,29 +24,12 @@ var dbg = function () { };
 var md_plug = {};
 
 
-function hesc(txt) {
-    return txt.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-}
-
-
-function cls(dom, name, add) {
-    var re = new RegExp('(^| )' + name + '( |$)');
-    var lst = (dom.getAttribute('class') + '').replace(re, "$1$2").replace(/  /, "");
-    dom.setAttribute('class', lst + (add ? ' ' + name : ''));
-}
-
-
-function statify(obj) {
-    return JSON.parse(JSON.stringify(obj));
-}
-
-
 // dodge browser issues
 (function () {
     var ua = navigator.userAgent;
     if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
         // necessary on ff-68.7 at least
-        var s = document.createElement('style');
+        var s = mknod('style');
         s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
         console.log(s.innerHTML);
         document.head.appendChild(s);
@@ -65,7 +48,7 @@ function statify(obj) {
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = hesc(uricom_dec(n[a])[0]);
+        var dec = esc(uricom_dec(n[a])[0]);
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -73,6 +56,26 @@ function statify(obj) {
 })();
 
 
+// image load handler
+var img_load = (function () {
+    var r = {};
+    r.callbacks = [];
+
+    function fire() {
+        for (var a = 0; a < r.callbacks.length; a++)
+            r.callbacks[a]();
+    }
+
+    var timeout = null;
+    r.done = function () {
+        clearTimeout(timeout);
+        timeout = setTimeout(fire, 500);
+    };
+
+    return r;
+})();
+
+
 // faster than replacing the entire html (chrome 1.8x, firefox 1.6x)
 function copydom(src, dst, lv) {
     var sc = src.childNodes,
@@ -161,10 +164,7 @@ function copydom(src, dst, lv) {
 
 
 function md_plug_err(ex, js) {
-    var errbox = ebi('md_errbox');
-    if (errbox)
-        errbox.parentNode.removeChild(errbox);
-
+    qsr('#md_errbox');
     if (!ex)
         return;
 
@@ -175,17 +175,17 @@ function md_plug_err(ex, js) {
         msg = "Line " + ln + ", " + msg;
         var lns = js.split('\n');
         if (ln < lns.length) {
-            o = document.createElement('span');
-            o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
+            o = mknod('span');
+            o.style.cssText = "color:#ac2;font-size:.9em;font-family:'scp',monospace,monospace;display:block";
             o.textContent = lns[ln - 1];
         }
     }
-    errbox = document.createElement('div');
+    var errbox = mknod('div');
     errbox.setAttribute('id', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;
     errbox.onclick = function () {
-        alert('' + ex.stack);
+        modal.alert('<pre>' + esc(ex.stack) + '</pre>');
     };
     if (o) {
         errbox.appendChild(o);
@@ -264,7 +264,14 @@ function convert_markdown(md_text, dest_dom) {
 
         throw ex;
     }
-    var md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
+    var md_dom = dest_dom;
+    try {
+        md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
+    }
+    catch (ex) {
+        md_dom.innerHTML = md_html;
+        window.copydom = noop;
+    }
 
     var nodes = md_dom.getElementsByTagName('a');
     for (var a = nodes.length - 1; a >= 0; a--) {
@@ -356,6 +363,10 @@ function convert_markdown(md_text, dest_dom) {
 
     copydom(md_dom, dest_dom, 0);
 
+    var imgs = dest_dom.getElementsByTagName('img');
+    for (var a = 0, aa = imgs.length; a < aa; a++)
+        imgs[a].onload = img_load.done;
+
     if (ext && ext[0].render2)
         try {
             ext[0].render2(dest_dom);
@@ -367,8 +378,7 @@ function convert_markdown(md_text, dest_dom) {
 
 
 function init_toc() {
-    var loader = ebi('ml');
-    loader.parentNode.removeChild(loader);
+    qsr('#ml');
 
     var anchors = [];  // list of toc entries, complex objects
     var anchor = null; // current toc node
@@ -490,13 +500,16 @@ function init_toc() {
 // "main" :p
 convert_markdown(dom_src.value, dom_pre);
 var toc = init_toc();
+img_load.callbacks = [toc.refresh];
 
 
 // scroll handler
 var redraw = (function () {
-    var sbs = false;
+    var sbs = true;
     function onresize() {
-        sbs = window.matchMedia('(min-width: 64em)').matches;
+        if (window.matchMedia)
+            sbs = window.matchMedia('(min-width: 64em)').matches;
+
         var y = (dom_hbar.offsetTop + dom_hbar.offsetHeight) + 'px';
         if (sbs) {
             dom_toc.style.top = y;
@@ -530,3 +543,6 @@ dom_navtgl.onclick = function () {
 
 if (sread('hidenav') == 1)
     dom_navtgl.onclick();
+
+if (window['tt'])
+    tt.init();

copyparty/web/md2.css

@@ -1,126 +1,110 @@
 #toc {
-    display: none;
+	display: none;
 }
 #mtw {
-    display: block;
-    position: fixed;
-    left: .5em;
-    bottom: 0;
-    width: calc(100% - 56em);
+	display: block;
+	position: fixed;
+	left: .5em;
+	bottom: 0;
+	width: calc(100% - 56em);
 }
 #mw {
-    left: calc(100% - 55em);
-    overflow-y: auto;
-    position: fixed;
-    bottom: 0;
+	left: calc(100% - 55em);
+	overflow-y: auto;
+	position: fixed;
+	bottom: 0;
 }
 
 
 /* single-screen */
 #mtw.preview,
 #mw.editor {
-    opacity: 0;
-    z-index: 1;
+	opacity: 0;
+	z-index: 1;
 }
 #mw.preview,
 #mtw.editor {
-    z-index: 5;
+	z-index: 5;
 }
 #mtw.single,
 #mw.single {
-    margin: 0;
-    left: 1em;
-    left: max(1em, calc((100% - 56em) / 2));
+	margin: 0;
+	left: 1em;
+	left: max(1em, calc((100% - 56em) / 2));
 }
 #mtw.single {
-    width: 55em;
-    width: min(55em, calc(100% - 2em));
+	width: 55em;
+	width: min(55em, calc(100% - 2em));
 }
 
 
 #mp {
-    position: relative;
+	position: relative;
 }
 #mt, #mtr {
-    width: 100%;
-    height: calc(100% - 1px);
-    color: #444;
-    background: #f7f7f7;
-    border: 1px solid #999;
-    outline: none;
-    padding: 0;
-    margin: 0;
-    font-family: 'consolas', monospace, monospace;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    word-wrap: break-word; /*ie*/
-    overflow-y: scroll;
-    line-height: 1.3em;
-    font-size: .9em;
-    position: relative;
-    scrollbar-color: #eb0 #f7f7f7;
+	width: 100%;
+	height: calc(100% - 1px);
+	color: #444;
+	background: #f7f7f7;
+	border: 1px solid #999;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	font-family: 'scp', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-word;
+	overflow-wrap: break-word;
+	word-wrap: break-word; /*ie*/
+	overflow-y: scroll;
+	line-height: 1.3em;
+	font-size: .9em;
+	position: relative;
+	scrollbar-color: #eb0 #f7f7f7;
 }
 html.dark #mt {
-    color: #eee;
-    background: #222;
-    border: 1px solid #777;
-    scrollbar-color: #b80 #282828;
+	color: #eee;
+	background: #222;
+	border: 1px solid #777;
+	scrollbar-color: #b80 #282828;
 }
 #mtr {
-    position: absolute;
-    top: 0;
-    left: 0;
+	position: absolute;
+	top: 0;
+	left: 0;
 }
 #save.force-save {
-    color: #400;
-    background: #f97;
-    border-radius: .15em;
+	color: #400;
+	background: #f97;
+	border-radius: .15em;
 }
 html.dark #save.force-save {
-    color: #fca;
-    background: #720;
+	color: #fca;
+	background: #720;
 }
 #save.disabled {
-    opacity: .4;
-}
-#helpbox,
-#toast {
-    background: #f7f7f7;
-    border-radius: .4em;
-    z-index: 9001;
+	opacity: .4;
 }
 #helpbox {
-    display: none;
-    position: fixed;
-    padding: 2em;
-    top: 4em;
-    overflow-y: auto;
-    box-shadow: 0 .5em 2em #777;
-    height: calc(100% - 12em);
-    left: calc(50% - 15em);
-    right: 0;
-    width: 30em;
+	background: #f7f7f7;
+	border-radius: .4em;
+	z-index: 9001;
+	display: none;
+	position: fixed;
+	padding: 2em;
+	top: 4em;
+	overflow-y: auto;
+	box-shadow: 0 .5em 2em #777;
+	height: calc(100% - 12em);
+	left: calc(50% - 15em);
+	right: 0;
+	width: 30em;
 }
 #helpclose {
-    display: block;
+	display: block;
 }
 html.dark #helpbox {
-    box-shadow: 0 .5em 2em #444;
-}
-html.dark #helpbox,
-html.dark #toast {
-    background: #222;
-    border: 1px solid #079;
-    border-width: 1px 0;
-}
-#toast {
-    font-weight: bold;
-    text-align: center;
-    padding: .6em 0;
-    position: fixed;
-    z-index: 9001;
-    top: 30%;
-    transition: opacity 0.2s ease-in-out;
-    opacity: 1;
+	box-shadow: 0 .5em 2em #444;
+	background: #222;
+	border: 1px solid #079;
+	border-width: 1px 0;
 }

copyparty/web/md2.js

@@ -16,7 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = document.createElement('div');
+    var d = mknod('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
@@ -71,7 +71,7 @@ var map_src = [];
 var map_pre = [];
 function genmap(dom, oldmap) {
     var find = nlines;
-    while (oldmap && find --> 0) {
+    while (oldmap && find-- > 0) {
         var tmap = genmapq(dom, '*[data-ln="' + find + '"]');
         if (!tmap || !tmap.length)
             continue;
@@ -94,11 +94,11 @@ var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
     function draw_md() {
-        var t0 = new Date().getTime();
+        var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
 
-        var lines = hesc(src).replace(/\r/g, "").split('\n');
+        var lines = esc(src).replace(/\r/g, "").split('\n');
         nlines = lines.length;
         var html = [];
         for (var a = 0; a < lines.length; a++)
@@ -108,9 +108,9 @@ var draw_md = (function () {
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
 
-        cls(ebi('save'), 'disabled', src == server_md);
+        clmod(ebi('save'), 'disabled', src == server_md);
 
-        var t1 = new Date().getTime();
+        var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
     }
 
@@ -127,6 +127,12 @@ var draw_md = (function () {
 })();
 
 
+// discard TOC callback, just regen editor scroll map
+img_load.callbacks = [function () {
+    map_pre = genmap(dom_pre, map_pre);
+}];
+
+
 // resize handler
 redraw = (function () {
     function onresize() {
@@ -136,7 +142,6 @@ redraw = (function () {
         dom_ref.style.width = getComputedStyle(dom_src).offsetWidth + 'px';
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
-        dbg(document.body.clientWidth + 'x' + document.body.clientHeight);
     }
     function setsbs() {
         dom_wrap.setAttribute('class', '');
@@ -225,44 +230,40 @@ redraw = (function () {
 
 // modification checker
 function Modpoll() {
-    this.skip_one = true;
-    this.disabled = false;
-
-    this.periodic = function () {
-        var that = this;
-        setTimeout(function () {
-            that.periodic();
-        }, 1000 * md_opt.modpoll_freq);
+    var r = {
+        skip_one: true,
+        disabled: false
+    };
 
+    r.periodic = function () {
         var skip = null;
 
-        if (ebi('toast'))
+        if (toast.visible)
             skip = 'toast';
 
-        else if (this.skip_one)
+        else if (r.skip_one)
             skip = 'saved';
 
-        else if (this.disabled)
+        else if (r.disabled)
             skip = 'disabled';
 
         if (skip) {
             console.log('modpoll skip, ' + skip);
-            this.skip_one = false;
+            r.skip_one = false;
             return;
         }
 
         console.log('modpoll...');
-        var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
         var xhr = new XMLHttpRequest();
-        xhr.modpoll = this;
         xhr.open('GET', url, true);
         xhr.responseType = 'text';
-        xhr.onreadystatechange = this.cb;
+        xhr.onreadystatechange = r.cb;
         xhr.send();
-    }
+    };
 
-    this.cb = function () {
-        if (this.modpoll.disabled || this.modpoll.skip_one) {
+    r.cb = function () {
+        if (r.disabled || r.skip_one) {
             console.log('modpoll abort');
             return;
         }
@@ -283,27 +284,26 @@ function Modpoll() {
 
         if (server_ref != server_now) {
             console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
-            this.modpoll.disabled = true;
+            r.disabled = true;
             var msg = [
-                "The document has changed on the server.<br />" +
+                "The document has changed on the server.",
                 "The changes will NOT be loaded into your editor automatically.",
-
-                "Press F5 or CTRL-R to refresh the page,<br />" +
+                "",
+                "Press F5 or CTRL-R to refresh the page,",
                 "replacing your document with the server copy.",
-
-                "You can click this message to ignore and contnue."
+                "",
+                "You can close this message to ignore and contnue."
             ];
-            return toast(false, "box-shadow:0 1em 2em rgba(64,64,64,0.8);font-weight:normal",
-                36, "<p>" + msg.join('</p>\n<p>') + '</p>');
+            return toast.warn(0, msg.join('\n'));
         }
 
         console.log('modpoll eq');
-    }
+    };
 
     if (md_opt.modpoll_freq > 0)
-        this.periodic();
+        setInterval(r.periodic, 1000 * md_opt.modpoll_freq);
 
-    return this;
+    return r;
 }
 var modpoll = new Modpoll();
 
@@ -323,57 +323,56 @@ function save(e) {
     var save_btn = ebi("save"),
         save_cls = save_btn.getAttribute('class') + '';
 
-    if (save_cls.indexOf('disabled') >= 0) {
-        toast(true, ";font-size:2em;color:#c90", 9, "no changes");
-        return;
-    }
+    if (save_cls.indexOf('disabled') >= 0)
+        return toast.inf(2, "no changes");
 
     var force = (save_cls.indexOf('force-save') >= 0);
-    if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
-        alert('ok, aborted');
-        return;
+    function save2() {
+        var txt = dom_src.value,
+            fd = new FormData();
+
+        fd.append("act", "tput");
+        fd.append("lastmod", (force ? -1 : last_modified));
+        fd.append("body", txt);
+
+        var url = (document.location + '').split('?')[0];
+        var xhr = new XMLHttpRequest();
+        xhr.open('POST', url, true);
+        xhr.responseType = 'text';
+        xhr.onreadystatechange = save_cb;
+        xhr.btn = save_btn;
+        xhr.txt = txt;
+
+        modpoll.skip_one = true;  // skip one iteration while we save
+        xhr.send(fd);
     }
 
-    var txt = dom_src.value;
-
-    var fd = new FormData();
-    fd.append("act", "tput");
-    fd.append("lastmod", (force ? -1 : last_modified));
-    fd.append("body", txt);
-
-    var url = (document.location + '').split('?')[0];
-    var xhr = new XMLHttpRequest();
-    xhr.open('POST', url, true);
-    xhr.responseType = 'text';
-    xhr.onreadystatechange = save_cb;
-    xhr.btn = save_btn;
-    xhr.txt = txt;
-
-    modpoll.skip_one = true;  // skip one iteration while we save
-    xhr.send(fd);
+    if (!force)
+        save2();
+    else
+        modal.confirm('confirm that you wish to lose the changes made on the server since you opened this document', save2, function () {
+            toast.inf(3, 'aborted');
+        });
 }
 
 function save_cb() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var r;
     try {
         r = JSON.parse(this.responseText);
     }
     catch (ex) {
-        alert('Failed to parse reply from server:\n\n' + this.responseText);
-        return;
+        return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
     }
 
     if (!r.ok) {
-        if (!this.btn.classList.contains('force-save')) {
-            this.btn.classList.add('force-save');
+        if (!clgot(this.btn, 'force-save')) {
+            clmod(this.btn, 'force-save', 1);
             var msg = [
                 'This file has been modified since you started editing it!\n',
                 'if you really want to overwrite, press save again.\n',
@@ -383,15 +382,13 @@ function save_cb() {
                 r.lastmod + ' lastmod on the server now,',
                 r.now + ' server time now,\n',
             ];
-            alert(msg.join('\n'));
+            return toast.err(0, msg.join('\n'));
         }
-        else {
-            alert('Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
-        }
-        return;
+        else
+            return toast.err(0, 'Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
     }
 
-    this.btn.classList.remove('force-save');
+    clmod(this.btn, 'force-save');
     //alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
 
     run_savechk(r.lastmod, this.txt, this.btn, 0);
@@ -399,7 +396,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
@@ -415,10 +412,8 @@ function savechk_cb() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var doc1 = this.txt.replace(/\r\n/g, "\n");
     var doc2 = this.responseText.replace(/\r\n/g, "\n");
@@ -431,58 +426,22 @@ function savechk_cb() {
             }, 100);
             return;
         }
-        alert(
+        modal.alert(
             'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
             'Length: yours=' + doc1.length + ', server=' + doc2.length
         );
-        alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
-        alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
+        modal.alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
+        modal.alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
         return;
     }
 
     last_modified = this.lastmod;
     server_md = this.txt;
     draw_md();
-    toast(true, ";font-size:6em;font-family:serif;color:#9b4", 4,
-        'OK✔️<span style="font-size:.2em;color:#999;position:absolute">' + this.ntry + '</span>');
-
+    toast.ok(2, 'save OK' + (this.ntry ? '\nattempt ' + this.ntry : ''));
     modpoll.disabled = false;
 }
 
-function toast(autoclose, style, width, msg) {
-    var ok = ebi("toast");
-    if (ok)
-        ok.parentNode.removeChild(ok);
-
-    style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
-    ok = document.createElement('div');
-    ok.setAttribute('id', 'toast');
-    ok.setAttribute('style', style);
-    ok.innerHTML = msg;
-    var parent = ebi('m');
-    document.documentElement.appendChild(ok);
-
-    var hide = function (delay) {
-        delay = delay || 0;
-
-        setTimeout(function () {
-            ok.style.opacity = 0;
-        }, delay);
-
-        setTimeout(function () {
-            if (ok.parentNode)
-                ok.parentNode.removeChild(ok);
-        }, delay + 250);
-    }
-
-    ok.onclick = function () {
-        hide(0);
-    };
-
-    if (autoclose)
-        hide(500);
-}
-
 
 // firefox bug: initial selection offset isn't cleared properly through js
 var ff_clearsel = (function () {
@@ -761,7 +720,7 @@ function fmt_table(e) {
 
         var ind2 = tab[a].match(re_ind)[0];
         if (ind != ind2 && a != 1)  // the table can be a list entry or something, ignore [0]
-            return alert(err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
+            return toast.err(7, err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
 
         var t = tab[a].slice(ind.length);
         t = t.replace(re_lpipe, "");
@@ -771,7 +730,7 @@ function fmt_table(e) {
         if (a == 0)
             ncols = tab[a].length;
         else if (ncols < tab[a].length)
-            return alert(err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
+            return toast.err(7, err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
 
         // if row has less columns than row2, fill them in
         while (tab[a].length < ncols)
@@ -788,7 +747,7 @@ function fmt_table(e) {
     for (var col = 0; col < tab[1].length; col++) {
         var m = tab[1][col].match(re_align);
         if (!m)
-            return alert(err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
+            return toast.err(7, err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
 
         if (m[2]) {
             if (m[1])
@@ -876,10 +835,9 @@ function mark_uni(e) {
         ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g'),
         mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
 
-    if (txt == mod) {
-        alert('no results;  no modifications were made');
-        return;
-    }
+    if (txt == mod)
+        return toast.inf(5, 'no results;  no modifications were made');
+
     dom_src.value = mod;
 }
 
@@ -893,10 +851,9 @@ function iter_uni(e) {
         re = new RegExp('([^' + js_uni_whitelist + ']+)'),
         m = re.exec(txt.slice(ofs));
 
-    if (!m) {
-        alert('no more hits from cursor onwards');
-        return;
-    }
+    if (!m)
+        return toast.inf(5, 'no more hits from cursor onwards');
+
     ofs += m.index;
 
     dom_src.setSelectionRange(ofs, ofs + m[0].length, "forward");
@@ -911,23 +868,54 @@ function iter_uni(e) {
 function cfg_uni(e) {
     if (e) e.preventDefault();
 
-    var reply = prompt("unicode whitelist", esc_uni_whitelist);
-    if (reply === null)
-        return;
-
-    esc_uni_whitelist = reply;
-    js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+    modal.prompt("unicode whitelist", esc_uni_whitelist, function (reply) {
+        esc_uni_whitelist = reply;
+        js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+    }, null);
 }
 
 
+var set_lno = (function () {
+    var t = null,
+        pi = null,
+        pv = null,
+        lno = ebi('lno');
+
+    function poke() {
+        clearTimeout(t);
+        t = setTimeout(fire, 20);
+    }
+
+    function fire() {
+        try {
+            clearTimeout(t);
+
+            var i = dom_src.selectionStart;
+            if (i === pi)
+                return;
+
+            var v = 'L' + dom_src.value.slice(0, i).split('\n').length;
+            if (v != pv)
+                lno.innerHTML = v;
+
+            pi = i;
+            pv = v;
+        }
+        catch (e) { }
+    }
+
+    timer.add(fire);
+    return poke;
+})();
+
+
 // hotkeys / toolbar
 (function () {
     function keydown(ev) {
         ev = ev || window.event;
-        var kc = ev.keyCode || ev.which;
-        var ctrl = ev.ctrlKey || ev.metaKey;
-        //console.log(ev.code, kc);
-        if (ctrl && (ev.code == "KeyS" || kc == 83)) {
+        var kc = ev.code || ev.keyCode || ev.which;
+        //console.log(ev.key, ev.code, ev.keyCode, ev.which);
+        if (ctrl(ev) && (ev.code == "KeyS" || kc == 83)) {
             save();
             return false;
         }
@@ -936,23 +924,17 @@ function cfg_uni(e) {
             if (d)
                 d.click();
         }
-        if (document.activeElement == dom_src) {
-            if (ev.code == "Tab" || kc == 9) {
-                md_indent(ev.shiftKey);
-                return false;
-            }
-            if (ctrl && (ev.code == "KeyH" || kc == 72)) {
+        if (document.activeElement != dom_src)
+            return true;
+
+        set_lno();
+
+        if (ctrl(ev)) {
+            if (ev.code == "KeyH" || kc == 72) {
                 md_header(ev.shiftKey);
                 return false;
             }
-            if (!ctrl && (ev.code == "Home" || kc == 36)) {
-                md_home(ev.shiftKey);
-                return false;
-            }
-            if (!ctrl && !ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
-                return md_newline();
-            }
-            if (ctrl && (ev.code == "KeyZ" || kc == 90)) {
+            if (ev.code == "KeyZ" || kc == 90) {
                 if (ev.shiftKey)
                     action_stack.redo();
                 else
@@ -960,33 +942,45 @@ function cfg_uni(e) {
 
                 return false;
             }
-            if (ctrl && (ev.code == "KeyY" || kc == 89)) {
+            if (ev.code == "KeyY" || kc == 89) {
                 action_stack.redo();
                 return false;
             }
-            if (!ctrl && !ev.shiftKey && kc == 8) {
-                return md_backspace();
-            }
-            if (ctrl && (ev.code == "KeyK")) {
+            if (ev.code == "KeyK") {
                 fmt_table();
                 return false;
             }
-            if (ctrl && (ev.code == "KeyU")) {
+            if (ev.code == "KeyU") {
                 iter_uni();
                 return false;
             }
-            if (ctrl && (ev.code == "KeyE")) {
+            if (ev.code == "KeyE") {
                 dom_nsbs.click();
-                //fmt_table();
                 return false;
             }
             var up = ev.code == "ArrowUp" || kc == 38;
             var dn = ev.code == "ArrowDown" || kc == 40;
-            if (ctrl && (up || dn)) {
+            if (up || dn) {
                 md_p_jump(dn);
                 return false;
             }
         }
+        else {
+            if (ev.code == "Tab" || kc == 9) {
+                md_indent(ev.shiftKey);
+                return false;
+            }
+            if (ev.code == "Home" || kc == 36) {
+                md_home(ev.shiftKey);
+                return false;
+            }
+            if (!ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
+                return md_newline();
+            }
+            if (!ev.shiftKey && kc == 8) {
+                return md_backspace();
+            }
+        }
     }
     document.onkeydown = keydown;
     ebi('save').onclick = save;
@@ -1049,7 +1043,7 @@ action_stack = (function () {
         var p1 = from.length,
             p2 = to.length;
 
-        while (p1 --> 0 && p2 --> 0)
+        while (p1-- > 0 && p2-- > 0)
             if (from[p1] != to[p2])
                 break;
 
@@ -1129,9 +1123,9 @@ action_stack = (function () {
         ref = newtxt;
         dbg('undos(%d) redos(%d)', hist.un.length, hist.re.length);
         if (hist.un.length > 0)
-            dbg(statify(hist.un.slice(-1)[0]));
+            dbg(jcp(hist.un.slice(-1)[0]));
         if (hist.re.length > 0)
-            dbg(statify(hist.re.slice(-1)[0]));
+            dbg(jcp(hist.re.slice(-1)[0]));
     }
 
     return {
@@ -1142,14 +1136,3 @@ action_stack = (function () {
         _ref: ref
     }
 })();
-
-/*
-ebi('help').onclick = function () {
-    var c1 = getComputedStyle(dom_src).cssText.split(';');
-    var c2 = getComputedStyle(dom_ref).cssText.split(';');
-    var max = Math.min(c1.length, c2.length);
-    for (var a = 0; a < max; a++)
-        if (c1[a] !== c2[a])
-            console.log(c1[a] + '\n' + c2[a]);
-}
-*/

copyparty/web/mde.css

@@ -7,315 +7,149 @@ html .editor-toolbar>button.active { border-color: rgba(0,0,0,0.4); background:
 html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
 html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
 
+
+
 html {
-    line-height: 1.5em;
+	line-height: 1.5em;
 }
 html, body {
-    margin: 0;
-    padding: 0;
-    min-height: 100%;
-    font-family: sans-serif;
-    background: #f7f7f7;
-    color: #333;
+	margin: 0;
+	padding: 0;
+	min-height: 100%;
+	font-family: sans-serif;
+	background: #f7f7f7;
+	color: #333;
 }
+#toast {
+	bottom: auto;
+	top: 1.4em;
+}
+#repl {
+	position: absolute;
+	top: 0;
+	right: .5em;
+	border: none;
+	color: inherit;
+	background: none;
+	text-decoration: none;
+}
+
+
+
 #mn {
-    font-weight: normal;
-    margin: 1.3em 0 .7em 1em;
+	font-weight: normal;
+	margin: 1.3em 0 .7em 1em;
 }
 #mn a {
-    color: #444;
-    margin: 0 0 0 -.2em;
-    padding: 0 0 0 .4em;
-    text-decoration: none;
-    /* ie: */
-    border-bottom: .1em solid #777\9;
-    margin-right: 1em\9;
+	color: #444;
+	margin: 0 0 0 -.2em;
+	padding: 0 0 0 .4em;
+	text-decoration: none;
+	/* ie: */
+	border-bottom: .1em solid #777\9;
+	margin-right: 1em\9;
 }
 #mn a:first-child {
-    padding-left: .5em;
+	padding-left: .5em;
 }
 #mn a:last-child {
-    padding-right: .5em;
+	padding-right: .5em;
 }
 #mn a:not(:last-child):after {
-    content: '';
-    width: 1.05em;
-    height: 1.05em;
-    margin: -.2em .3em -.2em -.4em;
-    display: inline-block;
-    border: 1px solid rgba(0,0,0,0.2);
-    border-width: .2em .2em 0 0;
-    transform: rotate(45deg);
+	content: '';
+	width: 1.05em;
+	height: 1.05em;
+	margin: -.2em .3em -.2em -.4em;
+	display: inline-block;
+	border: 1px solid rgba(0,0,0,0.2);
+	border-width: .2em .2em 0 0;
+	transform: rotate(45deg);
 }
 #mn a:hover {
-    color: #000;
-    text-decoration: underline;
+	color: #000;
+	text-decoration: underline;
 }
 
 html .editor-toolbar>button.disabled {
-    opacity: .35;
-    pointer-events: none;
+	opacity: .35;
+	pointer-events: none;
 }
 html .editor-toolbar>button.save.force-save {
-    background: #f97;
-}
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-.cm-header { font-size: .4em !important }
-*/
-
-
-
-
-
-/* copied from md.css for now */
-.mdo pre,
-.mdo code,
-.mdo a {
-	color: #480;
-	background: #f7f7f7;
-	border: .07em solid #ddd;
-	border-radius: .2em;
-	padding: .1em .3em;
-	margin: 0 .1em;
-}
-.mdo code {
-	font-size: .96em;
-}
-.mdo pre,
-.mdo code {
-	font-family: monospace, monospace;
-	white-space: pre-wrap;
-	word-break: break-all;
-}
-.mdo pre code {
-	display: block;
-	margin: 0 -.3em;
-	padding: .4em .5em;
-	line-height: 1.1em;
-}
-.mdo a {
-    color: #fff;
-    background: #39b;
-    text-decoration: none;
-    padding: 0 .3em;
-    border: none;
-    border-bottom: .07em solid #079;
-}
-.mdo h2 {
-    color: #fff;
-    background: #555;
-    margin-top: 2em;
-    border-bottom: .22em solid #999;
-    border-top: none;
-}
-.mdo h1 {
-    color: #fff;
-    background: #444;
-    font-weight: normal;
-    border-top: .4em solid #fb0;
-    border-bottom: .4em solid #777;
-    border-radius: 0 1em 0 1em;
-    margin: 3em 0 1em 0;
-    padding: .5em 0;
-}
-h1, h2 {
-	line-height: 1.5em;
-}
-h1 {
-	font-size: 1.7em;
-	text-align: center;
-	border: 1em solid #777;
-	border-width: .05em 0;
-	margin: 3em 0;
-}
-h2 {
-	font-size: 1.5em;
-	font-weight: normal;
-	background: #f7f7f7;
-	border-top: .07em solid #fff;
-	border-bottom: .07em solid #bbb;
-	border-radius: .5em .5em 0 0;
-	padding-left: .4em;
-	margin-top: 3em;
-}
-.mdo ul,
-.mdo ol {
-	border-left: .3em solid #ddd;
-}
-.mdo>ul,
-.mdo>ol {
-	border-color: #bbb;
-}
-.mdo ul>li {
-	list-style-type: disc;
-}
-.mdo ul>li,
-.mdo ol>li {
-	margin: .7em 0;
-}
-strong {
-	color: #000;
-}
-p>em,
-li>em,
-td>em {
-	color: #c50;
-	padding: .1em;
-	border-bottom: .1em solid #bbb;
-}
-blockquote {
-	font-family: serif;
-	background: #f7f7f7;
-	border: .07em dashed #ccc;
-	padding: 0 2em;
-	margin: 1em 0;
-}
-small {
-	opacity: .8;
-}
-table {
-	border-collapse: collapse;
-}
-td {
-	padding: .2em .5em;
-	border: .12em solid #aaa;
-}
-th {
-	border: .12em solid #aaa;
-}
-
-
-
-
-
-/* mde support */
-.mdo {
-    padding: 1em;
-    background: #f7f7f7;
-}
-html.dark .mdo {
-    background: #1c1c1c;
+	background: #f97;
 }
 .CodeMirror {
-    background: #f7f7f7;
+	background: #f7f7f7;
 }
 
 
 
-
-
 /* darkmode */
 html.dark .mdo,
 html.dark .CodeMirror {
-    border-color: #222;
+	border-color: #222;
 }
 html.dark,
 html.dark body,
 html.dark .CodeMirror {
-    background: #222;
-    color: #ccc;
+	background: #222;
+	color: #ccc;
 }
 html.dark .CodeMirror-cursor {
-    border-color: #fff;
+	border-color: #fff;
 }
 html.dark .CodeMirror-selected {
-    box-shadow: 0 0 1px #0cf inset;
+	box-shadow: 0 0 1px #0cf inset;
 }
 html.dark .CodeMirror-selected,
 html.dark .CodeMirror-selectedtext {
-    border-radius: .1em;
-    background: #246;
-    color: #fff;
-}
-html.dark .mdo a {
-    background: #057;
-}
-html.dark .mdo h1 a, html.dark .mdo h4 a,
-html.dark .mdo h2 a, html.dark .mdo h5 a,
-html.dark .mdo h3 a, html.dark .mdo h6 a {
-    color: inherit;
-    background: none;
-}
-html.dark pre,
-html.dark code {
-    color: #8c0;
-    background: #1a1a1a;
-    border: .07em solid #333;
-}
-html.dark .mdo ul,
-html.dark .mdo ol {
-    border-color: #444;
-}
-html.dark .mdo>ul,
-html.dark .mdo>ol {
-    border-color: #555;
-}
-html.dark strong {
-    color: #fff;
-}
-html.dark p>em,
-html.dark li>em,
-html.dark td>em {
-    color: #f94;
-    border-color: #666;
-}
-html.dark h1 {
-    background: #383838;
-    border-top: .4em solid #b80;
-    border-bottom: .4em solid #4c4c4c;
-}
-html.dark h2 {
-    background: #444;
-    border-bottom: .22em solid #555;
-}
-html.dark td,
-html.dark th {
-    border-color: #444;
-}
-html.dark blockquote {
-    background: #282828;
-    border: .07em dashed #444;
+	border-radius: .1em;
+	background: #246;
+	color: #fff;
 }
 
 
 
 html.dark #mn a {
-    color: #ccc;
+	color: #ccc;
 }
 html.dark #mn a:not(:last-child):after {
-    border-color: rgba(255,255,255,0.3);
+	border-color: rgba(255,255,255,0.3);
 }
 html.dark .editor-toolbar {
-    border-color: #2c2c2c;
-    background: #1c1c1c;
+	border-color: #2c2c2c;
+	background: #1c1c1c;
 }
 html.dark .editor-toolbar>i.separator {
-    border-left: 1px solid #444;
-    border-right: 1px solid #111;
+	border-left: 1px solid #444;
+	border-right: 1px solid #111;
 }
 html.dark .editor-toolbar>button {
-    margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
-    color: #aaa;
+	margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
+	color: #aaa;
 }
 
 
 
 html.dark .editor-toolbar>button:hover {
-    color: #333;
+	color: #333;
 }
 html.dark .editor-toolbar>button.active {
-    color: #333;
-    border-color: #ec1;
-    background: #c90;
+	color: #333;
+	border-color: #ec1;
+	background: #c90;
 }
 html.dark .editor-toolbar::after,
 html.dark .editor-toolbar::before {
-    background: none;
-}
+	background: none;
+}
+
+
+
+/* ui.css overrides */
+.mdo {
+	padding: 1em;
+	background: #f7f7f7;
+}
+html.dark .mdo {
+	background: #1c1c1c;
+}

copyparty/web/mde.html

@@ -3,9 +3,10 @@
 	<title>📝🎉 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
-	<link href="/.cpr/mde.css" rel="stylesheet">
-	<link href="/.cpr/deps/mini-fa.css" rel="stylesheet">
-	<link href="/.cpr/deps/easymde.css" rel="stylesheet">
+	<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/mde.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/deps/mini-fa.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/deps/easymde.css?_={{ ts }}">
 </head>
 <body>
 	<div id="mw">
@@ -20,6 +21,7 @@
 			<textarea id="mt" style="display:none" autocomplete="off">{{ md }}</textarea>
 		</div>
 	</div>
+	<a href="#" id="repl">π</a>
 	<script>
 
 var last_modified = {{ lastmod }};
@@ -30,20 +32,20 @@ var md_opt = {
 };
 
 var lightswitch = (function () {
-	var fun = function () {
-		var dark = !document.documentElement.getAttribute("class");
-		document.documentElement.setAttribute("class", dark ? "dark" : "");
-		if (window.localStorage)
-			localStorage.setItem('lightmode', dark ? 0 : 1);
-	};
-	if (window.localStorage && localStorage.getItem('lightmode') != 1)
-		fun();
-	
-	return fun;
+	var l = localStorage,
+		drk = l.lightmode != 1,
+		f = function (e) {
+if (e) drk = !drk;
+document.documentElement.setAttribute("class", drk? "dark":"light");
+l.lightmode = drk? 0:1;
+		};
+	f();
+	return f;
 })();
 
 	</script>
-    <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/easymde.js"></script>
-	<script src="/.cpr/mde.js"></script>
+    <script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
+	<script src="/.cpr/deps/easymde.js?_={{ ts }}"></script>
+	<script src="/.cpr/mde.js?_={{ ts }}"></script>
 </body></html>

copyparty/web/mde.js

@@ -65,17 +65,16 @@ var mde = (function () {
     mde.codemirror.on("change", function () {
         md_changed(mde);
     });
-    var loader = ebi('ml');
-    loader.parentNode.removeChild(loader);
+    qsr('#ml');
     return mde;
 })();
 
 function set_jumpto() {
-    document.querySelector('.editor-preview-side').onclick = jumpto;
+    QS('.editor-preview-side').onclick = jumpto;
 }
 
 function jumpto(ev) {
-    var tgt = ev.target || ev.srcElement;
+    var tgt = ev.target;
     var ln = null;
     while (tgt && !ln) {
         ln = tgt.getAttribute('data-ln');
@@ -94,67 +93,63 @@ function md_changed(mde, on_srv) {
         window.md_saved = mde.value();
 
     var md_now = mde.value();
-    var save_btn = document.querySelector('.editor-toolbar button.save');
-
-    if (md_now == window.md_saved)
-        save_btn.classList.add('disabled');
-    else
-        save_btn.classList.remove('disabled');
+    var save_btn = QS('.editor-toolbar button.save');
 
+    clmod(save_btn, 'disabled', md_now == window.md_saved);
     set_jumpto();
 }
 
 function save(mde) {
-    var save_btn = document.querySelector('.editor-toolbar button.save');
-    if (save_btn.classList.contains('disabled')) {
-        alert('there is nothing to save');
-        return;
-    }
-    var force = save_btn.classList.contains('force-save');
-    if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
-        alert('ok, aborted');
-        return;
+    var save_btn = QS('.editor-toolbar button.save');
+    if (clgot(save_btn, 'disabled'))
+        return toast.inf(2, 'no changes');
+
+    var force = clgot(save_btn, 'force-save');
+    function save2() {
+        var txt = mde.value();
+
+        var fd = new FormData();
+        fd.append("act", "tput");
+        fd.append("lastmod", (force ? -1 : last_modified));
+        fd.append("body", txt);
+
+        var url = (document.location + '').split('?')[0];
+        var xhr = new XMLHttpRequest();
+        xhr.open('POST', url, true);
+        xhr.responseType = 'text';
+        xhr.onreadystatechange = save_cb;
+        xhr.btn = save_btn;
+        xhr.mde = mde;
+        xhr.txt = txt;
+        xhr.send(fd);
     }
 
-    var txt = mde.value();
-
-    var fd = new FormData();
-    fd.append("act", "tput");
-    fd.append("lastmod", (force ? -1 : last_modified));
-    fd.append("body", txt);
-
-    var url = (document.location + '').split('?')[0];
-    var xhr = new XMLHttpRequest();
-    xhr.open('POST', url, true);
-    xhr.responseType = 'text';
-    xhr.onreadystatechange = save_cb;
-    xhr.btn = save_btn;
-    xhr.mde = mde;
-    xhr.txt = txt;
-    xhr.send(fd);
+    if (!force)
+        save2();
+    else
+        modal.confirm('confirm that you wish to lose the changes made on the server since you opened this document', save2, function () {
+            toast.inf(3, 'aborted');
+        });
 }
 
 function save_cb() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var r;
     try {
         r = JSON.parse(this.responseText);
     }
     catch (ex) {
-        alert('Failed to parse reply from server:\n\n' + this.responseText);
-        return;
+        return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
     }
 
     if (!r.ok) {
-        if (!this.btn.classList.contains('force-save')) {
-            this.btn.classList.add('force-save');
+        if (!clgot(this.btn, 'force-save')) {
+            clmod(this.btn, 'force-save', 1);
             var msg = [
                 'This file has been modified since you started editing it!\n',
                 'if you really want to overwrite, press save again.\n',
@@ -164,15 +159,13 @@ function save_cb() {
                 r.lastmod + ' lastmod on the server now,',
                 r.now + ' server time now,\n',
             ];
-            alert(msg.join('\n'));
+            return toast.err(0, msg.join('\n'));
         }
-        else {
-            alert('Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
-        }
-        return;
+        else
+            return toast.err(0, 'Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
     }
 
-    this.btn.classList.remove('force-save');
+    clmod(this.btn, 'force-save');
     //alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
 
     // download the saved doc from the server and compare
@@ -192,35 +185,23 @@ function save_chk() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var doc1 = this.txt.replace(/\r\n/g, "\n");
     var doc2 = this.responseText.replace(/\r\n/g, "\n");
     if (doc1 != doc2) {
-        alert(
+        modal.alert(
             'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
             'Length: yours=' + doc1.length + ', server=' + doc2.length
         );
-        alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
-        alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
+        modal.alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
+        modal.alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
         return;
     }
 
     last_modified = this.lastmod;
     md_changed(this.mde, true);
 
-    var ok = document.createElement('div');
-    ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
-    ok.innerHTML = 'OK✔️';
-    var parent = ebi('m');
-    document.documentElement.appendChild(ok);
-    setTimeout(function () {
-        ok.style.opacity = 0;
-    }, 500);
-    setTimeout(function () {
-        ok.parentNode.removeChild(ok);
-    }, 750);
+    toast.ok(2, 'save OK' + (this.ntry ? '\nattempt ' + this.ntry : ''));
 }

copyparty/web/msg.css

@@ -3,7 +3,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -11,21 +11,19 @@ html {
 	background: #333;
 	font-family: sans-serif;
 	text-shadow: 1px 1px 0px #000;
+	touch-action: manipulation;
 }
 html, body {
 	margin: 0;
 	padding: 0;
 }
-body {
-	padding-bottom: 5em;
-}
 #box {
-    padding: .5em 1em;
-    background: #2c2c2c;
+	padding: .5em 1em;
+	background: #2c2c2c;
 }
 pre {
 	font-family: monospace, monospace;
 }
 a {
 	color: #fc5;
-}
+}

copyparty/web/msg.html

@@ -6,7 +6,7 @@
     <title>copyparty</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/msg.css">
+    <link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
 </head>
 
 <body>

copyparty/web/splash.css

@@ -3,6 +3,9 @@ html, body, #wrap {
 	background: #f7f7f7;
 	font-family: sans-serif;
 }
+html {
+	touch-action: manipulation;
+}
 #wrap {
 	max-width: 40em;
 	margin: 2em auto;
@@ -22,10 +25,59 @@ a {
 	color: #047;
 	background: #fff;
 	text-decoration: none;
-	border-bottom: 1px solid #aaa;
+	border-bottom: 1px solid #8ab;
 	border-radius: .2em;
 	padding: .2em .8em;
 }
+a+a {
+	margin-left: .5em;
+}
+.refresh,
+.logout {
+	float: right;
+	margin: -.2em 0 0 .5em;
+}
+.logout {
+	color: #c04;
+	border-color: #c7a;
+}
+#repl {
+	border: none;
+	background: none;
+	color: inherit;
+	padding: 0;
+}
+table {
+	border-collapse: collapse;
+}
+.vols td,
+.vols th {
+	padding: .3em .6em;
+	text-align: left;
+	white-space: nowrap;
+}
+.num {
+	border-right: 1px solid #bbb;
+}
+.num td {
+	padding: .1em .7em .1em 0;
+}
+.num td:first-child {
+	text-align: right;
+}
+.btns {
+	margin: 1em 0;
+}
+#msg {
+	margin: 3em 0;
+}
+#msg h1 {
+	margin-bottom: 0;
+}
+#msg h1 + p {
+	margin-top: .3em;
+	text-align: right;
+}
 
 
 html.dark,
@@ -42,12 +94,19 @@ html.dark a {
 	background: #057;
 	border-color: #37a;
 }
+html.dark .logout {
+	background: #804;
+	border-color: #c28;
+}
 html.dark input {
 	color: #fff;
-	background: #624;
-	border: 1px solid #c27;
+	background: #626;
+	border: 1px solid #c2c;
 	border-width: 1px 0 0 0;
 	border-radius: .5em;
 	padding: .5em .7em;
 	margin: 0 .5em 0 0;
-}
+}
+html.dark .num {
+	border-color: #777;
+}

copyparty/web/splash.html

@@ -6,41 +6,89 @@
     <title>copyparty</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/splash.css">
+    <link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
+    <link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
 </head>
 
 <body>
     <div id="wrap">
-        <p>hello {{ this.uname }}</p>
+        <a href="/?h" class="refresh">refresh</a>
 
+        {%- if this.uname == '*' %}
+            <p>howdy stranger &nbsp; <small>(you're not logged in)</small></p>
+        {%- else %}
+            <a href="/?pw=x" class="logout">logout</a>
+            <p>welcome back, <strong>{{ this.uname }}</strong></p>
+        {%- endif %}
+
+        {%- if msg %}
+        <div id="msg">
+            {{ msg }}
+        </div>
+        {%- endif %}
+
+        {%- if avol %}
+        <h1>admin panel:</h1>
+        <table><tr><td> <!-- hehehe -->
+            <table class="num">
+                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
+                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
+                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
+                <tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+            </table>
+        </td><td>
+            <table class="vols">
+                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+                <tbody>
+                    {% for mp in avol %}
+                    {%- if mp in vstate and vstate[mp] %}
+                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                    {%- endif %}
+                    {% endfor %}
+                </tbody>
+            </table>
+        </td></tr></table>
+        <div class="btns">
+            <a href="/?stack" tt="shows the state of all active threads">dump stack</a>
+            <a href="/?reload=cfg" tt="reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes">reload cfg</a>
+        </div>
+        {%- endif %}
+
+        {%- if rvol %}
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
+        {%- if wvol %}
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
+            <form method="post" enctype="multipart/form-data" action="/{{ qvpath }}">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />
             </form>
         </ul>
     </div>
+	<a href="#" id="repl">π</a>
     <script>
 
-if (window.localStorage && localStorage.getItem('lightmode') != 1)
+if (localStorage.lightmode != 1)
     document.documentElement.setAttribute("class", "dark");
 
 </script>
+<script src="/.cpr/util.js?_={{ ts }}"></script>
+<script>tt.init();</script>
 </body>
-</html>
+</html>

copyparty/web/ui.css

@@ -0,0 +1,465 @@
+@font-face {
+	font-family: 'scp';
+	font-display: swap;
+	src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(/.cpr/deps/scp.woff2) format('woff2');
+}
+html {
+	touch-action: manipulation;
+}
+#tt, #toast {
+	position: fixed;
+	max-width: 34em;
+	max-width: min(34em, 90%);
+	max-width: min(34em, calc(100% - 7em));
+	background: #333;
+	border: 0 solid #777;
+	box-shadow: 0 .2em .5em #111;
+	border-radius: .4em;
+	z-index: 9001;
+}
+#tt {
+	max-width: min(34em, calc(100% - 3.3em));
+	overflow: hidden;
+	margin: .7em 0;
+	padding: 0 1.3em;
+	height: 0;
+	opacity: .1;
+	transition: opacity 0.14s, height 0.14s, padding 0.14s;
+}
+#toast {
+	bottom: 5em;
+	right: -1em;
+	line-height: 1.5em;
+	padding: 1em 1.3em;
+	margin-left: 3em;
+	border-width: .4em 0;
+	overflow-wrap: break-word;
+	transform: translateX(100%);
+	transition:
+		transform .4s cubic-bezier(.2, 1.2, .5, 1),
+		right .4s cubic-bezier(.2, 1.2, .5, 1);
+	text-shadow: 1px 1px 0 #000;
+	color: #fff;
+}
+#toast a {
+	color: inherit;
+	text-shadow: inherit;
+	background: rgba(0, 0, 0, 0.4);
+	border-radius: .3em;
+	padding: .2em .3em;
+}
+#toast a#toastc {
+	display: inline-block;
+	position: absolute;
+	overflow: hidden;
+	left: 0;
+	width: 0;
+	opacity: 0;
+	padding: .3em 0;
+	margin: -.3em 0 0 0;
+	line-height: 1.3em;
+	color: #000;
+	border: none;
+	outline: none;
+	text-shadow: none;
+	border-radius: .5em 0 0 .5em;
+	transition: left .3s, width .3s, padding .3s, opacity .3s;
+}
+#toastb {
+	max-height: 70vh;
+	overflow-y: auto;
+}
+#toast.scroll #toastb {
+	overflow-y: scroll;
+	margin-right: -1.2em;
+	padding-right: .7em;
+}
+#toast pre {
+	margin: 0;
+}
+#toast.vis {
+	right: 1.3em;
+	transform: inherit;
+	transform: initial;
+}
+#toast.vis #toastc {
+	left: -2em;
+	width: .4em;
+	padding: .3em .8em;
+	opacity: 1;
+}
+#toast.inf {
+	background: #07a;
+	border-color: #0be;
+}
+#toast.inf #toastc {
+	background: #0be;
+}
+#toast.ok {
+	background: #380;
+	border-color: #8e4;
+}
+#toast.ok #toastc {
+	background: #8e4;
+}
+#toast.warn {
+	background: #960;
+	border-color: #fc0;
+}
+#toast.warn #toastc {
+	background: #fc0;
+}
+#toast.err {
+	background: #900;
+	border-color: #d06;
+}
+#toast.err #toastc {
+	background: #d06;
+}
+#tt.b {
+	padding: 0 2em;
+	border-radius: .5em;
+	box-shadow: 0 .2em 1em #000;
+}
+#tt.show {
+	padding: 1em 1.3em;
+	border-width: .4em 0;
+	height: auto;
+	opacity: 1;
+}
+#tt.show.b {
+	padding: 1.5em 2em;
+	border-width: .5em 0;
+}
+#modalc code,
+#tt code {
+	background: #3c3c3c;
+	padding: .1em .3em;
+	border-top: 1px solid #777;
+	border-radius: .3em;
+	line-height: 1.7em;
+}
+#tt em {
+	color: #f6a;
+}
+html.light #tt {
+	background: #fff;
+	border-color: #888 #000 #777 #000;
+}
+html.light #tt,
+html.light #toast {
+	box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
+}
+#modalc code,
+html.light #tt code {
+	background: #060;
+	color: #fff;
+}
+html.light #tt em {
+	color: #d38;
+}
+#modal {
+	position: fixed;
+    overflow: auto;
+	top: 0;
+	left: 0;
+	right: 0;
+	bottom: 0;
+	width: 100%;
+	height: 100%;
+	z-index: 9001;
+	background: rgba(64,64,64,0.6);
+}
+#modal>table {
+    width: 100%;
+    height: 100%;
+}
+#modal td {
+	text-align: center;
+}
+#modalc {
+	position: relative;
+	display: inline-block;
+	background: #f7f7f7;
+	color: #333;
+	text-shadow: none;
+	text-align: left;
+	margin: 3em;
+	padding: 1em 1.1em;
+	border-radius: .6em;
+    box-shadow: 0 .3em 3em rgba(0,0,0,0.5);
+	max-width: 50em;
+	max-height: 30em;
+	overflow: auto;
+}
+@media (min-width: 40em) {
+    #modalc {
+        min-width: 30em;
+    }
+}
+#modalc li {
+	margin: 1em 0;
+}
+#modalc h6 {
+	font-size: 1.3em;
+	border-bottom: 1px solid #999;
+	margin: 0;
+	padding: .3em;
+	text-align: center;
+}
+#modalb {
+	position: sticky;
+	text-align: right;
+	padding-top: 1em;
+	bottom: 0;
+	right: 0;
+}
+#modalb a {
+	color: #000;
+	background: #ccc;
+	display: inline-block;
+	border-radius: .3em;
+	padding: .5em 1em;
+	outline: none;
+	border: none;
+}
+#modalb a:focus,
+#modalb a:hover {
+	background: #06d;
+	color: #fff;
+}
+#modalb a+a {
+	margin-left: .5em;
+}
+#modali {
+	display: block;
+	background: #fff;
+	color: #000;
+	width: calc(100% - 1.25em);
+    margin: 1em -.1em 0 -.1em;
+	padding: .5em;
+	outline: none;
+	border: .25em solid #ccc;
+	border-radius: .4em;
+}
+#modali:focus {
+	border-color: #06d;
+}
+#repl_pre {
+	max-width: 24em;
+}
+*:focus,
+#pctl *:focus,
+.btn:focus {
+	box-shadow: 0 .1em .2em #fc0 inset;
+	border-radius: .2em;
+}
+html.light *:focus,
+html.light #pctl *:focus,
+html.light .btn:focus {
+	box-shadow: 0 .1em .2em #037 inset;
+}
+input[type="text"]:focus,
+input:not([type]):focus,
+textarea:focus {
+	box-shadow: 0 .1em .3em #fc0, 0 -.1em .3em #fc0;
+}
+html.light input[type="text"]:focus,
+html.light input:not([type]):focus,
+html.light textarea:focus {
+	box-shadow: 0 .1em .3em #037, 0 -.1em .3em #037;
+}
+
+
+
+
+
+
+
+
+
+
+
+.mdo pre,
+.mdo code,
+.mdo a {
+	color: #480;
+	background: #f7f7f7;
+	border: .07em solid #ddd;
+	border-radius: .2em;
+	padding: .1em .3em;
+	margin: 0 .1em;
+}
+.mdo pre,
+.mdo code,
+.mdo tt {
+	font-family: 'scp', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-all;
+}
+.mdo code {
+	font-size: .96em;
+}
+.mdo h1,
+.mdo h2 {
+	line-height: 1.5em;
+}
+.mdo h1 {
+	font-size: 1.7em;
+	text-align: center;
+	border: 1em solid #777;
+	border-width: .05em 0;
+	margin: 3em 0;
+}
+.mdo h2 {
+	font-size: 1.5em;
+	font-weight: normal;
+	background: #f7f7f7;
+	border-top: .07em solid #fff;
+	border-bottom: .07em solid #bbb;
+	border-radius: .5em .5em 0 0;
+	padding-left: .4em;
+	margin-top: 3em;
+}
+.mdo h3 {
+	border-bottom: .1em solid #999;
+}
+.mdo h1 a, .mdo h3 a, .mdo h5 a,
+.mdo h2 a, .mdo h4 a, .mdo h6 a {
+	color: inherit;
+	display: block;
+	background: none;
+	border: none;
+	padding: 0;
+	margin: 0;
+}
+.mdo ul,
+.mdo ol {
+	border-left: .3em solid #ddd;
+}
+.mdo ul>li,
+.mdo ol>li {
+	margin: .7em 0;
+	list-style-type: disc;
+}
+.mdo strong {
+	color: #000;
+}
+.mdo p>em,
+.mdo li>em,
+.mdo td>em {
+	color: #c50;
+	padding: .1em;
+	border-bottom: .1em solid #bbb;
+}
+.mdo blockquote {
+	font-family: serif;
+	background: #f7f7f7;
+	border: .07em dashed #ccc;
+	padding: 0 2em;
+	margin: 1em 0;
+}
+.mdo small {
+	opacity: .8;
+}
+.mdo pre code {
+	display: block;
+	margin: 0 -.3em;
+	padding: .4em .5em;
+	line-height: 1.1em;
+}
+.mdo pre code:hover {
+	background: #fec;
+	color: #360;
+}
+.mdo table {
+	border-collapse: collapse;
+	margin: 1em 0;
+}
+.mdo th,
+.mdo td {
+	padding: .2em .5em;
+	border: .12em solid #aaa;
+}
+
+@media screen {
+	.mdo {
+		word-break: break-word;
+		overflow-wrap: break-word;
+		word-wrap: break-word; /*ie*/
+	}
+	html.light .mdo a,
+	.mdo a {
+		color: #fff;
+		background: #39b;
+		text-decoration: none;
+		padding: 0 .3em;
+		border: none;
+		border-bottom: .07em solid #079;
+	}
+	.mdo h1 {
+		color: #fff;
+		background: #444;
+		font-weight: normal;
+		border-top: .4em solid #fb0;
+		border-bottom: .4em solid #777;
+		border-radius: 0 1em 0 1em;
+		margin: 3em 0 1em 0;
+		padding: .5em 0;
+	}
+	.mdo h2 {
+		color: #fff;
+		background: #555;
+		margin-top: 2em;
+		border-bottom: .22em solid #999;
+		border-top: none;
+	}
+
+
+
+	html.dark .mdo a {
+		background: #057;
+	}
+	html.dark .mdo h1 a, html.dark .mdo h4 a,
+	html.dark .mdo h2 a, html.dark .mdo h5 a,
+	html.dark .mdo h3 a, html.dark .mdo h6 a {
+		color: inherit;
+		background: none;
+	}
+	html.dark .mdo pre,
+	html.dark .mdo code {
+		color: #8c0;
+		background: #1a1a1a;
+		border: .07em solid #333;
+	}
+	html.dark .mdo ul,
+	html.dark .mdo ol {
+		border-color: #444;
+	}
+	html.dark .mdo strong {
+		color: #fff;
+	}
+	html.dark .mdo p>em,
+	html.dark .mdo li>em,
+	html.dark .mdo td>em {
+		color: #f94;
+		border-color: #666;
+	}
+	html.dark .mdo h1 {
+		background: #383838;
+		border-top: .4em solid #b80;
+		border-bottom: .4em solid #4c4c4c;
+	}
+	html.dark .mdo h2 {
+		background: #444;
+		border-bottom: .22em solid #555;
+	}
+	html.dark .mdo td,
+	html.dark .mdo th {
+		border-color: #444;
+	}
+	html.dark .mdo blockquote {
+		background: #282828;
+		border: .07em dashed #444;
+	}
+}

copyparty/web/upload.css

@@ -1,275 +0,0 @@
-
-#op_up2k {
-	padding: 0 1em 1em 1em;
-}
-#u2form {
-	position: absolute;
-	top: 0;
-	left: 0;
-	width: 2px;
-	height: 2px;
-	overflow: hidden;
-}
-#u2form input {
-	background: #444;
-	border: 0px solid #444;
-	outline: none;
-}
-#u2err.err {
-	color: #f87;
-	padding: .5em;
-}
-#u2btn {
-	color: #eee;
-	background: #555;
-	background: -moz-linear-gradient(top, #367 0%, #489 50%, #38788a 51%, #367 100%);
-	background: -webkit-linear-gradient(top, #367 0%, #489 50%, #38788a 51%, #367 100%);
-	background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%);
-	filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0);
-	text-decoration: none;
-	line-height: 1.3em;
-	border: 1px solid #222;
-	border-radius: .4em;
-	text-align: center;
-	font-size: 1.5em;
-	margin: .5em auto;
-	padding: .8em 0;
-	width: 16em;
-	cursor: pointer;
-	box-shadow: .4em .4em 0 #111;
-}
-#op_up2k.srch #u2btn {
-	background: linear-gradient(to bottom, #ca3 0%, #fd8 50%, #fc6 51%, #b92 100%);
-	text-shadow: 1px 1px 1px #fc6;
-	color: #333;
-}
-#u2conf #u2btn {
-	margin: -1.5em 0;
-	padding: .8em 0;
-	width: 100%;
-	max-width: 12em;
-	display: inline-block;
-}
-#u2conf #u2btn_cw {
-	text-align: right;
-}
-#u2notbtn {
-	display: none;
-	text-align: center;
-	background: #333;
-	padding-top: 1em;
-}
-#u2notbtn * {
-	line-height: 1.3em;
-}
-#u2tab {
-	margin: 3em auto;
-	width: calc(100% - 2em);
-	max-width: 100em;
-}
-#op_up2k.srch #u2tab {
-	max-width: none;
-}
-#u2tab td {
-	border: 1px solid #ccc;
-	border-width: 0 0px 1px 0;
-	padding: .1em .3em;
-}
-#u2tab td:nth-child(2) {
-	width: 5em;
-	white-space: nowrap;
-}
-#u2tab td:nth-child(3) {
-	width: 40%;
-}
-#op_up2k.srch #u2tab td:nth-child(3) {
-	font-family: sans-serif;
-	width: auto;
-}
-#u2tab tr+tr:hover td {
-	background: #222;
-}
-#u2cards {
-	margin: 2.5em auto -2.5em auto;
-	text-align: center;
-}
-#u2cards.w {
-	width: 45em;
-	text-align: left;
-}
-#u2cards a {
-	padding: .2em 1em;
-	border: 1px solid #777;
-	border-width: 0 0 1px 0;
-	background: linear-gradient(to bottom, #333, #222);
-}
-#u2cards a:first-child {
-	border-radius: .4em 0 0 0;
-}
-#u2cards a:last-child {
-	border-radius: 0 .4em 0 0;
-}
-#u2cards a.act {
-	border-width: 1px 1px 0 1px;
-	border-radius: .3em .3em 0 0;
-	margin-left: -1px;
-	background: transparent;
-}
-#u2cards span {
-	color: #fff;
-}
-#u2conf {
-	margin: 1em auto;
-	width: 30em;
-}
-#u2conf.has_btn {
-	width: 48em;
-}
-#u2conf * {
-	text-align: center;
-	line-height: 1em;
-	margin: 0;
-	padding: 0;
-	border: none;
-	outline: none;
-}
-#u2conf .txtbox {
-	width: 4em;
-	color: #fff;
-	background: #444;
-	border: 1px solid #777;
-	font-size: 1.2em;
-	padding: .15em 0;
-}
-#u2conf .txtbox.err {
-	background: #922;
-}
-#u2conf a {
-	color: #fff;
-	background: #c38;
-	text-decoration: none;
-	border-radius: .1em;
-	font-size: 1.5em;
-	padding: .1em 0;
-	margin: 0 -.25em;
-	width: 1.5em;
-	height: 1em;
-	display: inline-block;
-	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
-}
-#u2conf input+a {
-	background: #d80;
-}
-#u2conf label {
-	font-size: 1.6em;
-	width: 2em;
-	height: 1em;
-	padding: .4em 0;
-	display: block;
-	user-select: none;
-	border-radius: .25em;
-}
-#u2conf input[type="checkbox"] {
-	position: relative;
-	opacity: .02;
-	top: 2em;
-}
-#u2conf input[type="checkbox"]+label {
-	position: relative;
-	background: #603;
-	border-bottom: .2em solid #a16;
-	box-shadow: 0 .1em .3em #a00 inset;
-}
-#u2conf input[type="checkbox"]:checked+label {
-	background: #6a1;
-	border-bottom: .2em solid #efa;
-	box-shadow: 0 .1em .5em #0c0;
-}
-#u2conf input[type="checkbox"]+label:hover {
-	box-shadow: 0 .1em .3em #fb0;
-	border-color: #fb0;
-}
-#op_up2k.srch #u2conf td:nth-child(1)>*,
-#op_up2k.srch #u2conf td:nth-child(2)>*,
-#op_up2k.srch #u2conf td:nth-child(3)>* {
-	background: #777;
-	border-color: #ccc;
-	box-shadow: none;
-	opacity: .2;
-}
-#u2cdesc {
-	position: absolute;
-	width: 34em;
-	left: calc(50% - 15em);
-	background: #222;
-	border: 0 solid #555;
-	text-align: center;
-	overflow: hidden;
-	margin: 0 -2em;
-	height: 0;
-	padding: 0 1em;
-	opacity: .1;
-    transition: all 0.14s ease-in-out;
-	border-radius: .4em;
-	box-shadow: 0 .2em .5em #222;
-}
-#u2cdesc.show {
-	padding: 1em;
-	height: auto;
-	border-width: .2em 0;
-	opacity: 1;
-}
-#u2foot {
-	color: #fff;
-	font-style: italic;
-}
-#u2footfoot {
-	margin-bottom: -1em;
-}
-.prog {
-	font-family: monospace;
-}
-#u2tab a>span {
-	font-weight: bold;
-	font-style: italic;
-	color: #fff;
-	padding-left: .2em;
-}
-#u2cleanup {
-	float: right;
-	margin-bottom: -.3em;
-}
-
-
-
-
-
-html.light #u2btn {
-	box-shadow: .4em .4em 0 #ccc;
-}
-html.light #u2cards span {
-	color: #000;
-}
-html.light #u2cards a {
-	background: linear-gradient(to bottom, #eee, #fff);
-}
-html.light #u2cards a.act {
-	background: inherit;
-}
-html.light #u2conf .txtbox {
-	background: #fff;
-	color: #444;
-}
-html.light #u2conf .txtbox.err {
-	background: #f96;
-	color: #300;
-}
-html.light #u2cdesc {
-	background: #fff;
-	border: none;
-}
-html.light #op_up2k.srch #u2btn {
-	border-color: #a80;
-}

copyparty/web/upload.html

@@ -1,103 +0,0 @@
-
-    <div id="op_bup" class="opview opbox act">
-        <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
-            <input type="hidden" name="act" value="bput" />
-            <input type="file" name="f" multiple><br />
-            <input type="submit" value="start upload">
-        </form>
-    </div>
-
-    <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
-            <input type="hidden" name="act" value="mkdir" />
-            <input type="text" name="name" size="30">
-            <input type="submit" value="mkdir">
-        </form>
-    </div>
-
-    <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
-            <input type="hidden" name="act" value="new_md" />
-            <input type="text" name="name" size="30">
-            <input type="submit" value="create doc">
-        </form>
-    </div>
-
-    <div id="op_msg" class="opview opbox act">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
-            <input type="text" name="msg" size="30">
-            <input type="submit" value="send msg">
-        </form>
-    </div>
-
-    <div id="op_up2k" class="opview">
-        <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
-
-            <table id="u2conf">
-                <tr>
-                    <td>parallel uploads</td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="multitask" />
-                        <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="ask_up" />
-                        <label for="ask_up" alt="ask for confirmation befofre upload starts">💭</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="flag_en" />
-                        <label for="flag_en" alt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
-                    </td>
-                {%- if have_up2k_idx %}
-                    <td data-perm="read" rowspan="2">
-                        <input type="checkbox" id="fsearch" />
-                        <label for="fsearch" alt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
-                    </td>
-                {%- endif %}
-                    <td data-perm="read" rowspan="2" id="u2btn_cw"></td>
-                </tr>
-                <tr>
-                    <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
-                    </td>
-                </tr>
-            </table>
-
-            <div id="u2cdesc"></div>
-
-            <div id="u2notbtn"></div>
-
-            <div id="u2btn_ct">
-                <div id="u2btn">
-                    <span id="u2bm"></span><br />
-                    drag/drop files<br />
-                    and folders here<br />
-                    (or click me)
-                </div>
-            </div>
-
-            <div id="u2cards">
-                <a href="#" act="ok">ok <span>0</span></a><a
-                href="#" act="ng">ng <span>0</span></a><a
-                href="#" act="done">done <span>0</span></a><a
-                href="#" act="bz" class="act">busy <span>0</span></a><a
-                href="#" act="q">que <span>0</span></a>
-            </div>
-
-            <table id="u2tab">
-                <thead>
-                    <tr>
-                        <td>filename</td>
-                        <td>status</td>
-                        <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
-                    </tr>
-                </thead>
-                <tbody></tbody>
-            </table>
-
-            <p id="u2foot"></p>
-            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
-    </div>

docs/README.md

@@ -0,0 +1,51 @@
+**NOTE:** there's more stuff (sharex config, service scripts, nginx configs, ...) in [`/contrib/`](/contrib/)
+
+
+
+# example resource files
+
+can be provided to copyparty to tweak things
+
+
+
+## example `.epilogue.html`
+save one of these as `.epilogue.html` inside a folder to customize it:
+
+* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+
+
+## example browser-css
+point `--css-browser` to one of these by URL:
+
+* [`browser.css`](browser.css) changes the background
+* [`browser-icons.css`](browser-icons.css) adds filetype icons
+
+
+
+# other stuff
+
+## [`rclone.md`](rclone.md)
+* notes on using rclone as a fuse client/server
+
+## [`example.conf`](example.conf)
+* example config file for `-c`
+
+
+
+# junk
+
+alphabetical list of the remaining files
+
+| what | why |
+| -- | -- |
+| [biquad.html](biquad.html) | bruteforce calibrator for the audio equalizer since im not that good at maths |
+| [design.txt](design.txt) | initial brainstorming of the copyparty design, unmaintained, incorrect, sentimental value only |
+| [hls.html](hls.html) | experimenting with hls playback using `hls.js`, works p well, almost became a thing |
+| [music-analysis.sh](music-analysis.sh) | testing various bpm/key detection libraries before settling on the ones used in [`/bin/mtag/`](/bin/mtag/) |
+| [notes.sh](notes.sh) | notepad, just scraps really |
+| [nuitka.txt](nuitka.txt) | how to build a copyparty exe using nuitka (not maintained) |
+| [pretend-youre-qnap.patch](pretend-youre-qnap.patch) | simulate a NAS which keeps returning old cached data even though you just modified the file yourself |
+| [tcp-debug.sh](tcp-debug.sh) | looks like this was to debug stuck tcp connections? |
+| [unirange.py](unirange.py) | uhh |
+| [up2k.txt](up2k.txt) | initial ideas for how up2k should work, another unmaintained sentimental-value-only thing |

docs/biquad.html

@@ -0,0 +1,113 @@
+<!DOCTYPE html><html><head></head><body><script>
+
+setTimeout(location.reload.bind(location), 700);
+document.documentElement.scrollLeft = 0;
+
+var cali = (function() {
+    var ac = new AudioContext(),
+        fi = ac.createBiquadFilter(),
+        freqs = new Float32Array(1),
+        mag = new Float32Array(1),
+        phase = new Float32Array(1);
+
+    freqs[0] = 14000;
+    fi.type = 'peaking';
+    fi.frequency.value = 18000;
+    fi.Q.value = 0.8;
+    fi.gain.value = 1;
+    fi.getFrequencyResponse(freqs, mag, phase);
+
+    return mag[0];  // 1.0407 good, 1.0563 bad
+})(),
+    mp = cali < 1.05;
+
+var can = document.createElement('canvas'),
+    cc = can.getContext('2d'),
+    w = 2048,
+    h = 1024;
+
+w = 2048;
+
+can.width = w;
+can.height = h;
+document.body.appendChild(can);
+can.style.cssText = 'width:' + w + 'px;height:' + h + 'px';
+
+cc.fillStyle = '#000';
+cc.fillRect(0, 0, w, h);
+
+var cfg = [ // hz, q, g
+    [31.25 * 0.88, 0, 1.4],  // shelf
+    [31.25 * 1.04, 0.7, 0.96],  // peak
+    [62.5, 0.7, 1],
+    [125, 0.8, 1],
+    [250, 0.9, 1.03],
+    [500, 0.9, 1.1],
+    [1000, 0.9, 1.1],
+    [2000, 0.9, 1.105],
+    [4000, 0.88, 1.05],
+    [8000 * 1.006, 0.73, mp ? 1.24 : 1.2],
+    //[16000 * 1.00, 0.5, 1.75],  // peak.v1
+    //[16000 * 1.19, 0, 1.8]  // shelf.v1
+    [16000 * 0.89, 0.7, mp ? 1.26 : 1.2],  // peak
+    [16000 * 1.13, 0.82, mp ? 1.09 : 0.75],  // peak
+    [16000 * 1.205, 0, mp ? 1.9 : 1.85]  // shelf
+];
+
+var freqs = new Float32Array(22000),
+    sum = new Float32Array(freqs.length),
+    ac = new AudioContext(),
+    step = w / freqs.length,
+    colors = [
+        'rgba(255, 0, 0, 0.7)',
+        'rgba(0, 224, 0, 0.7)',
+        'rgba(0, 64, 255, 0.7)'
+    ];
+
+var order = [];
+
+for (var a = 0; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var a = 1; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var ia = 0; ia < order.length; ia++) {
+    var a = order[ia],
+        fi = ac.createBiquadFilter(),
+        mag = new Float32Array(freqs.length),
+        phase = new Float32Array(freqs.length);
+
+    for (var b = 0; b < freqs.length; b++)
+        freqs[b] = b;
+
+    fi.type = a == 0 ? 'lowshelf' : a == cfg.length - 1 ? 'highshelf' : 'peaking';
+    fi.frequency.value = cfg[a][0];
+    fi.Q.value = cfg[a][1];
+    fi.gain.value = 1;
+
+    fi.getFrequencyResponse(freqs, mag, phase);
+    cc.fillStyle = colors[a % colors.length];
+    for (var b = 0; b < sum.length; b++) {
+        mag[b] -= 1;
+        sum[b] += mag[b] * cfg[a][2];
+        var y = h - (mag[b] * h * 3);
+        cc.fillRect(b * step, y, step, h - y);
+        cc.fillRect(b * step - 1, y - 1, 3, 3);
+    }
+}
+
+var min = 999999, max = 0;
+for (var a = 0; a < sum.length; a++) {
+    min = Math.min(min, sum[a]);
+    max = Math.max(max, sum[a]);
+}
+cc.fillStyle = 'rgba(255,255,255,1)';
+for (var a = 0; a < sum.length; a++) {
+    var v = (sum[a] - min) / (max - min);
+    cc.fillRect(a * step, 0, step, v * h / 2);
+}
+
+cc.fillRect(0, 460, w, 1);
+
+</script></body></html>

docs/browser-icons.css

@@ -0,0 +1,71 @@
+/* video, alternative 1:
+   top-left icon, just like the other formats
+=======================================================================
+
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '📺';
+}
+*/
+
+
+
+/* video, alternative 2:
+   play-icon in the middle of the thumbnail
+=======================================================================
+*/
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+) {
+	position: relative;
+	overflow: hidden;
+}
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '▶';
+	opacity: .8;
+	margin: 0;
+	padding: 1em .5em 1em .7em;
+	border-radius: 9em;
+	line-height: 0;
+	color: #fff;
+	text-shadow: none;
+	background: rgba(0, 0, 0, 0.7);
+	left: calc(50% - 1em);
+	top: calc(50% - 1.4em);
+}
+
+
+
+/* audio */
+#ggrid>a:is(
+[href$=".mp3"i],
+[href$=".ogg"i],
+[href$=".opus"i],
+[href$=".flac"i],
+[href$=".m4a"i],
+[href$=".aac"i],
+):before {
+    content: '🎵';
+}
+
+
+
+/* image */
+#ggrid>a:is(
+[href$=".jpg"i],
+[href$=".jpeg"i],
+[href$=".png"i],
+[href$=".gif"i],
+[href$=".webp"i],
+):before {
+    content: '🎨';
+}

docs/browser.css

@@ -0,0 +1,30 @@
+html {
+    background: #222 url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
+}
+#files th {
+    background: rgba(32, 32, 32, 0.9) !important;
+}
+#ops,
+#tree,
+#files td {
+    background: rgba(32, 32, 32, 0.3) !important;
+}
+
+
+html.light {
+    background: #eee url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
+}
+html.light #files th {
+    background: rgba(255, 255, 255, 0.9) !important;
+}
+html.light .logue,
+html.light #ops,
+html.light #tree,
+html.light #files td {
+    background: rgba(248, 248, 248, 0.8) !important;
+}
+
+
+#files * {
+    background: transparent !important;
+}

docs/example.conf

@@ -1,3 +1,10 @@
+# append some arguments to the commandline;
+# the first space in a line counts as a separator,
+# any additional spaces are part of the value
+-e2dsa
+-e2ts
+-i 127.0.0.1
+
 # create users:
 # u username:password
 u ed:123
@@ -10,19 +17,26 @@ u k:k
 # share "." (the current directory)
 # as "/" (the webroot) for the following users:
 # "r" grants read-access for anyone
-# "a ed" grants read-write to ed
+# "rw ed" grants read-write to ed
 .
 /
 r
-a ed
+rw ed
 
 # custom permissions for the "priv" folder:
-# user "k" can see/read the contents
-# and "ed" gets read-write access
+# user "k" can only see/read the contents
+# user "ed" gets read-write access
 ./priv
 /priv
 r k
-a ed
+rw ed
+
+# this does the same thing,
+# and will cause an error on startup since /priv is already taken:
+./priv
+/priv
+r ed k
+w ed
 
 # share /home/ed/Music/ as /music and let anyone read it
 # (this will replace any folder called "music" in the webroot)
@@ -32,10 +46,14 @@ r
 
 # and a folder where anyone can upload
 # but nobody can see the contents
+# and set the e2d flag to enable the uploads database
+# and set the nodupe flag to reject duplicate uploads
 /home/ed/inc
 /dump
 w
+c e2d
+c nodupe
 
 # this entire config file can be replaced with these arguments:
-# -u ed:123 -u k:k -v .::r:aed -v priv:priv:rk:aed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w
+# -u ed:123 -u k:k -v .::r:a,ed -v priv:priv:r,k:rw,ed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w:c,e2d,nodupe
 # but note that the config file always wins in case of conflicts

docs/hls.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html><html lang="en"><head>
+	<meta charset="utf-8">
+	<title>hls-test</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+</head><body>
+
+<video id="vid" controls></video>
+<script src="hls.light.js"></script>
+<script>
+
+var video = document.getElementById('vid');
+var hls = new Hls({
+	debug: true,
+	autoStartLoad: false
+});
+hls.loadSource('live/v.m3u8');
+hls.attachMedia(video);
+hls.on(Hls.Events.MANIFEST_PARSED, function() {
+	hls.startLoad(0);
+});
+hls.on(Hls.Events.MEDIA_ATTACHED, function() {
+	video.muted = true;
+	video.play();
+});
+
+/*
+general good news:
+- doesn't need fixed-length segments; ok to let x264 pick optimal keyframes and slice on those
+- hls.js polls the m3u8 for new segments, scales the duration accordingly, seeking works great
+- the sfx will grow by 66 KiB since that's how small hls.js can get, wait thats not good
+
+# vod, creates m3u8 at the end, fixed keyframes, v bad
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -g 120 -keyint_min 120 -sc_threshold 0 -hls_time 4 -hls_playlist_type vod -hls_segment_filename v%05d.ts v.m3u8
+
+# live, updates m3u8 as it goes, dynamic keyframes, streamable with hls.js
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format mpegts -segment_list_flags live v%05d.ts
+
+# fmp4 (fragmented mp4), doesn't work with hls.js, gets duratoin 149:07:51 (536871s), probably the tkhd/mdhd 0xffffffff (timebase 8000? ok)
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format fmp4 -segment_list_flags live v%05d.mp4
+
+# try 2, works, uses tempfiles for m3u8 updates, good, 6% smaller
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f hls -hls_segment_type fmp4 -hls_list_size 0 -hls_segment_filename v%05d.mp4 v.m3u8
+
+more notes
+- adding -hls_flags single_file makes duration wack during playback (for both fmp4 and ts), ok once finalized and refreshed, gives no size reduction anyways
+- bebop op has good keyframe spacing for testing hls.js, in particular it hops one seg back and immediately resumes if it hits eof with the explicit hls.startLoad(0); otherwise it jumps into the middle of a seg and becomes art
+- can probably -c:v copy most of the time, is there a way to check for cgop? todo
+
+*/
+</script>
+</body></html>

docs/minimal-up2k.html

@@ -0,0 +1,37 @@
+<!--
+  save this as .epilogue.html inside a write-only folder to declutter the UI,  makes it look like
+  https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
+-->
+
+<style>
+
+    /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
+
+    #ops, #tree, #path, #wrap>h2:last-child,  /* main tabs and navigators (tree/breadcrumbs) */
+
+    #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
+
+    #srch_dz, #srch_zd,  /* the filesearch dropzone */
+
+    #u2cards, #u2etaw  /* and the upload progress tabs */
+
+    {display: none !important}  /* do it! */
+
+
+
+    /* add some margins because now it's weird */
+    .opview {margin-top: 2.5em}
+    #op_up2k {margin-top: 6em}
+
+    /* and embiggen the upload button */
+    #u2conf #u2btn, #u2btn {padding:1.5em 0}
+
+    /* adjust the button area a bit */
+    #u2conf.w, #u2conf.ww {width: 35em !important; margin: 5em auto}
+
+    /* a */
+    #op_up2k {min-height: 0}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

docs/mistakes.py

@@ -1,26 +0,0 @@
-
-                method = self.s.recv(4)
-                self.s.unrecv(method)
-                print("xxx unrecv'd [{}]".format(method))
-
-                # jython used to do this, they stopped since it's broken
-                # but reimplementing sendall is out of scope for now
-                if not getattr(self.s.s, "sendall", None):
-                    self.s.s.sendall = self.s.s.send
-
-                # TODO this is also pretty bad
-                have = dir(self.s)
-                for k in self.s.s.__dict__:
-                    if k not in have and not k.startswith("__"):
-                        if k == "recv":
-                            raise Exception("wait what")
-
-                        self.s.__dict__[k] = self.s.s.__dict__[k]
-
-                have = dir(self.s)
-                for k in dir(self.s.s):
-                    if k not in have and not k.startswith("__"):
-                        if k == "recv":
-                            raise Exception("wait what")
-
-                        setattr(self.s, k, getattr(self.s.s, k))

docs/notes.sh

@@ -38,13 +38,20 @@ para() { for s in 1 2 3 4 5 6 7 8 12 16 24 32 48 64; do echo $s; for r in {1..4}
 avg() { awk 'function pr(ncsz) {if (nsmp>0) {printf "%3s %s\n", csz, sum/nsmp} csz=$1;sum=0;nsmp=0} {sub(/\r$/,"")} /^[0-9]+$/ {pr($1);next} / MiB/ {sub(/ MiB.*/,"");sub(/.* /,"");sum+=$1;nsmp++} END {pr(0)}' "$1"; }
 
 
+##
+## time between first and last upload
+
+python3 -um copyparty -nw -v srv::rw -i 127.0.0.1 2>&1 | tee log 
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+
+
 ##
 ## bad filenames
 
-dirs=("$HOME/vfs/ほげ" "$HOME/vfs/ほげ/ぴよ" "$HOME/vfs/$(printf \\xed\\x91)" "$HOME/vfs/$(printf \\xed\\x91/\\xed\\x92)")
+dirs=("./ほげ" "./ほげ/ぴよ" "./$(printf \\xed\\x91)" "./$(printf \\xed\\x91/\\xed\\x92)" './qw,er;ty%20as df?gh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh')
 mkdir -p "${dirs[@]}"
-for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qwe,rty;asd fgh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh'; do echo "$dir" > "$dir/$fn.html"; done; done
-
+for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qw,er;ty%20as df?gh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh'; do echo "$dir" > "$dir/$fn.html"; done; done
+# qw er+ty%20ui%%20op<as>df&gh&amp;jk#zx'cv"bn`m=qw*er^ty?ui@op,as.df-gh_jk
 
 ##
 ## upload mojibake
@@ -79,6 +86,18 @@ command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (ti
 # get all up2k search result URLs
 var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
 
+# debug md-editor line tracking
+var s=mknod('style');s.innerHTML='*[data-ln]:before {content:attr(data-ln)!important;color:#f0c;background:#000;position:absolute;left:-1.5em;font-size:1rem}';document.head.appendChild(s);
+
+##
+## bash oneliners
+
+# get the size and video-id of all youtube vids in folder, assuming filename ends with -id.ext, and create a copyparty search query
+find -maxdepth 1 -printf '%s %p\n' | sort -n | awk '!/-([0-9a-zA-Z_-]{11})\.(mkv|mp4|webm)$/{next} {sub(/\.[^\.]+$/,"");n=length($0);v=substr($0,n-10);print $1, v}' | tee /dev/stderr | awk 'BEGIN {p="("} {printf("%s name like *-%s.* ",p,$2);p="or"} END {print ")\n"}' | cat >&2
+
+# unique stacks in a stackdump
+f=a; rm -rf stacks; mkdir stacks; grep -E '^#' $f | while IFS= read -r n; do awk -v n="$n" '!$0{o=0} o; $0==n{o=1}' <$f >stacks/f; h=$(sha1sum <stacks/f | cut -c-16); mv stacks/f stacks/$h-"$n"; done ; find stacks/ | sort | uniq -cw24
+
 
 ##
 ## sqlite3 stuff
@@ -93,6 +112,15 @@ cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '
 # dump all dbs
 find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
 
+# unschedule mtp scan for all files somewhere under "enc/"
+sqlite3 -readonly up2k.db 'select substr(up.w,1,16) from up inner join mt on mt.w = substr(up.w,1,16) where rd like "enc/%" and +mt.k = "t:mtp"' > keys; awk '{printf "delete from mt where w = \"%s\" and +k = \"t:mtp\";\n", $0}' <keys | tee /dev/stderr | sqlite3 up2k.db
+
+# compare metadata key "key" between two databases
+sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select w, v from mt where k = "key" order by w' > k2; ok=0; ng=0; while IFS='|' read w k2; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$(sqlite3 -readonly up2k.db.key-full "select * from up where substr(w,1,16) = '$w'" | sed -r 's/\|/ | /g')"; }; done < <(cat k2); echo "match $ok   diff $ng"
+
+# actually this is much better
+sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select mt.w, mt.v, up.rd, up.fn from mt inner join up on mt.w = substr(up.w,1,16) where mt.k = "key" order by up.rd, up.fn' > k2; ok=0; ng=0; while IFS='|' read w k2 path; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$path"; }; done < <(cat k2); echo "match $ok   diff $ng"
+
 
 ##
 ## media
@@ -103,6 +131,13 @@ e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d
 -v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
 sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
 
+# generate the sine meme
+for ((f=420;f<1200;f++)); do sz=$(ffmpeg -y -f lavfi -i sine=frequency=$f:duration=2 -vf volume=0.1 -ac 1 -ar 44100 -f s16le /dev/shm/a.wav 2>/dev/null; base64 -w0 </dev/shm/a.wav | gzip -c | wc -c); printf '%d %d\n' $f $sz; done | tee /dev/stderr | sort -nrk2,2
+ffmpeg -y -f lavfi -i sine=frequency=1050:duration=2 -vf volume=0.1 -ac 1 -ar 44100 /dev/shm/a.wav
+
+# play icon calibration pics
+for w in 150 170 190 210 230 250; do for h in 130 150 170 190 210; do /c/Program\ Files/ImageMagick-7.0.11-Q16-HDRI/magick.exe convert -size ${w}x${h} xc:brown -fill orange -draw "circle $((w/2)),$((h/2)) $((w/2)),$((h/3))" $w-$h.png; done; done
+
 
 ##
 ## vscode
@@ -134,7 +169,7 @@ brew install python@2
 pip install virtualenv
 
 # readme toc
-cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
+cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
 
 # fix firefox phantom breakpoints,
 # suggestions from bugtracker, doesnt work (debugger is not attachable)
@@ -146,6 +181,12 @@ dbg.asyncStore.pendingBreakpoints = {}
 # fix firefox phantom breakpoints
 about:config >> devtools.debugger.prefs-schema-version = -1
 
+# determine server version
+git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser,up2k}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser,up2k}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+
+# download all sfx versions
+curl https://api.github.com/repos/9001/copyparty/releases?per_page=100 | jq -r '.[] | .tag_name + " " + .name' | tr -d '\r' | while read v t; do fn="copyparty $v $t.py"; [ -e "$fn" ] || curl https://github.com/9001/copyparty/releases/download/$v/copyparty-sfx.py -Lo "$fn"; done
+
 
 ##
 ## http 206
@@ -171,7 +212,7 @@ Range: bytes=26-         Content-Range: bytes */26
 
 var tsh = [];
 function convert_markdown(md_text, dest_dom) {
-    tsh.push(new Date().getTime());
+    tsh.push(Date.now());
     while (tsh.length > 10)
         tsh.shift();
     if (tsh.length > 1) {
@@ -187,3 +228,4 @@ mk() { rm -rf /tmp/foo; sudo -u ed bash -c 'mkdir /tmp/foo; echo hi > /tmp/foo/b
 mk && t0="$(date)" && while true; do date -s "$(date '+ 1 hour')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
 mk && sudo -u ed flock /tmp/foo sleep 40 & sleep 1; ps aux | grep -E 'sleep 40$' && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
 mk && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; tar -cf/dev/null /tmp/foo; done; echo "$t0"
+