v1.3.12

.gitignore

@@ -5,13 +5,16 @@ __pycache__/
 MANIFEST.in
 MANIFEST
 copyparty.egg-info/
-buildenv/
-build/
-dist/
-sfx/
-py2/
 .venv/
 
+/buildenv/
+/build/
+/dist/
+/py2/
+/sfx/
+/unt/
+/log/
+
 # ide
 *.sublime-workspace
 

README.md

@@ -56,8 +56,11 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
     * [searching](#searching) - search by size, date, path/name, mp3-tags, ...
 * [server config](#server-config) - using arguments or config files, or a mix of both
     * [ftp-server](#ftp-server) - an FTP server can be started using `--ftp 3921`
-    * [file indexing](#file-indexing)
-    * [upload rules](#upload-rules) - set upload rules using volume flags
+    * [file indexing](#file-indexing) - enables dedup and music search ++
+        * [exclude-patterns](#exclude-patterns) - to save some time
+        * [filesystem guards](#filesystem-guards) - avoid traversing into other filesystems
+        * [periodic rescan](#periodic-rescan) - filesystem monitoring
+    * [upload rules](#upload-rules) - set upload rules using volflags
     * [compress uploads](#compress-uploads) - files can be autocompressed on upload
     * [database location](#database-location) - in-volume (`.hist/up2k.db`, default) or somewhere else
     * [metadata from audio files](#metadata-from-audio-files) - set `-e2t` to index tags on upload
@@ -246,12 +249,18 @@ some improvement ideas
 * Windows: if the `up2k.db` (filesystem index) is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit
   * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db on a local disk instead
 * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* [the database can get stuck](https://github.com/9001/copyparty/issues/10)
+  * has only happened once but that is once too many
+  * luckily not dangerous for file integrity and doesn't really stop uploads or anything like that
+  * but would really appreciate some logs if anyone ever runs into it again
 * probably more, pls let me know
 
 ## not my bugs
 
 * [Chrome issue 1317069](https://bugs.chromium.org/p/chromium/issues/detail?id=1317069) -- if you try to upload a folder which contains symlinks by dragging it into the browser, the symlinked files will not get uploaded
 
+* [Chrome issue 1352210](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210) -- plaintext http may be faster at filehashing than https (but also extremely CPU-intensive)
+
 * iPhones: the volume control doesn't work because [apple doesn't want it to](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11)
   * *future workaround:* enable the equalizer, make it all-zero, and set a negative boost to reduce the volume
     * "future" because `AudioContext` is broken in the current iOS version (15.1), maybe one day...
@@ -309,7 +318,7 @@ examples:
   * `u1` can open the `inc` folder, but cannot see the contents, only upload new files to it
   * `u2` can browse it and move files *from* `/inc` into any folder where `u2` has write-access
 * make folder `/mnt/ss` available at `/i`, read-write for u1, get-only for everyone else, and enable accesskeys: `-v /mnt/ss:i:rw,u1:g:c,fk=4`
-  * `c,fk=4` sets the `fk` volume-flag to 4, meaning each file gets a 4-character accesskey
+  * `c,fk=4` sets the `fk` volflag to 4, meaning each file gets a 4-character accesskey
   * `u1` can upload files, browse the folder, and see the generated accesskeys
   * other users cannot browse the folder, but can access the files if they have the full file URL with the accesskey
 
@@ -373,6 +382,7 @@ the browser has the following hotkeys  (always qwerty)
   * `Esc` close viewer
   * videos:
     * `U/O` skip 10sec back/forward
+    * `0..9` jump to 0%..90%
     * `P/K/Space` play/pause
     * `M` mute
     * `C` continue playing next video
@@ -655,7 +665,9 @@ an FTP server can be started using `--ftp 3921`,  and/or `--ftps` for explicit T
 
 ## file indexing
 
-file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volume flags, or a mix of both.
+enables dedup and music search ++
+
+file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volflags, or a mix of both.
 
 through arguments:
 * `-e2d` enables file indexing on upload
@@ -668,7 +680,7 @@ through arguments:
 * `-e2vu` patches the database with the new hashes from the filesystem
 * `-e2vp` panics and kills copyparty instead
 
-the same arguments can be set as volume flags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
+the same arguments can be set as volflags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
 * `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
 * `-v ~/music::r:c,d2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:c,d2t` disables all `-e2t*` (tags), does not affect `-e2d*`
@@ -680,7 +692,9 @@ note:
 * `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those, unless there is a new copyparty version with new parsers and the release note says otherwise
 * the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
 
-to save some time, you can provide a regex pattern for filepaths to only index by filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash \.iso$` or the volume-flag `:c,nohash=\.iso$`, this has the following consequences:
+### exclude-patterns
+
+to save some time,  you can provide a regex pattern for filepaths to only index by filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash \.iso$` or the volflag `:c,nohash=\.iso$`, this has the following consequences:
 * initial indexing is way faster, especially when the volume is on a network disk
 * makes it impossible to [file-search](#file-search)
 * if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
@@ -689,12 +703,29 @@ similarly, you can fully ignore files/folders using `--no-idx [...]` and `:c,noi
 
 if you set `--no-hash [...]` globally, you can enable hashing for specific volumes using flag `:c,nohash=`
 
+### filesystem guards
+
+avoid traversing into other filesystems  using `--xdev` / volflag `:c,xdev`, skipping any symlinks or bind-mounts to another HDD for example
+
+and/or you can `--xvol` / `:c,xvol` to ignore all symlinks leaving the volume's top directory, but still allow bind-mounts pointing elsewhere
+
+**NB: only affects the indexer** -- users can still access anything inside a volume, unless shadowed by another volume
+
+### periodic rescan
+
+filesystem monitoring;  if copyparty is not the only software doing stuff on your filesystem, you may want to enable periodic rescans to keep the index up to date
+
+argument `--re-maxage 60` will rescan all volumes every 60 sec, same as volflag `:c,scan=60` to specify it per-volume
+
+uploads are disabled while a rescan is happening, so rescans will be delayed by `--db-act` (default 10 sec) when there is write-activity going on (uploads, renames, ...)
+
 
 ## upload rules
 
-set upload rules using volume flags,  some examples:
+set upload rules using volflags,  some examples:
 
 * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
+* `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
 * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
 * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
 * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format
@@ -713,16 +744,16 @@ you can also set transaction limits which apply per-IP and per-volume, but these
 
 files can be autocompressed on upload,  either on user-request (if config allows) or forced by server-config
 
-* volume flag `gz` allows gz compression
-* volume flag `xz` allows lzma compression
-* volume flag `pk` **forces** compression on all files
+* volflag `gz` allows gz compression
+* volflag `xz` allows lzma compression
+* volflag `pk` **forces** compression on all files
 * url parameter `pk` requests compression with server-default algorithm
 * url parameter `gz` or `xz` requests compression with a specific algorithm
 * url parameter `xz` requests xz compression
 
 things to note,
 * the `gz` and `xz` arguments take a single optional argument, the compression level (range 0 to 9)
-* the `pk` volume flag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
+* the `pk` volflag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
 * default compression is gzip level 9
 * all upload methods except up2k are supported
 * the files will be indexed after compression, so dupe-detection and file-search will not work as expected
@@ -742,7 +773,7 @@ in-volume (`.hist/up2k.db`, default) or somewhere else
 
 copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
 
-this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volflag, or a mix of both:
 * `--hist ~/.cache/copyparty -v ~/music::r:c,hist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
 
 note:
@@ -780,7 +811,7 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
 
 provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
 
-copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volume flag), there is a default timeout of 30sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 30sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
 
 * `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
 * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
@@ -821,8 +852,8 @@ if this becomes popular maybe there should be a less janky way to do it actually
 tell search engines you dont wanna be indexed,  either using the good old [robots.txt](https://www.robotstxt.org/robotstxt.html) or through copyparty settings:
 
 * `--no-robots` adds HTTP (`X-Robots-Tag`) and HTML (`<meta>`) headers with `noindex, nofollow` globally
-* volume-flag `[...]:c,norobots` does the same thing for that single volume
-* volume-flag `[...]:c,robots` ALLOWS search-engine crawling for that volume, even if `--no-robots` is set globally
+* volflag `[...]:c,norobots` does the same thing for that single volume
+* volflag `[...]:c,robots` ALLOWS search-engine crawling for that volume, even if `--no-robots` is set globally
 
 also, `--force-js` disables the plain HTML folder listing, making things harder to parse for search engines
 
@@ -969,10 +1000,10 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
 up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
 
 regarding the frequent server log message during uploads;  
-`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198`
+`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`
 * this chunk was `6 MiB`, uploaded at `106 MiB/s`
 * on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
-* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` left
+* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
 
 
 ## why chunk-hashes
@@ -983,6 +1014,10 @@ this is due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues
 
 as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
 
+however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
+
+* both the [browser uploader](#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
+
 hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
 
 * blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
@@ -1016,6 +1051,7 @@ when uploading files,
 
 * if you're cpu-bottlenecked, or the browser is maxing a cpu core:
   * up to 30% faster uploads if you hide the upload status list by switching away from the `[🚀]` up2k ui-tab (or closing it)
+    * optionally you can switch to the lightweight potato ui by clicking the `[🥔]`
     * switching to another browser-tab also works, the favicon will update every 10 seconds in that case
   * unlikely to be a problem, but can happen when uploding many small files, or your internet is too fast, or PC too slow
 
@@ -1045,7 +1081,7 @@ some notes on hardening
 other misc notes:
 
 * you can disable directory listings by giving permission `g` instead of `r`, only accepting direct URLs to files
-  * combine this with volume-flag `c,fk` to generate per-file accesskeys; users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
+  * combine this with volflag `c,fk` to generate per-file accesskeys; users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
 
 
 ## gotchas

bin/mtag/README.md

@@ -42,7 +42,7 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
 * `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
 
 
-## usage with volume-flags
+## usage with volflags
 
 instead of affecting all volumes, you can set the options for just one volume like so:
 

bin/prisonparty.sh

@@ -11,13 +11,13 @@ sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
 help() { cat <<'EOF'
 
 usage:
-  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]"
+  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
 
 example:
-  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd"
+  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
 
 example for running straight from source (instead of using an sfx):
-  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd"
+  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
 
 note that if you have python modules installed as --user (such as bpm/key detectors),
   you should add /home/foo/.local as a VOLDIR

bin/up2k.py

@@ -3,7 +3,7 @@ from __future__ import print_function, unicode_literals
 
 """
 up2k.py: upload to copyparty
-2022-06-16, v0.15, ed <irc.rizon.net>, MIT-Licensed
+2022-08-13, v0.18, ed <irc.rizon.net>, MIT-Licensed
 https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
 
 - dependencies: requests
@@ -22,12 +22,29 @@ import atexit
 import signal
 import base64
 import hashlib
-import argparse
 import platform
 import threading
 import datetime
 
-import requests
+try:
+    import argparse
+except:
+    m = "\n  ERROR: need 'argparse'; download it here:\n   https://github.com/ThomasWaldmann/argparse/raw/master/argparse.py\n"
+    print(m)
+    raise
+
+try:
+    import requests
+except:
+    if sys.version_info > (2, 7):
+        m = "\n  ERROR: need 'requests'; run this:\n   python -m pip install --user requests\n"
+    else:
+        m = "requests/2.18.4 urllib3/1.23 chardet/3.0.4 certifi/2020.4.5.1 idna/2.7"
+        m = ["   https://pypi.org/project/" + x + "/#files" for x in m.split()]
+        m = "\n  ERROR: need these:\n" + "\n".join(m) + "\n"
+
+    print(m)
+    raise
 
 
 # from copyparty/__init__.py
@@ -126,6 +143,89 @@ class FileSlice(object):
         return ret
 
 
+class MTHash(object):
+    def __init__(self, cores):
+        self.f = None
+        self.sz = 0
+        self.csz = 0
+        self.omutex = threading.Lock()
+        self.imutex = threading.Lock()
+        self.work_q = Queue()
+        self.done_q = Queue()
+        self.thrs = []
+        for _ in range(cores):
+            t = threading.Thread(target=self.worker)
+            t.daemon = True
+            t.start()
+            self.thrs.append(t)
+
+    def hash(self, f, fsz, chunksz, pcb=None, pcb_opaque=None):
+        with self.omutex:
+            self.f = f
+            self.sz = fsz
+            self.csz = chunksz
+
+            chunks = {}
+            nchunks = int(math.ceil(fsz / chunksz))
+            for nch in range(nchunks):
+                self.work_q.put(nch)
+
+            ex = ""
+            for nch in range(nchunks):
+                qe = self.done_q.get()
+                try:
+                    nch, dig, ofs, csz = qe
+                    chunks[nch] = [dig, ofs, csz]
+                except:
+                    ex = ex or qe
+
+                if pcb:
+                    pcb(pcb_opaque, chunksz * nch)
+
+            if ex:
+                raise Exception(ex)
+
+            ret = []
+            for n in range(nchunks):
+                ret.append(chunks[n])
+
+            self.f = None
+            self.csz = 0
+            self.sz = 0
+            return ret
+
+    def worker(self):
+        while True:
+            ofs = self.work_q.get()
+            try:
+                v = self.hash_at(ofs)
+            except Exception as ex:
+                v = str(ex)
+
+            self.done_q.put(v)
+
+    def hash_at(self, nch):
+        f = self.f
+        ofs = ofs0 = nch * self.csz
+        hashobj = hashlib.sha512()
+        chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
+        while chunk_rem > 0:
+            with self.imutex:
+                f.seek(ofs)
+                buf = f.read(min(chunk_rem, 1024 * 1024 * 12))
+
+            if not buf:
+                raise Exception("EOF at " + str(ofs))
+
+            hashobj.update(buf)
+            chunk_rem -= len(buf)
+            ofs += len(buf)
+
+        digest = hashobj.digest()[:33]
+        digest = base64.urlsafe_b64encode(digest).decode("utf-8")
+        return nch, digest, ofs0, chunk_sz
+
+
 _print = print
 
 
@@ -230,8 +330,8 @@ def _scd(err, top):
             abspath = os.path.join(top, fh.name)
             try:
                 yield [abspath, fh.stat()]
-            except:
-                err.append(abspath)
+            except Exception as ex:
+                err.append((abspath, str(ex)))
 
 
 def _lsd(err, top):
@@ -240,8 +340,8 @@ def _lsd(err, top):
         abspath = os.path.join(top, name)
         try:
             yield [abspath, os.stat(abspath)]
-        except:
-            err.append(abspath)
+        except Exception as ex:
+            err.append((abspath, str(ex)))
 
 
 if hasattr(os, "scandir"):
@@ -250,15 +350,21 @@ else:
     statdir = _lsd
 
 
-def walkdir(err, top):
+def walkdir(err, top, seen):
     """recursive statdir"""
+    atop = os.path.abspath(os.path.realpath(top))
+    if atop in seen:
+        err.append((top, "recursive-symlink"))
+        return
+
+    seen = seen[:] + [atop]
     for ap, inf in sorted(statdir(err, top)):
         if stat.S_ISDIR(inf.st_mode):
             try:
-                for x in walkdir(err, ap):
+                for x in walkdir(err, ap, seen):
                     yield x
-            except:
-                err.append(ap)
+            except Exception as ex:
+                err.append((ap, str(ex)))
         else:
             yield ap, inf
 
@@ -273,7 +379,7 @@ def walkdirs(err, tops):
             stop = os.path.dirname(top)
 
         if os.path.isdir(top):
-            for ap, inf in walkdir(err, top):
+            for ap, inf in walkdir(err, top, []):
                 yield stop, ap[len(stop) :].lstrip(sep), inf
         else:
             d, n = top.rsplit(sep, 1)
@@ -322,8 +428,8 @@ def up2k_chunksize(filesize):
 
 
 # mostly from copyparty/up2k.py
-def get_hashlist(file, pcb):
-    # type: (File, any) -> None
+def get_hashlist(file, pcb, mth):
+    # type: (File, any, any) -> None
     """generates the up2k hashlist from file contents, inserts it into `file`"""
 
     chunk_sz = up2k_chunksize(file.size)
@@ -331,7 +437,12 @@ def get_hashlist(file, pcb):
     file_ofs = 0
     ret = []
     with open(file.abs, "rb", 512 * 1024) as f:
+        if mth and file.size >= 1024 * 512:
+            ret = mth.hash(f, file.size, chunk_sz, pcb, file)
+            file_rem = 0
+
         while file_rem > 0:
+            # same as `hash_at` except for `imutex` / bufsz
             hashobj = hashlib.sha512()
             chunk_sz = chunk_rem = min(chunk_sz, file_rem)
             while chunk_rem > 0:
@@ -388,8 +499,9 @@ def handshake(req_ses, url, file, pw, search):
         try:
             r = req_ses.post(url, headers=headers, json=req)
             break
-        except:
-            eprint("handshake failed, retrying: {0}\n".format(file.name))
+        except Exception as ex:
+            em = str(ex).split("SSLError(")[-1]
+            eprint("handshake failed, retrying: {0}\n  {1}\n\n".format(file.name, em))
             time.sleep(1)
 
     try:
@@ -398,7 +510,7 @@ def handshake(req_ses, url, file, pw, search):
         raise Exception(r.text)
 
     if search:
-        return r["hits"]
+        return r["hits"], False
 
     try:
         pre, url = url.split("://")
@@ -470,12 +582,19 @@ class Ctl(object):
 
         if err:
             eprint("\n# failed to access {0} paths:\n".format(len(err)))
-            for x in err:
-                eprint(x.decode("utf-8", "replace") + "\n")
+            for ap, msg in err:
+                if ar.v:
+                    eprint("{0}\n `-{1}\n\n".format(ap.decode("utf-8", "replace"), msg))
+                else:
+                    eprint(ap.decode("utf-8", "replace") + "\n")
 
             eprint("^ failed to access those {0} paths ^\n\n".format(len(err)))
+
+            if not ar.v:
+                eprint("hint: set -v for detailed error messages\n")
+
             if not ar.ok:
-                eprint("aborting because --ok is not set\n")
+                eprint("hint: aborting because --ok is not set\n")
                 return
 
         eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
@@ -516,6 +635,8 @@ class Ctl(object):
             self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
             self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
 
+            self.mth = MTHash(ar.J) if ar.J > 1 else None
+
             self._fancy()
 
     def _safe(self):
@@ -526,7 +647,7 @@ class Ctl(object):
             upath = file.abs.decode("utf-8", "replace")
 
             print("{0} {1}\n  hash...".format(self.nfiles - nf, upath))
-            get_hashlist(file, None)
+            get_hashlist(file, None, None)
 
             burl = self.ar.url[:12] + self.ar.url[8:].split("/")[0] + "/"
             while True:
@@ -679,7 +800,7 @@ class Ctl(object):
 
                 time.sleep(0.05)
 
-            get_hashlist(file, self.cb_hasher)
+            get_hashlist(file, self.cb_hasher, self.mth)
             with self.mutex:
                 self.hash_f += 1
                 self.hash_c += len(file.cids)
@@ -808,6 +929,9 @@ def main():
     if not VT100:
         os.system("rem")  # enables colors
 
+    cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+    hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
+
     # fmt: off
     ap = app = argparse.ArgumentParser(formatter_class=APF, epilog="""
 NOTE:
@@ -818,11 +942,13 @@ source file/folder selection uses rsync syntax, meaning that:
 
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
     ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
+    ap.add_argument("-v", action="store_true", help="verbose")
     ap.add_argument("-a", metavar="PASSWORD", help="password")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
     ap = app.add_argument_group("performance tweaks")
     ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
+    ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")

contrib/plugins/up2k-hook-ytid.js

@@ -2,32 +2,154 @@
 // assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
 // locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
 //
+// also tries to find the youtube-id in the embedded metadata
+//
 // assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
 
 function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
-    var filenames = [],
-        file_lists = [good_files, nil_files, bad_files];
+    var passthru = up2k.uc.fsearch;
+    if (passthru)
+        return hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
 
-    for (var lst of file_lists)
-        for (var ent of lst)
-            filenames.push(ent[1]);
+    a_up2k_namefilter(good_files, nil_files, bad_files, hooks).then(() => { });
+}
 
-    var yt_ids = new Set();
-    for (var lst of file_lists)
-        for (var ent of lst) {
-            var m, name = ent[1];
-            while (true) {
-                // some ytdl fork did %(title)-%(id).%(ext) ...
-                m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(name);
-                if (!m)
-                    break;
+function bstrpos(buf, ptn) {
+    var ofs = 0,
+        ch0 = ptn[0],
+        sz = buf.byteLength;
 
-                yt_ids.add(m[1]);
-                name = name.replace(m[1], '');
-            }
+    while (true) {
+        ofs = buf.indexOf(ch0, ofs);
+        if (ofs < 0 || ofs >= sz)
+            return -1;
+
+        for (var a = 1; a < ptn.length; a++)
+            if (buf[ofs + a] !== ptn[a])
+                break;
+
+        if (a === ptn.length)
+            return ofs;
+
+        ++ofs;
+    }
+}
+
+async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    var t0 = Date.now(),
+        yt_ids = new Set(),
+        textdec = new TextDecoder('latin1'),
+        md_ptn = new TextEncoder().encode('youtube.com/watch?v='),
+        file_ids = [],  // all IDs found for each good_files
+        md_only = [],  // `${id} ${fn}` where ID was only found in metadata
+        mofs = 0,
+        mnchk = 0,
+        mfile = '';
+
+    for (var a = 0; a < good_files.length; a++) {
+        var [fobj, name] = good_files[a],
+            cname = name,  // will clobber
+            sz = fobj.size,
+            ids = [],
+            id_ok = false,
+            m;
+
+        // all IDs found in this file
+        file_ids.push(ids);
+
+        // look for ID in filename; reduce the
+        // metadata-scan intensity if the id looks safe
+        m = /[\[(-]([\w-]{11})[\])]?\.(?:mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
+        id_ok = !!m;
+
+        while (true) {
+            // fuzzy catch-all;
+            // some ytdl fork did %(title)-%(id).%(ext) ...
+            m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(cname);
+            if (!m)
+                break;
+
+            cname = cname.replace(m[1], '');
+            yt_ids.add(m[1]);
+            ids.push(m[1]);
         }
 
-    toast.inf(5, `running query for ${yt_ids.size} videos...`);
+        // look for IDs in video metadata,
+        if (/\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name)) {
+            toast.show('inf r', 0, `analyzing file ${a + 1} / ${good_files.length} :\n${name}\n\nhave analysed ${++mnchk} files in ${(Date.now() - t0) / 1000} seconds, ${humantime((good_files.length - (a + 1)) * (((Date.now() - t0) / 1000) / mnchk))} remaining,\n\nbiggest offset so far is ${mofs}, in this file:\n\n${mfile}`);
+
+            // check first and last 128 MiB;
+            // pWxOroN5WCo.mkv @  6edb98 (6.92M)
+            // Nf-nN1wF5Xo.mp4 @ 4a98034 (74.6M)
+            var chunksz = 1024 * 1024 * 2,  // byte
+                aspan = id_ok ? 128 : 512;  // MiB
+
+            aspan = parseInt(Math.min(sz / 2, aspan * 1024 * 1024) / chunksz) * chunksz;
+
+            for (var side = 0; side < 2; side++) {
+                var ofs = side ? Math.max(0, sz - aspan) : 0,
+                    nchunks = aspan / chunksz;
+
+                for (var chunk = 0; chunk < nchunks; chunk++) {
+                    var bchunk = await fobj.slice(ofs, ofs + chunksz + 16).arrayBuffer(),
+                        uchunk = new Uint8Array(bchunk, 0, bchunk.byteLength),
+                        bofs = bstrpos(uchunk, md_ptn),
+                        absofs = Math.min(ofs + bofs, (sz - ofs) + bofs),
+                        txt = bofs < 0 ? '' : textdec.decode(uchunk.subarray(bofs)),
+                        m;
+
+                    //console.log(`side ${ side }, chunk ${ chunk }, ofs ${ ofs }, bchunk ${ bchunk.byteLength }, txt ${ txt.length }`);
+                    while (true) {
+                        // mkv/webm have [a-z] immediately after url
+                        m = /(youtube\.com\/watch\?v=[\w-]{11})/.exec(txt);
+                        if (!m)
+                            break;
+
+                        txt = txt.replace(m[1], '');
+                        m = m[1].slice(-11);
+
+                        console.log(`found ${m} @${bofs}, ${name} `);
+                        yt_ids.add(m);
+                        if (!has(ids, m)) {
+                            ids.push(m);
+                            md_only.push(`${m} ${name}`);
+                        }
+
+                        // bail after next iteration
+                        chunk = nchunks - 1;
+                        side = 9;
+
+                        if (mofs < absofs) {
+                            mofs = absofs;
+                            mfile = name;
+                        }
+                    }
+                    ofs += chunksz;
+                    if (ofs >= sz)
+                        break;
+                }
+            }
+        }
+    }
+
+    if (md_only.length)
+        console.log('recovered the following youtube-IDs by inspecting metadata:\n\n' + md_only.join('\n'));
+    else if (yt_ids.size)
+        console.log('did not discover any additional youtube-IDs by inspecting metadata; all the IDs also existed in the filenames');
+    else
+        console.log('failed to find any youtube-IDs at all, sorry');
+
+    if (false) {
+        var msg = `finished analysing ${mnchk} files in ${(Date.now() - t0) / 1000} seconds,\n\nbiggest offset was ${mofs} in this file:\n\n${mfile}`,
+            mfun = function () { toast.ok(0, msg); };
+
+        mfun();
+        setTimeout(mfun, 200);
+
+        return hooks[0]([], [], [], hooks.slice(1));
+    }
+
+    toast.inf(5, `running query for ${yt_ids.size} youtube-IDs...`);
 
     var xhr = new XHR();
     xhr.open('POST', '/ytq', true);
@@ -36,34 +158,61 @@ function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
         if (this.status != 200)
             return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
 
-        var new_lists = [],
-            ptn = new RegExp(this.responseText.trim().split('\n').join('|') || '\n'),
-            nothing_to_do = true,
-            n_skip = 0;
-
-        for (var lst of file_lists) {
-            var keep = [];
-            new_lists.push(keep);
-
-            for (var ent of lst)
-                if (ptn.exec(ent[1]))
-                    keep.push(ent);
-                else
-                    n_skip++;
-
-            if (keep.length)
-                nothing_to_do = false;
-        }
-
-        if (nothing_to_do)
-            return modal.alert('Good news -- turns out we already have all those videos.\n\nBut thank you for checking in!');
-        else if (n_skip)
-            toast.inf(0, `skipped ${n_skip} files which already exist on the server`);
-
-        [good_files, nil_files, bad_files] = new_lists;
-        hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+        process_id_list(this.responseText);
     };
     xhr.send(Array.from(yt_ids).join('\n'));
+
+    function process_id_list(txt) {
+        var wanted_ids = new Set(txt.trim().split('\n')),
+            wanted_names = new Set(),  // basenames with a wanted ID
+            wanted_files = new Set();  // filedrops
+
+        for (var a = 0; a < good_files.length; a++) {
+            var name = good_files[a][1];
+            for (var b = 0; b < file_ids[a].length; b++)
+                if (wanted_ids.has(file_ids[a][b])) {
+                    wanted_files.add(good_files[a]);
+
+                    var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
+                    if (m)
+                        wanted_names.add(m[1]);
+
+                    break;
+                }
+        }
+
+        // add all files with the same basename as each explicitly wanted file
+        // (infojson/chatlog/etc when ID was discovered from metadata)
+        for (var a = 0; a < good_files.length; a++) {
+            var name = good_files[a][1];
+            for (var b = 0; b < 3; b++) {
+                name = name.replace(/\.[^\.]+$/, '');
+                if (wanted_names.has(name)) {
+                    wanted_files.add(good_files[a]);
+                    break;
+                }
+            }
+        }
+
+        function upload_filtered() {
+            if (!wanted_files.size)
+                return modal.alert('Good news -- turns out we already have all those.\n\nBut thank you for checking in!');
+
+            hooks[0](Array.from(wanted_files), nil_files, bad_files, hooks.slice(1));
+        }
+
+        function upload_all() {
+            hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+        }
+
+        var n_skip = good_files.length - wanted_files.size,
+            msg = `you added ${good_files.length} files; ${good_files.length == n_skip ? 'all' : n_skip} of them were skipped --\neither because we already have them,\nor because there is no youtube-ID in your filenames.\n\n<code>OK</code> / <code>Enter</code> = continue uploading just the ${wanted_files.size} files we definitely need\n\n<code>Cancel</code> / <code>ESC</code> = override the filter; upload ALL the files you added`;
+
+        if (!n_skip)
+            upload_filtered();
+        else
+            modal.confirm(msg, upload_filtered, upload_all);
+    };
 }
 
 up2k_hooks.push(function () {

copyparty/__main__.py

@@ -24,7 +24,18 @@ from .__init__ import ANYWIN, PY2, VT100, WINDOWS, E, unicode
 from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
 from .authsrv import re_vol
 from .svchub import SvcHub
-from .util import IMPLICATIONS, align_tab, ansi_re, min_ex, py_desc, termsize, wrap
+from .util import (
+    IMPLICATIONS,
+    JINJA_VER,
+    PYFTPD_VER,
+    SQLITE_VER,
+    align_tab,
+    ansi_re,
+    min_ex,
+    py_desc,
+    termsize,
+    wrap,
+)
 
 try:
     from types import FrameType
@@ -107,12 +118,13 @@ class BasicDodge11874(
 
 
 def lprint(*a: Any, **ka: Any) -> None:
-    txt: str = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    eol = ka.pop("end", "\n")
+    txt: str = " ".join(unicode(x) for x in a) + eol
     printed.append(txt)
     if not VT100:
         txt = ansi_re.sub("", txt)
 
-    print(txt, **ka)
+    print(txt, end="", **ka)
 
 
 def warn(msg: str) -> None:
@@ -127,7 +139,7 @@ def ensure_locale() -> None:
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            lprint("Locale:", x)
+            lprint("Locale: {}\n".format(x))
             break
         except:
             continue
@@ -324,6 +336,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
         fk_salt = "hunter2"
 
     cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+    hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
 
     sects = [
         [
@@ -382,6 +395,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36mmaxn=250,600\033[35m max 250 uploads over 15min
               \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
               \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
+              \033[36mdf=1g\033[35m ensure 1 GiB free disk space
 
             \033[0mupload rotation:
             (moves all uploads into the specified folder structure)
@@ -396,10 +410,12 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36md2t\033[35m disables metadata collection, overrides -e2t*
               \033[36md2v\033[35m disables file verification, overrides -e2v*
               \033[36md2d\033[35m disables all database stuff, overrides -e2*
-              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
-              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
               \033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location
               \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
+              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
+              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
+              \033[36mxdev\033[35m do not descend into other filesystems
+              \033[36mxvol\033[35m skip symlinks leaving the volume root
 
             \033[0mdatabase, audio tags:
             "mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...
@@ -482,6 +498,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
     ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
+    ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
     ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
@@ -535,9 +552,10 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything")
     ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity (0.0028=10sec, 0.1=6min, 24=day, 168=week, 720=month, 8760=year)")
 
-    ap2 = ap.add_argument_group('yolo options')
+    ap2 = ap.add_argument_group('shutdown options')
     ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
     ap2.add_argument("--ign-ebind-all", action="store_true", help="continue running even if it's impossible to receive connections at all")
+    ap2.add_argument("--exit", metavar="WHEN", type=u, default="", help="shutdown after WHEN has finished; for example 'idx' will do volume indexing + metadata analysis")
 
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")
@@ -593,7 +611,12 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
     ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
+    ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
+    ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)")
+    ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root")
+    ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
+    ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline -- terminate searches running for more than SEC seconds")
     ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
 
@@ -631,6 +654,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
     ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
     ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
+    ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; '.' (a single dot) = all files")
     # fmt: on
 
     ap2 = ap.add_argument_group("help sections")
@@ -656,10 +680,17 @@ def main(argv: Optional[list[str]] = None) -> None:
     if argv is None:
         argv = sys.argv
 
-    desc = py_desc().replace("[", "\033[1;30m[")
-
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0;36m\n   sqlite v{} | jinja2 v{} | pyftpd v{}\n\033[0m'
+    f = f.format(
+        S_VERSION,
+        CODENAME,
+        S_BUILD_DT,
+        py_desc().replace("[", "\033[1;30m["),
+        SQLITE_VER,
+        JINJA_VER,
+        PYFTPD_VER,
+    )
+    lprint(f)
 
     ensure_locale()
     if HAVE_SSL:

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 3, 7)
+VERSION = (1, 3, 12)
 CODENAME = "god dag"
-BUILD_DT = (2022, 7, 16)
+BUILD_DT = (2022, 8, 13)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -17,9 +17,12 @@ from .bos import bos
 from .util import (
     IMPLICATIONS,
     META_NOBOTS,
+    SQLITE_VER,
     Pebkac,
     absreal,
     fsenc,
+    get_df,
+    humansize,
     relchk,
     statdir,
     uncyg,
@@ -72,15 +75,23 @@ class AXS(object):
 
 
 class Lim(object):
-    def __init__(self) -> None:
+    def __init__(self, log_func: Optional["RootLogger"]) -> None:
+        self.log_func = log_func
+
+        self.reg: Optional[dict[str, dict[str, Any]]] = None  # up2k registry
+
         self.nups: dict[str, list[float]] = {}  # num tracker
         self.bups: dict[str, list[tuple[float, int]]] = {}  # byte tracker list
         self.bupc: dict[str, int] = {}  # byte tracker cache
 
         self.nosub = False  # disallow subdirectories
 
-        self.smin = -1  # filesize min
-        self.smax = -1  # filesize max
+        self.dfl = 0  # free disk space limit
+        self.dft = 0  # last-measured time
+        self.dfv = 0  # currently free
+
+        self.smin = 0  # filesize min
+        self.smax = 0  # filesize max
 
         self.bwin = 0  # bytes window
         self.bmax = 0  # bytes max
@@ -92,18 +103,34 @@ class Lim(object):
         self.rotf = ""  # rot datefmt
         self.rot_re = re.compile("")  # rotf check
 
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        if self.log_func:
+            self.log_func("up-lim", msg, c)
+
     def set_rotf(self, fmt: str) -> None:
         self.rotf = fmt
         r = re.escape(fmt).replace("%Y", "[0-9]{4}").replace("%j", "[0-9]{3}")
         r = re.sub("%[mdHMSWU]", "[0-9]{2}", r)
         self.rot_re = re.compile("(^|/)" + r + "$")
 
-    def all(self, ip: str, rem: str, sz: float, abspath: str) -> tuple[str, str]:
+    def all(
+        self,
+        ip: str,
+        rem: str,
+        sz: int,
+        abspath: str,
+        reg: Optional[dict[str, dict[str, Any]]] = None,
+    ) -> tuple[str, str]:
+        if reg is not None and self.reg is None:
+            self.reg = reg
+            self.dft = 0
+
         self.chk_nup(ip)
         self.chk_bup(ip)
         self.chk_rem(rem)
         if sz != -1:
             self.chk_sz(sz)
+            self.chk_df(abspath, sz)  # side effects; keep last-ish
 
         ap2, vp2 = self.rot(abspath)
         if abspath == ap2:
@@ -111,13 +138,33 @@ class Lim(object):
 
         return ap2, ("{}/{}".format(rem, vp2) if rem else vp2)
 
-    def chk_sz(self, sz: float) -> None:
-        if self.smin != -1 and sz < self.smin:
+    def chk_sz(self, sz: int) -> None:
+        if sz < self.smin:
             raise Pebkac(400, "file too small")
 
-        if self.smax != -1 and sz > self.smax:
+        if self.smax and sz > self.smax:
             raise Pebkac(400, "file too big")
 
+    def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None:
+        if not self.dfl:
+            return
+
+        if self.dft < time.time():
+            self.dft = int(time.time()) + 300
+            self.dfv = get_df(abspath)[0] or 0
+            for j in list(self.reg.values()) if self.reg else []:
+                self.dfv -= int(j["size"] / len(j["hash"]) * len(j["need"]))
+
+            if already_written:
+                sz = 0
+
+        if self.dfv - sz < self.dfl:
+            self.dft = min(self.dft, int(time.time()) + 10)
+            t = "server HDD is full; {} free, need {}"
+            raise Pebkac(500, t.format(humansize(self.dfv - self.dfl), humansize(sz)))
+
+        self.dfv -= int(sz)
+
     def chk_rem(self, rem: str) -> None:
         if self.nosub and rem:
             raise Pebkac(500, "no subdirectories allowed")
@@ -226,7 +273,7 @@ class VFS(object):
 
     def __init__(
         self,
-        log: Optional[RootLogger],
+        log: Optional["RootLogger"],
         realpath: str,
         vpath: str,
         axs: AXS,
@@ -569,7 +616,7 @@ class AuthSrv(object):
     def __init__(
         self,
         args: argparse.Namespace,
-        log_func: Optional[RootLogger],
+        log_func: Optional["RootLogger"],
         warn_anonwrite: bool = True,
     ) -> None:
         self.args = args
@@ -661,7 +708,7 @@ class AuthSrv(object):
                     raise Exception('invalid mountpoint "{}"'.format(vol_dst))
 
                 # cfg files override arguments and previous files
-                vol_src = bos.path.abspath(vol_src)
+                vol_src = absreal(vol_src)
                 vol_dst = vol_dst.strip("/")
                 self._map_volume(vol_src, vol_dst, mount, daxs, mflags)
                 continue
@@ -682,12 +729,12 @@ class AuthSrv(object):
         self, lvl: str, uname: str, axs: AXS, flags: dict[str, Any]
     ) -> None:
         if lvl.strip("crwmdg"):
-            raise Exception("invalid volume flag: {},{}".format(lvl, uname))
+            raise Exception("invalid volflag: {},{}".format(lvl, uname))
 
         if lvl == "c":
             cval: Union[bool, str] = True
             try:
-                # volume flag with arguments, possibly with a preceding list of bools
+                # volflag with arguments, possibly with a preceding list of bools
                 uname, cval = uname.split("=", 1)
             except:
                 # just one or more bools
@@ -772,7 +819,7 @@ class AuthSrv(object):
                     src = uncyg(src)
 
                 # print("\n".join([src, dst, perms]))
-                src = bos.path.abspath(src)
+                src = absreal(src)
                 dst = dst.strip("/")
                 self._map_volume(src, dst, mount, daxs, mflags)
 
@@ -801,7 +848,7 @@ class AuthSrv(object):
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             axs = AXS(["*"], ["*"], None, None)
-            vfs = VFS(self.log_func, bos.path.abspath("."), "", axs, {})
+            vfs = VFS(self.log_func, absreal("."), "", axs, {})
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
             vfs = VFS(self.log_func, "", "", AXS(), {})
@@ -917,13 +964,20 @@ class AuthSrv(object):
         vfs.histtab = {zv.realpath: zv.histpath for zv in vfs.all_vols.values()}
 
         for vol in vfs.all_vols.values():
-            lim = Lim()
+            lim = Lim(self.log_func)
             use = False
 
             if vol.flags.get("nosub"):
                 use = True
                 lim.nosub = True
 
+            zs = vol.flags.get("df") or (
+                "{}g".format(self.args.df) if self.args.df else ""
+            )
+            if zs:
+                use = True
+                lim.dfl = unhumanize(zs)
+
             zs = vol.flags.get("sz")
             if zs:
                 use = True
@@ -976,10 +1030,15 @@ class AuthSrv(object):
                 vol.flags["dathumb"] = True
                 vol.flags["dithumb"] = True
 
+        have_fk = False
         for vol in vfs.all_vols.values():
             fk = vol.flags.get("fk")
             if fk:
                 vol.flags["fk"] = int(fk) if fk is not True else 8
+                have_fk = True
+
+        if have_fk and re.match(r"^[0-9\.]+$", self.args.fk_salt):
+            self.log("filekey salt: {}".format(self.args.fk_salt))
 
         for vol in vfs.all_vols.values():
             if "pk" in vol.flags and "gz" not in vol.flags and "xz" not in vol.flags:
@@ -1008,7 +1067,7 @@ class AuthSrv(object):
                 if ptn:
                     vol.flags[vf] = re.compile(ptn)
 
-            for k in ["e2t", "e2ts", "e2tsr", "e2v", "e2vu", "e2vp"]:
+            for k in ["e2t", "e2ts", "e2tsr", "e2v", "e2vu", "e2vp", "xdev", "xvol"]:
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
@@ -1026,7 +1085,7 @@ class AuthSrv(object):
             if "mth" not in vol.flags:
                 vol.flags["mth"] = self.args.mth
 
-            # append parsers from argv to volume-flags
+            # append parsers from argv to volflags
             self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
 
             # d2d drops all database features for a volume
@@ -1089,7 +1148,7 @@ class AuthSrv(object):
 
             for mtp in local_only_mtp:
                 if mtp not in local_mte:
-                    t = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)'
+                    t = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volflags)'
                     self.log(t.format(vol.vpath, mtp), 1)
                     errors = True
 
@@ -1098,7 +1157,7 @@ class AuthSrv(object):
         tags = [y for x in tags for y in x.split(",")]
         for mtp in tags:
             if mtp not in all_mte:
-                t = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volume-flag)'
+                t = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volflag)'
                 self.log(t.format(mtp), 1)
                 errors = True
 
@@ -1107,7 +1166,7 @@ class AuthSrv(object):
 
         vfs.bubble_flags()
 
-        e2vs = []
+        have_e2d = False
         t = "volumes and permissions:\n"
         for zv in vfs.all_vols.values():
             if not self.warn_anonwrite:
@@ -1126,24 +1185,27 @@ class AuthSrv(object):
                 u = u if u else "\033[36m--none--\033[0m"
                 t += "\n|  {}:  {}".format(txt, u)
 
-            if "e2v" in zv.flags and zv.axs.uwrite:
-                e2vs.append(zv.vpath or "/")
+            if "e2d" in zv.flags:
+                have_e2d = True
 
             t += "\n"
 
-        if e2vs:
-            t += "\n\033[33me2v enabled for the following volumes;\nuploads will be blocked until scan has finished:\n  \033[0m"
-            t += "  ".join(e2vs) + "\n"
+        if self.warn_anonwrite:
+            if not self.args.no_voldump:
+                self.log(t)
 
-        if self.warn_anonwrite and not self.args.no_voldump:
-            self.log(t)
+            if have_e2d:
+                t = self.chk_sqlite_threadsafe()
+                if t:
+                    self.log("\n\033[{}\033[0m\n".format(t))
 
         try:
             zv, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == zv.realpath:
-                self.warn_anonwrite = False
                 t = "anyone can write to the current directory: {}\n"
                 self.log(t.format(zv.realpath), c=1)
+
+            self.warn_anonwrite = False
         except Pebkac:
             self.warn_anonwrite = True
 
@@ -1157,6 +1219,23 @@ class AuthSrv(object):
             if pwds:
                 self.re_pwd = re.compile("=(" + "|".join(pwds) + ")([]&; ]|$)")
 
+    def chk_sqlite_threadsafe(self) -> str:
+        v = SQLITE_VER[-1:]
+
+        if v == "1":
+            # threadsafe (linux, windows)
+            return ""
+
+        if v == "2":
+            # module safe, connections unsafe (macos)
+            return "33m  your sqlite3 was compiled with reduced thread-safety;\n   database features (-e2d, -e2t) SHOULD be fine\n    but MAY cause database-corruption and crashes"
+
+        if v == "0":
+            # everything unsafe
+            return "31m  your sqlite3 was compiled WITHOUT thread-safety!\n   database features (-e2d, -e2t) will PROBABLY cause crashes!"
+
+        return "36m  cannot verify sqlite3 thread-safety; strange but probably fine"
+
     def dbg_ls(self) -> None:
         users = self.args.ls
         vol = "*"

copyparty/broker_util.py

@@ -42,7 +42,7 @@ class BrokerCli(object):
     """
 
     def __init__(self) -> None:
-        self.log: RootLogger = None
+        self.log: "RootLogger" = None
         self.args: argparse.Namespace = None
         self.asrv: AuthSrv = None
         self.httpsrv: "HttpSrv" = None

copyparty/fsutil.py

@@ -1,17 +1,17 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import ctypes
 import os
 import re
 import time
 
 from .__init__ import ANYWIN, MACOS
 from .authsrv import AXS, VFS
+from .bos import bos
 from .util import chkcmd, min_ex
 
 try:
-    from typing import Any, Optional, Union
+    from typing import Optional, Union
 
     from .util import RootLogger
 except:
@@ -19,7 +19,7 @@ except:
 
 
 class Fstab(object):
-    def __init__(self, log: RootLogger):
+    def __init__(self, log: "RootLogger"):
         self.log_func = log
 
         self.trusted = False
@@ -40,13 +40,9 @@ class Fstab(object):
         msg = "failed to determine filesystem at [{}]; assuming {}\n{}"
 
         if ANYWIN:
-            fs = "vfat"  # can smb do sparse files? gonna guess no
+            fs = "vfat"
             try:
-                # good enough
-                disk = path.split(":", 1)[0]
-                disk = "{}:\\".format(disk).lower()
-                assert len(disk) == 3
-                path = disk
+                path = self._winpath(path)
             except:
                 self.log(msg.format(path, fs, min_ex()), 3)
                 return fs
@@ -67,6 +63,19 @@ class Fstab(object):
         self.log("found {} at {}".format(fs, path))
         return fs
 
+    def _winpath(self, path: str) -> str:
+        # try to combine volume-label + st_dev (vsn)
+        path = path.replace("/", "\\")
+        vid = path.split(":", 1)[0].strip("\\").split("\\", 1)[0]
+        try:
+            return "{}*{}".format(vid, bos.stat(path).st_dev)
+        except:
+            return vid
+
+    def build_fallback(self) -> None:
+        self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
+        self.trusted = False
+
     def build_tab(self) -> None:
         self.log("building tab")
 
@@ -96,6 +105,9 @@ class Fstab(object):
     def relabel(self, path: str, nval: str) -> None:
         assert self.tab
         self.cache = {}
+        if ANYWIN:
+            path = self._winpath(path)
+
         path = path.lstrip("/")
         ptn = re.compile(r"^[^\\/]*")
         vn, rem = self.tab._find(path)
@@ -124,8 +136,7 @@ class Fstab(object):
             except:
                 # prisonparty or other restrictive environment
                 self.log("failed to build tab:\n{}".format(min_ex()), 3)
-                self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
-                self.trusted = False
+                self.build_fallback()
 
         assert self.tab
         ret = self.tab._find(path)[0]
@@ -135,43 +146,9 @@ class Fstab(object):
             return "idk"
 
     def get_w32(self, path: str) -> str:
-        # list mountpoints: fsutil fsinfo drives
+        if not self.tab:
+            self.build_fallback()
 
-        from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPDWORD, LPWSTR, MAX_PATH
-
-        def echk(rc: int, fun: Any, args: Any) -> None:
-            if not rc:
-                raise ctypes.WinError(ctypes.get_last_error())
-            return None
-
-        k32 = ctypes.WinDLL("kernel32", use_last_error=True)
-        k32.GetVolumeInformationW.errcheck = echk
-        k32.GetVolumeInformationW.restype = BOOL
-        k32.GetVolumeInformationW.argtypes = (
-            LPCWSTR,
-            LPWSTR,
-            DWORD,
-            LPDWORD,
-            LPDWORD,
-            LPDWORD,
-            LPWSTR,
-            DWORD,
-        )
-
-        bvolname = ctypes.create_unicode_buffer(MAX_PATH + 1)
-        bfstype = ctypes.create_unicode_buffer(MAX_PATH + 1)
-        serial = DWORD()
-        max_name_len = DWORD()
-        fs_flags = DWORD()
-
-        k32.GetVolumeInformationW(
-            path,
-            bvolname,
-            ctypes.sizeof(bvolname),
-            ctypes.byref(serial),
-            ctypes.byref(max_name_len),
-            ctypes.byref(fs_flags),
-            bfstype,
-            ctypes.sizeof(bfstype),
-        )
-        return bfstype.value
+        assert self.tab
+        ret = self.tab._find(path)[0]
+        return ret.realpath

copyparty/ftpd.py

@@ -391,7 +391,7 @@ class Ftpd(object):
             for h, lp in hs:
                 FTPServer((ip, int(lp)), h, ioloop)
 
-        thr = threading.Thread(target=ioloop.loop)
+        thr = threading.Thread(target=ioloop.loop, name="ftp")
         thr.daemon = True
         thr.start()
 

copyparty/httpcli.py

@@ -24,12 +24,7 @@ try:
 except:
     pass
 
-try:
-    import ctypes
-except:
-    pass
-
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E, unicode
 from .authsrv import VFS  # typechk
 from .bos import bos
 from .star import StreamTar
@@ -47,7 +42,9 @@ from .util import (
     exclude_dotfiles,
     fsenc,
     gen_filekey,
+    gen_filekey_dbg,
     gencookie,
+    get_df,
     get_spd,
     guess_mime,
     gzip_orig_sz,
@@ -112,6 +109,7 @@ class HttpCli(object):
         self.u2fh = conn.u2fh  # mypy404
         self.log_func = conn.log_func  # mypy404
         self.log_src = conn.log_src  # mypy404
+        self.gen_fk = self._gen_fk if self.args.log_fk else gen_filekey
         self.tls: bool = hasattr(self.s, "cipher")
 
         # placeholders; assigned by run()
@@ -181,6 +179,9 @@ class HttpCli(object):
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def _gen_fk(self, salt: str, fspath: str, fsize: int, inode: int) -> str:
+        return gen_filekey_dbg(salt, fspath, fsize, inode, self.log, self.args.log_fk)
+
     def j2s(self, name: str, **ka: Any) -> str:
         tpl = self.conn.hsrv.j2[name]
         ka["ts"] = self.conn.hsrv.cachebuster()
@@ -382,7 +383,10 @@ class HttpCli(object):
 
                 em = str(ex)
                 msg = em if pex == ex else min_ex()
-                self.log("{}\033[0m, {}".format(msg, self.vpath), 3)
+                self.log(
+                    "{}\033[0m, {}".format(msg, self.vpath),
+                    6 if em.startswith("client d/c ") else 3,
+                )
 
                 msg = "{}\r\nURL: {}\r\n".format(em, self.vpath)
                 if self.hint:
@@ -712,7 +716,7 @@ class HttpCli(object):
         reader, remains = self.get_body_reader()
         vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         lim = vfs.get_dbv(rem)[0].lim
-        fdir = os.path.join(vfs.realpath, rem)
+        fdir = vfs.canonical(rem)
         if lim:
             fdir, rem = lim.all(self.ip, rem, remains, fdir)
 
@@ -814,7 +818,7 @@ class HttpCli(object):
 
         vsuf = ""
         if self.can_read and "fk" in vfs.flags:
-            vsuf = "?k=" + gen_filekey(
+            vsuf = "?k=" + self.gen_fk(
                 self.args.fk_salt,
                 path,
                 post_sz,
@@ -922,7 +926,7 @@ class HttpCli(object):
         except:
             raise Pebkac(422, "you POSTed invalid json")
 
-        # self.reply(b" DD" + b"oS Protection ", 503)
+        # self.reply(b"cloudflare", 503)
         # return True
 
         if "srch" in self.uparam or "srch" in body:
@@ -951,7 +955,7 @@ class HttpCli(object):
 
         if rem:
             try:
-                dst = os.path.join(vfs.realpath, rem)
+                dst = vfs.canonical(rem)
                 if not bos.path.isdir(dst):
                     bos.makedirs(dst)
             except OSError as ex:
@@ -1186,7 +1190,7 @@ class HttpCli(object):
         sanitized = sanitize_fn(new_dir, "", [])
 
         if not nullwrite:
-            fdir = os.path.join(vfs.realpath, rem)
+            fdir = vfs.canonical(rem)
             fn = os.path.join(fdir, sanitized)
 
             if not bos.path.isdir(fdir):
@@ -1225,7 +1229,7 @@ class HttpCli(object):
         sanitized = sanitize_fn(new_file, "", [])
 
         if not nullwrite:
-            fdir = os.path.join(vfs.realpath, rem)
+            fdir = vfs.canonical(rem)
             fn = os.path.join(fdir, sanitized)
 
             if bos.path.exists(fn):
@@ -1246,7 +1250,7 @@ class HttpCli(object):
 
         upload_vpath = self.vpath
         lim = vfs.get_dbv(rem)[0].lim
-        fdir_base = os.path.join(vfs.realpath, rem)
+        fdir_base = vfs.canonical(rem)
         if lim:
             fdir_base, rem = lim.all(self.ip, rem, -1, fdir_base)
             upload_vpath = "{}/{}".format(vfs.vpath, rem).strip("/")
@@ -1287,14 +1291,19 @@ class HttpCli(object):
                 else:
                     open_args = {}
                     tnam = fname = os.devnull
-                    fdir = ""
+                    fdir = abspath = ""
 
                 if lim:
                     lim.chk_bup(self.ip)
                     lim.chk_nup(self.ip)
 
                 try:
-                    max_sz = lim.smax if lim else 0
+                    max_sz = 0
+                    if lim:
+                        v1 = lim.smax
+                        v2 = lim.dfv - lim.dfl
+                        max_sz = min(v1, v2) if v1 and v2 else v1 or v2
+
                     with ren_open(tnam, "wb", 512 * 1024, **open_args) as zfw:
                         f, tnam = zfw["orz"]
                         tabspath = os.path.join(fdir, tnam)
@@ -1309,16 +1318,20 @@ class HttpCli(object):
                         lim.nup(self.ip)
                         lim.bup(self.ip, sz)
                         try:
+                            lim.chk_df(tabspath, sz, True)
                             lim.chk_sz(sz)
                             lim.chk_bup(self.ip)
                             lim.chk_nup(self.ip)
                         except:
-                            bos.unlink(tabspath)
-                            bos.unlink(abspath)
+                            if not nullwrite:
+                                bos.unlink(tabspath)
+                                bos.unlink(abspath)
                             fname = os.devnull
                             raise
 
-                    atomic_move(tabspath, abspath)
+                    if not nullwrite:
+                        atomic_move(tabspath, abspath)
+
                     files.append(
                         (sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath)
                     )
@@ -1368,9 +1381,9 @@ class HttpCli(object):
         for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
             vsuf = ""
             if self.can_read and "fk" in vfs.flags:
-                vsuf = "?k=" + gen_filekey(
+                vsuf = "?k=" + self.gen_fk(
                     self.args.fk_salt,
-                    abspath,
+                    ap,
                     sz,
                     0 if ANYWIN or not ap else bos.stat(ap).st_ino,
                 )[: vfs.flags["fk"]]
@@ -1448,7 +1461,7 @@ class HttpCli(object):
             raise Pebkac(411)
 
         rp, fn = vsplit(rem)
-        fp = os.path.join(vfs.realpath, rp)
+        fp = vfs.canonical(rp)
         lim = vfs.get_dbv(rem)[0].lim
         if lim:
             fp, rp = lim.all(self.ip, rp, clen, fp)
@@ -1922,7 +1935,13 @@ class HttpCli(object):
             vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()}
         else:
             vstate = {}
-            vs = {"scanning": None, "hashq": None, "tagq": None, "mtpq": None}
+            vs = {
+                "scanning": None,
+                "hashq": None,
+                "tagq": None,
+                "mtpq": None,
+                "dbwt": None,
+            }
 
         if self.uparam.get("ls") in ["v", "t", "txt"]:
             if self.uname == "*":
@@ -1932,7 +1951,7 @@ class HttpCli(object):
 
             if vstate:
                 txt += "\nstatus:"
-                for k in ["scanning", "hashq", "tagq", "mtpq"]:
+                for k in ["scanning", "hashq", "tagq", "mtpq", "dbwt"]:
                     txt += " {}({})".format(k, vs[k])
 
             if rvol:
@@ -1961,6 +1980,7 @@ class HttpCli(object):
             hashq=vs["hashq"],
             tagq=vs["tagq"],
             mtpq=vs["mtpq"],
+            dbwt=vs["dbwt"],
             url_suf=suf,
             k304=self.k304(),
         )
@@ -2298,7 +2318,7 @@ class HttpCli(object):
 
         if not is_dir and (self.can_read or self.can_get):
             if not self.can_read and "fk" in vn.flags:
-                correct = gen_filekey(
+                correct = self.gen_fk(
                     self.args.fk_salt, abspath, st.st_size, 0 if ANYWIN else st.st_ino
                 )[: vn.flags["fk"]]
                 got = self.uparam.get("k")
@@ -2322,26 +2342,14 @@ class HttpCli(object):
         except:
             self.log("#wow #whoa")
 
-        try:
-            # some fuses misbehave
-            if not self.args.nid:
-                if WINDOWS:
-                    try:
-                        bfree = ctypes.c_ulonglong(0)
-                        ctypes.windll.kernel32.GetDiskFreeSpaceExW(  # type: ignore
-                            ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
-                        )
-                        srv_info.append(humansize(bfree.value) + " free")
-                    except:
-                        pass
-                else:
-                    sv = os.statvfs(fsenc(abspath))
-                    free = humansize(sv.f_frsize * sv.f_bfree, True)
-                    total = humansize(sv.f_frsize * sv.f_blocks, True)
-
-                    srv_info.append("{} free of {}".format(free, total))
-        except:
-            pass
+        if not self.args.nid:
+            free, total = get_df(abspath)
+            if total is not None:
+                h1 = humansize(free or 0)
+                h2 = humansize(total)
+                srv_info.append("{} free of {}".format(h1, h2))
+            elif free is not None:
+                srv_info.append(humansize(free, True) + " free")
 
         srv_infot = "</span> // <span>".join(srv_info)
 
@@ -2534,7 +2542,7 @@ class HttpCli(object):
             if add_fk:
                 href = "{}?k={}".format(
                     quotep(href),
-                    gen_filekey(
+                    self.gen_fk(
                         self.args.fk_salt, fspath, sz, 0 if ANYWIN else inf.st_ino
                     )[:add_fk],
                 )

copyparty/httpconn.py

@@ -62,7 +62,7 @@ class HttpConn(object):
         self.nreq: int = 0  # mypy404
         self.nbyte: int = 0  # mypy404
         self.u2idx: Optional[U2idx] = None
-        self.log_func: Util.RootLogger = hsrv.log  # mypy404
+        self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
         self.log_src: str = "httpconn"  # mypy404
         self.lf_url: Optional[Pattern[str]] = (
             re.compile(self.args.lf_url) if self.args.lf_url else None

copyparty/httpsrv.py

@@ -102,7 +102,7 @@ class HttpSrv(object):
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
         self.th_cfg: dict[str, Any] = {}
-        t = threading.Thread(target=self.post_init)
+        t = threading.Thread(target=self.post_init, name="hsrv-init2")
         t.daemon = True
         t.start()
 
@@ -165,13 +165,13 @@ class HttpSrv(object):
         """listens on a shared tcp server"""
         ip, port = srv_sck.getsockname()
         fno = srv_sck.fileno()
-        msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
+        msg = "subscribed @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
         self.log(self.name, msg)
 
         def fun() -> None:
             self.broker.say("cb_httpsrv_up")
 
-        threading.Thread(target=fun).start()
+        threading.Thread(target=fun, name="sig-hsrv-up1").start()
 
         while not self.stopping:
             if self.args.log_conn:
@@ -261,8 +261,11 @@ class HttpSrv(object):
                 )
                 self.thr_client(sck, addr)
                 me.name = self.name + "-poolw"
-            except:
-                self.log(self.name, "thr_client: " + min_ex(), 3)
+            except Exception as ex:
+                if str(ex).startswith("client d/c "):
+                    self.log(self.name, "thr_client: " + str(ex), 6)
+                else:
+                    self.log(self.name, "thr_client: " + min_ex(), 3)
 
     def shutdown(self) -> None:
         self.stopping = True

copyparty/ico.py

@@ -15,7 +15,7 @@ class Ico(object):
     def get(self, ext: str, as_thumb: bool) -> tuple[str, bytes]:
         """placeholder to make thumbnails not break"""
 
-        zb = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
         if PY2:
             zb = [ord(x) for x in zb]
 

copyparty/mtag.py

@@ -248,7 +248,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
 
 
 class MTag(object):
-    def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None:
+    def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
         self.log_func = log_func
         self.args = args
         self.usable = True
@@ -437,6 +437,8 @@ class MTag(object):
         return r1
 
     def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
+        ret: dict[str, tuple[int, Any]] = {}
+
         if not bos.path.isfile(abspath):
             return {}
 
@@ -450,7 +452,10 @@ class MTag(object):
             return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
         sz = bos.path.getsize(abspath)
-        ret = {".q": (0, int((sz / md.info.length) / 128))}
+        try:
+            ret[".q"] = (0, int((sz / md.info.length) / 128))
+        except:
+            pass
 
         for attr, k, norm in [
             ["codec", "ac", unicode],

copyparty/star.py

@@ -44,7 +44,7 @@ class StreamTar(StreamArc):
 
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):
@@ -65,17 +65,19 @@ class StreamTar(StreamArc):
         w.start()
 
     def gen(self) -> Generator[Optional[bytes], None, None]:
-        while True:
-            buf = self.qfile.q.get()
-            if not buf:
-                break
+        try:
+            while True:
+                buf = self.qfile.q.get()
+                if not buf:
+                    break
 
-            self.co += len(buf)
-            yield buf
+                self.co += len(buf)
+                yield buf
 
-        yield None
-        if self.errf:
-            bos.unlink(self.errf["ap"])
+            yield None
+        finally:
+            if self.errf:
+                bos.unlink(self.errf["ap"])
 
     def ser(self, f: dict[str, Any]) -> None:
         name = f["vp"]

copyparty/sutil.py

@@ -17,7 +17,7 @@ except:
 class StreamArc(object):
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):

copyparty/svchub.py

@@ -6,6 +6,7 @@ import base64
 import calendar
 import gzip
 import os
+import re
 import shlex
 import signal
 import socket
@@ -19,7 +20,7 @@ try:
     from types import FrameType
 
     import typing
-    from typing import Optional, Union
+    from typing import Any, Optional, Union
 except:
     pass
 
@@ -29,7 +30,15 @@ from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
 from .tcpsrv import TcpSrv
 from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
 from .up2k import Up2k
-from .util import ansi_re, min_ex, mp, start_log_thrs, start_stackmon, alltrace
+from .util import (
+    VERSIONS,
+    alltrace,
+    ansi_re,
+    min_ex,
+    mp,
+    start_log_thrs,
+    start_stackmon,
+)
 
 
 class SvcHub(object):
@@ -49,10 +58,12 @@ class SvcHub(object):
         self.logf: Optional[typing.TextIO] = None
         self.logf_base_fn = ""
         self.stop_req = False
-        self.reload_req = False
         self.stopping = False
+        self.stopped = False
+        self.reload_req = False
         self.reloading = False
         self.stop_cond = threading.Condition()
+        self.nsigs = 3
         self.retcode = 0
         self.httpsrv_up = 0
 
@@ -113,6 +124,9 @@ class SvcHub(object):
         if not args.hardlink and args.never_symlink:
             args.no_dedup = True
 
+        if args.log_fk:
+            args.log_fk = re.compile(args.log_fk)
+
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log)
         if args.ls:
@@ -132,8 +146,8 @@ class SvcHub(object):
         self.args.th_dec = list(decs.keys())
         self.thumbsrv = None
         if not args.no_thumb:
-            t = "decoder preference: {}".format(", ".join(self.args.th_dec))
-            self.log("thumb", t)
+            t = ", ".join(self.args.th_dec) or "(None available)"
+            self.log("thumb", "decoder preference: {}".format(t))
 
             if "pil" in self.args.th_dec and not HAVE_WEBP:
                 msg = "disabling webp thumbnails because either libwebp is not available or your Pillow is too old"
@@ -192,6 +206,9 @@ class SvcHub(object):
         self.log("root", t, 1)
 
         self.retcode = 1
+        self.sigterm()
+
+    def sigterm(self) -> None:
         os.kill(os.getpid(), signal.SIGTERM)
 
     def cb_httpsrv_up(self) -> None:
@@ -255,7 +272,7 @@ class SvcHub(object):
     def run(self) -> None:
         self.tcpsrv.run()
 
-        thr = threading.Thread(target=self.thr_httpsrv_up)
+        thr = threading.Thread(target=self.thr_httpsrv_up, name="sig-hsrv-up2")
         thr.daemon = True
         thr.start()
 
@@ -283,7 +300,9 @@ class SvcHub(object):
                 pass
 
             self.shutdown()
-            thr.join()
+            # cant join; eats signals on win10
+            while not self.stopped:
+                time.sleep(0.1)
         else:
             self.stop_thr()
 
@@ -292,7 +311,7 @@ class SvcHub(object):
             return "cannot reload; already in progress"
 
         self.reloading = True
-        t = threading.Thread(target=self._reload)
+        t = threading.Thread(target=self._reload, name="reloading")
         t.daemon = True
         t.start()
         return "reload initiated"
@@ -319,9 +338,22 @@ class SvcHub(object):
 
     def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
         if self.stopping:
-            return
+            if self.nsigs <= 0:
+                try:
+                    threading.Thread(target=self.pr, args=("OMBO BREAKER",)).start()
+                    time.sleep(0.1)
+                except:
+                    pass
 
-        if sig == signal.SIGUSR1:
+                if ANYWIN:
+                    os.system("taskkill /f /pid {}".format(os.getpid()))
+                else:
+                    os.kill(os.getpid(), signal.SIGKILL)
+            else:
+                self.nsigs -= 1
+                return
+
+        if not ANYWIN and sig == signal.SIGUSR1:
             self.reload_req = True
         else:
             self.stop_req = True
@@ -342,9 +374,7 @@ class SvcHub(object):
 
         ret = 1
         try:
-            with self.log_mutex:
-                print("OPYTHAT")
-
+            self.pr("OPYTHAT")
             self.tcpsrv.shutdown()
             self.broker.shutdown()
             self.up2k.shutdown()
@@ -357,22 +387,23 @@ class SvcHub(object):
                         break
 
                     if n == 3:
-                        print("waiting for thumbsrv (10sec)...")
+                        self.pr("waiting for thumbsrv (10sec)...")
 
-            print("nailed it", end="")
+            self.pr("nailed it", end="")
             ret = self.retcode
         except:
-            print("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
+            self.pr("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
             raise
         finally:
             if self.args.wintitle:
                 print("\033]0;\033\\", file=sys.stderr, end="")
                 sys.stderr.flush()
 
-            print("\033[0m")
+            self.pr("\033[0m")
             if self.logf:
                 self.logf.close()
 
+            self.stopped = True
             sys.exit(ret)
 
     def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
@@ -439,6 +470,10 @@ class SvcHub(object):
             if self.logf:
                 self.logf.write(msg)
 
+    def pr(self, *a: Any, **ka: Any) -> None:
+        with self.log_mutex:
+            print(*a, **ka)
+
     def check_mp_support(self) -> str:
         vmin = sys.version_info[1]
         if WINDOWS:
@@ -511,7 +546,8 @@ class SvcHub(object):
             return
 
         self.tstack = time.time()
-        zb = alltrace().encode("utf-8", "replace")
+        zs = "{}\n{}".format(VERSIONS, alltrace())
+        zb = zs.encode("utf-8", "replace")
         zb = gzip.compress(zb)
         zs = base64.b64encode(zb).decode("ascii")
         self.log("stacks", zs)

copyparty/szip.py

@@ -218,7 +218,7 @@ def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
 class StreamZip(StreamArc):
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         utf8: bool = False,
         pre_crc: bool = False,
@@ -272,41 +272,44 @@ class StreamZip(StreamArc):
 
     def gen(self) -> Generator[bytes, None, None]:
         errors = []
-        for f in self.fgen:
-            if "err" in f:
-                errors.append((f["vp"], f["err"]))
-                continue
+        try:
+            for f in self.fgen:
+                if "err" in f:
+                    errors.append((f["vp"], f["err"]))
+                    continue
 
-            try:
-                for x in self.ser(f):
+                try:
+                    for x in self.ser(f):
+                        yield x
+                except GeneratorExit:
+                    raise
+                except:
+                    ex = min_ex(5, True).replace("\n", "\n-- ")
+                    errors.append((f["vp"], ex))
+
+            if errors:
+                errf, txt = errdesc(errors)
+                self.log("\n".join(([repr(errf)] + txt[1:])))
+                for x in self.ser(errf):
                     yield x
-            except:
-                ex = min_ex(5, True).replace("\n", "\n-- ")
-                errors.append((f["vp"], ex))
 
-        if errors:
-            errf, txt = errdesc(errors)
-            self.log("\n".join(([repr(errf)] + txt[1:])))
-            for x in self.ser(errf):
-                yield x
+            cdir_pos = self.pos
+            for name, sz, ts, crc, h_pos in self.items:
+                buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+                yield self._ct(buf)
+            cdir_end = self.pos
 
-        cdir_pos = self.pos
-        for name, sz, ts, crc, h_pos in self.items:
-            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
-            yield self._ct(buf)
-        cdir_end = self.pos
+            _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+            if need_64:
+                ecdir64_pos = self.pos
+                buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+                yield self._ct(buf)
 
-        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
-        if need_64:
-            ecdir64_pos = self.pos
-            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
-            yield self._ct(buf)
+                buf = gen_ecdr64_loc(ecdir64_pos)
+                yield self._ct(buf)
 
-            buf = gen_ecdr64_loc(ecdir64_pos)
-            yield self._ct(buf)
-
-        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
-        yield self._ct(ecdr)
-
-        if errors:
-            bos.unlink(errf["ap"])
+            ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+            yield self._ct(ecdr)
+        finally:
+            if errors:
+                bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import os
 import re
 import socket
 import sys
@@ -128,7 +129,7 @@ class TcpSrv(object):
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
             fno = srv.fileno()
-            msg = "listening @ {}:{}  f{}".format(ip, port, fno)
+            msg = "listening @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)

copyparty/util.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import base64
 import contextlib
 import hashlib
+import math
 import mimetypes
 import os
 import platform
@@ -21,9 +22,17 @@ import traceback
 from collections import Counter
 from datetime import datetime
 
+from queue import Queue
+
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, WINDOWS
+from .__version__ import S_BUILD_DT, S_VERSION
 from .stolen import surrogateescape
 
+try:
+    import ctypes
+except:
+    pass
+
 try:
     HAVE_SQLITE3 = True
     import sqlite3  # pylint: disable=unused-import  # typechk
@@ -41,7 +50,7 @@ try:
     from collections.abc import Callable, Iterable
 
     import typing
-    from typing import Any, Generator, Optional, Protocol, Union
+    from typing import Any, Generator, Optional, Pattern, Protocol, Union
 
     class RootLogger(Protocol):
         def __call__(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
@@ -78,8 +87,6 @@ else:
     from urllib import quote  # pylint: disable=no-name-in-module
     from urllib import unquote  # pylint: disable=no-name-in-module
 
-_: Any = (mp, BytesIO, quote, unquote)
-__all__ = ["mp", "BytesIO", "quote", "unquote"]
 
 try:
     struct.unpack(b">i", b"idgi")
@@ -208,6 +215,72 @@ REKOBO_KEY = {
 REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
 
 
+def py_desc() -> str:
+    interp = platform.python_implementation()
+    py_ver = ".".join([str(x) for x in sys.version_info])
+    ofs = py_ver.find(".final.")
+    if ofs > 0:
+        py_ver = py_ver[:ofs]
+
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
+    host_os = platform.system()
+    compiler = platform.python_compiler()
+
+    m = re.search(r"([0-9]+\.[0-9\.]+)", platform.version())
+    os_ver = m.group(1) if m else ""
+
+    return "{:>9} v{} on {}{} {} [{}]".format(
+        interp, py_ver, host_os, bitness, os_ver, compiler
+    )
+
+
+def _sqlite_ver() -> str:
+    try:
+        co = sqlite3.connect(":memory:")
+        cur = co.cursor()
+        try:
+            vs = cur.execute("select * from pragma_compile_options").fetchall()
+        except:
+            vs = cur.execute("pragma compile_options").fetchall()
+
+        v = next(x[0].split("=")[1] for x in vs if x[0].startswith("THREADSAFE="))
+        cur.close()
+        co.close()
+    except:
+        v = "W"
+
+    return "{}*{}".format(sqlite3.sqlite_version, v)
+
+
+try:
+    SQLITE_VER = _sqlite_ver()
+except:
+    SQLITE_VER = "(None)"
+
+try:
+    from jinja2 import __version__ as JINJA_VER
+except:
+    JINJA_VER = "(None)"
+
+try:
+    from pyftpdlib.__init__ import __ver__ as PYFTPD_VER
+except:
+    PYFTPD_VER = "(None)"
+
+
+VERSIONS = "copyparty v{} ({})\n{}\n   sqlite v{} | jinja v{} | pyftpd v{}".format(
+    S_VERSION, S_BUILD_DT, py_desc(), SQLITE_VER, JINJA_VER, PYFTPD_VER
+)
+
+
+_: Any = (mp, BytesIO, quote, unquote, SQLITE_VER, JINJA_VER, PYFTPD_VER)
+__all__ = ["mp", "BytesIO", "quote", "unquote", "SQLITE_VER", "JINJA_VER", "PYFTPD_VER"]
+
+
 class Cooldown(object):
     def __init__(self, maxage: float) -> None:
         self.maxage = maxage
@@ -243,7 +316,7 @@ class _Unrecv(object):
     undo any number of socket recv ops
     """
 
-    def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
+    def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
         self.s = s
         self.log = log
         self.buf: bytes = b""
@@ -287,7 +360,7 @@ class _LUnrecv(object):
     with expensive debug logging
     """
 
-    def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
+    def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
         self.s = s
         self.log = log
         self.buf = b""
@@ -424,6 +497,104 @@ class ProgressPrinter(threading.Thread):
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
 
 
+class MTHash(object):
+    def __init__(self, cores: int):
+        self.pp: Optional[ProgressPrinter] = None
+        self.f: Optional[typing.BinaryIO] = None
+        self.sz = 0
+        self.csz = 0
+        self.stop = False
+        self.omutex = threading.Lock()
+        self.imutex = threading.Lock()
+        self.work_q: Queue[int] = Queue()
+        self.done_q: Queue[tuple[int, str, int, int]] = Queue()
+        self.thrs = []
+        for n in range(cores):
+            t = threading.Thread(target=self.worker, name="mth-" + str(n))
+            t.daemon = True
+            t.start()
+            self.thrs.append(t)
+
+    def hash(
+        self,
+        f: typing.BinaryIO,
+        fsz: int,
+        chunksz: int,
+        pp: Optional[ProgressPrinter] = None,
+        prefix: str = "",
+        suffix: str = "",
+    ) -> list[tuple[str, int, int]]:
+        with self.omutex:
+            self.f = f
+            self.sz = fsz
+            self.csz = chunksz
+
+            chunks: dict[int, tuple[str, int, int]] = {}
+            nchunks = int(math.ceil(fsz / chunksz))
+            for nch in range(nchunks):
+                self.work_q.put(nch)
+
+            ex = ""
+            for nch in range(nchunks):
+                qe = self.done_q.get()
+                try:
+                    nch, dig, ofs, csz = qe
+                    chunks[nch] = (dig, ofs, csz)
+                except:
+                    ex = ex or str(qe)
+
+                if pp:
+                    mb = int((fsz - nch * chunksz) / 1024 / 1024)
+                    pp.msg = prefix + str(mb) + suffix
+
+            if ex:
+                raise Exception(ex)
+
+            ret = []
+            for n in range(nchunks):
+                ret.append(chunks[n])
+
+            self.f = None
+            self.csz = 0
+            self.sz = 0
+            return ret
+
+    def worker(self) -> None:
+        while True:
+            ofs = self.work_q.get()
+            try:
+                v = self.hash_at(ofs)
+            except Exception as ex:
+                v = str(ex)  # type: ignore
+
+            self.done_q.put(v)
+
+    def hash_at(self, nch: int) -> tuple[int, str, int, int]:
+        f = self.f
+        ofs = ofs0 = nch * self.csz
+        chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
+        if self.stop:
+            return nch, "", ofs0, chunk_sz
+
+        assert f
+        hashobj = hashlib.sha512()
+        while chunk_rem > 0:
+            with self.imutex:
+                f.seek(ofs)
+                buf = f.read(min(chunk_rem, 1024 * 1024 * 12))
+
+            if not buf:
+                raise Exception("EOF at " + str(ofs))
+
+            hashobj.update(buf)
+            chunk_rem -= len(buf)
+            ofs += len(buf)
+
+        bdig = hashobj.digest()[:33]
+        udig = base64.urlsafe_b64encode(bdig).decode("utf-8")
+        return nch, udig, ofs0, chunk_sz
+
+
 def uprint(msg: str) -> None:
     try:
         print(msg, end="")
@@ -662,7 +833,9 @@ def ren_open(
 
 
 class MultipartParser(object):
-    def __init__(self, log_func: NamedLogger, sr: Unrecv, http_headers: dict[str, str]):
+    def __init__(
+        self, log_func: "NamedLogger", sr: Unrecv, http_headers: dict[str, str]
+    ):
         self.sr = sr
         self.log = log_func
         self.headers = http_headers
@@ -925,6 +1098,24 @@ def gen_filekey(salt: str, fspath: str, fsize: int, inode: int) -> str:
     ).decode("ascii")
 
 
+def gen_filekey_dbg(
+    salt: str,
+    fspath: str,
+    fsize: int,
+    inode: int,
+    log: "NamedLogger",
+    log_ptn: Optional[Pattern[str]],
+) -> str:
+    ret = gen_filekey(salt, fspath, fsize, inode)
+
+    assert log_ptn
+    if log_ptn.search(fspath):
+        t = "fk({}) salt({}) size({}) inode({}) fspath({})"
+        log(t.format(ret[:8], salt, fsize, inode, fspath))
+
+    return ret
+
+
 def gencookie(k: str, v: str, dur: Optional[int]) -> str:
     v = v.replace(";", "")
     if dur:
@@ -982,6 +1173,11 @@ def s2hms(s: float, optional_h: bool = False) -> str:
     return "{}:{:02}:{:02}".format(h, m, s)
 
 
+def djoin(*paths: str) -> str:
+    """joins without adding a trailing slash on blank args"""
+    return os.path.join(*[x for x in paths if x])
+
+
 def uncyg(path: str) -> str:
     if len(path) < 2 or not path.startswith("/"):
         return path
@@ -1131,6 +1327,10 @@ def vsplit(vpath: str) -> tuple[str, str]:
     return vpath.rsplit("/", 1)  # type: ignore
 
 
+def vjoin(rd: str, fn: str) -> str:
+    return rd + "/" + fn if rd else fn
+
+
 def w8dec(txt: bytes) -> str:
     """decodes filesystem-bytes to wtf8"""
     if PY2:
@@ -1184,15 +1384,30 @@ def s3enc(mem_cur: "sqlite3.Cursor", rd: str, fn: str) -> tuple[str, str]:
 
 
 def s3dec(rd: str, fn: str) -> tuple[str, str]:
-    ret = []
-    for v in [rd, fn]:
-        if v.startswith("//"):
-            ret.append(w8b64dec(v[2:]))
-            # self.log("mojide [{}] {}".format(ret[-1], v[2:]))
-        else:
-            ret.append(v)
+    return (
+        w8b64dec(rd[2:]) if rd.startswith("//") else rd,
+        w8b64dec(fn[2:]) if fn.startswith("//") else fn,
+    )
 
-    return ret[0], ret[1]
+
+def db_ex_chk(log: "NamedLogger", ex: Exception, db_path: str) -> bool:
+    if str(ex) != "database is locked":
+        return False
+
+    thr = threading.Thread(target=lsof, args=(log, db_path), name="dbex")
+    thr.daemon = True
+    thr.start()
+
+    return True
+
+
+def lsof(log: "NamedLogger", abspath: str) -> None:
+    try:
+        rc, so, se = runcmd([b"lsof", b"-R", fsenc(abspath)], timeout=5)
+        zs = (so.strip() + "\n" + se.strip()).strip()
+        log("lsof {} = {}\n{}".format(abspath, rc, zs), 3)
+    except:
+        log("lsof failed; " + min_ex(), 3)
 
 
 def atomic_move(usrc: str, udst: str) -> None:
@@ -1207,6 +1422,24 @@ def atomic_move(usrc: str, udst: str) -> None:
         os.rename(src, dst)
 
 
+def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]:
+    try:
+        # some fuses misbehave
+        if ANYWIN:
+            bfree = ctypes.c_ulonglong(0)
+            ctypes.windll.kernel32.GetDiskFreeSpaceExW(  # type: ignore
+                ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+            )
+            return (bfree.value, None)
+        else:
+            sv = os.statvfs(fsenc(abspath))
+            free = sv.f_frsize * sv.f_bfree
+            total = sv.f_frsize * sv.f_blocks
+            return (free, total)
+    except:
+        return (None, None)
+
+
 def read_socket(sr: Unrecv, total_size: int) -> Generator[bytes, None, None]:
     remains = total_size
     while remains > 0:
@@ -1233,7 +1466,7 @@ def read_socket_unbounded(sr: Unrecv) -> Generator[bytes, None, None]:
 
 
 def read_socket_chunked(
-    sr: Unrecv, log: Optional[NamedLogger] = None
+    sr: Unrecv, log: Optional["NamedLogger"] = None
 ) -> Generator[bytes, None, None]:
     err = "upload aborted: expected chunk length, got [{}] |{}| instead"
     while True:
@@ -1311,7 +1544,7 @@ def hashcopy(
 
 
 def sendfile_py(
-    log: NamedLogger,
+    log: "NamedLogger",
     lower: int,
     upper: int,
     f: typing.BinaryIO,
@@ -1339,7 +1572,7 @@ def sendfile_py(
 
 
 def sendfile_kern(
-    log: NamedLogger,
+    log: "NamedLogger",
     lower: int,
     upper: int,
     f: typing.BinaryIO,
@@ -1380,7 +1613,7 @@ def sendfile_kern(
 
 
 def statdir(
-    logger: Optional[RootLogger], scandir: bool, lstat: bool, top: str
+    logger: Optional["RootLogger"], scandir: bool, lstat: bool, top: str
 ) -> Generator[tuple[str, os.stat_result], None, None]:
     if lstat and ANYWIN:
         lstat = False
@@ -1423,7 +1656,7 @@ def statdir(
 
 
 def rmdirs(
-    logger: RootLogger, scandir: bool, lstat: bool, top: str, depth: int
+    logger: "RootLogger", scandir: bool, lstat: bool, top: str, depth: int
 ) -> tuple[list[str], list[str]]:
     """rmdir all descendants, then self"""
     if not os.path.isdir(fsenc(top)):
@@ -1644,7 +1877,7 @@ def retchk(
     rc: int,
     cmd: Union[list[bytes], list[str]],
     serr: str,
-    logger: Optional[NamedLogger] = None,
+    logger: Optional["NamedLogger"] = None,
     color: Union[int, str] = 0,
     verbose: bool = False,
 ) -> None:
@@ -1696,29 +1929,6 @@ def gzip_orig_sz(fn: str) -> int:
         return sunpack(b"I", rv)[0]  # type: ignore
 
 
-def py_desc() -> str:
-    interp = platform.python_implementation()
-    py_ver = ".".join([str(x) for x in sys.version_info])
-    ofs = py_ver.find(".final.")
-    if ofs > 0:
-        py_ver = py_ver[:ofs]
-
-    try:
-        bitness = struct.calcsize(b"P") * 8
-    except:
-        bitness = struct.calcsize("P") * 8
-
-    host_os = platform.system()
-    compiler = platform.python_compiler()
-
-    m = re.search(r"([0-9]+\.[0-9\.]+)", platform.version())
-    os_ver = m.group(1) if m else ""
-
-    return "{:>9} v{} on {}{} {} [{}]".format(
-        interp, py_ver, host_os, bitness, os_ver, compiler
-    )
-
-
 def align_tab(lines: list[str]) -> list[str]:
     rows = []
     ncols = 0

copyparty/web/baguettebox.js

@@ -224,6 +224,7 @@ window.baguetteBox = (function () {
             ['space, P, K', 'video: play / pause'],
             ['U', 'video: seek 10sec back'],
             ['P', 'video: seek 10sec ahead'],
+            ['0..9', 'video: seek 0%..90%'],
             ['M', 'video: toggle mute'],
             ['V', 'video: toggle loop'],
             ['C', 'video: toggle auto-next'],
@@ -248,7 +249,7 @@ window.baguetteBox = (function () {
         if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing || modal.busy)
             return;
 
-        var k = e.code + '', v = vid();
+        var k = e.code + '', v = vid(), pos = -1;
 
         if (k == "ArrowLeft" || k == "KeyJ")
             showPreviousImage();
@@ -264,6 +265,8 @@ window.baguetteBox = (function () {
             playpause();
         else if (k == "KeyU" || k == "KeyO")
             relseek(k == "KeyU" ? -10 : 10);
+        else if (k.indexOf('Digit') === 0)
+            vid().currentTime = vid().duration * parseInt(k.slice(-1)) * 0.1;
         else if (k == "KeyM" && v) {
             v.muted = vmute = !vmute;
             mp_ctl();
@@ -696,18 +699,12 @@ window.baguetteBox = (function () {
             showOverlay(index);
             return true;
         }
-        if (index < 0) {
-            if (options.animation)
-                bounceAnimation('left');
 
-            return false;
-        }
-        if (index >= imagesElements.length) {
-            if (options.animation)
-                bounceAnimation('right');
+        if (index < 0)
+            return bounceAnimation('left');
 
-            return false;
-        }
+        if (index >= imagesElements.length)
+            return bounceAnimation('right');
 
         var v = vid();
         if (v) {
@@ -890,10 +887,11 @@ window.baguetteBox = (function () {
     }
 
     function bounceAnimation(direction) {
-        slider.className = 'bounce-from-' + direction;
+        slider.className = options.animation == 'slideIn' ? 'bounce-from-' + direction : 'eog';
         setTimeout(function () {
             slider.className = '';
-        }, 400);
+        }, 300);
+        return false;
     }
 
     function updateOffset() {

copyparty/web/browser.css

@@ -259,7 +259,7 @@ html.bz {
 	--bg-d2: #34384e;
 	--bg-d3: #34384e;
 
-	--row-alt: rgba(139, 150, 205, 0.06);
+	--row-alt: #181a27;
 
 	--btn-bg: #202231;
 	--btn-h-bg: #2d2f45;
@@ -309,7 +309,7 @@ html.c {
 	--a-gray: #0ae;
 
 	--tab-alt: #6ef;
-	--row-alt: rgba(180,0,255,0.3);
+	--row-alt: #47237d;
 	--scroll: #ff0;
 
 	--btn-fg: #fff;
@@ -504,7 +504,7 @@ html.dy {
 	--a: #000;
 	--a-b: #000;
 	--a-hil: #000;
-	--a-gray: #000;
+	--a-gray: #bbb;
 	--a-dark: #000;
 
 	--btn-fg: #000;
@@ -544,6 +544,9 @@ html.dy {
 
 	--tree-bg: #fff;
 
+	--g-sel-bg: #000;
+	--g-fsel-bg: #444;
+	--g-fsel-ts: #000;
 	--g-fg: a;
 	--g-bg: a;
 	--g-b1: a;
@@ -707,6 +710,7 @@ html.y #files thead th {
 #files td {
 	margin: 0;
 	padding: .3em .5em;
+	background: var(--bg);
 }
 #files tr:nth-child(2n) td {
 	background: var(--row-alt);
@@ -1595,9 +1599,6 @@ html.y #tree.nowrap .ntree a+a:hover {
 	margin: .7em 0 .7em .5em;
 	padding-left: .5em;
 }
-.opwide>div.fill {
-	display: block;
-}
 .opwide>div>div>a {
 	line-height: 2em;
 }
@@ -1908,10 +1909,13 @@ html.y #bbox-overlay figcaption a {
 	transition: left .2s ease, transform .2s ease;
 }
 .bounce-from-right {
-	animation: bounceFromRight .4s ease-out;
+	animation: bounceFromRight .3s ease-out;
 }
 .bounce-from-left {
-	animation: bounceFromLeft .4s ease-out;
+	animation: bounceFromLeft .3s ease-out;
+}
+.eog {
+	animation: eog .2s;
 }
 @keyframes bounceFromRight {
 	0% {margin-left: 0}
@@ -1923,6 +1927,9 @@ html.y #bbox-overlay figcaption a {
 	50% {margin-left: 30px}
 	100% {margin-left: 0}
 }
+@keyframes eog {
+	0% {filter: brightness(1.5)}
+}
 #bbox-next,
 #bbox-prev {
 	top: 50%;

copyparty/web/browser.js

@@ -106,6 +106,7 @@ var Ls = {
 
 		"ct_thumb": "in icon view, toggle icons or thumbnails$NHotkey: T",
 		"ct_dots": "show hidden files (if server permits)",
+		"ct_dir1st": "sort folders before files",
 		"ct_readme": "show README.md in folder listings",
 
 		"cut_turbo": "the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them",
@@ -116,6 +117,8 @@ var Ls = {
 
 		"cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN",
 
+		"cut_mt": "use multithreading to accelerate file hashing$N$Nthis uses web-workers and requires$Nmore RAM (up to 512 MiB extra)$N$N30% faster https, 4.5x faster http,$Nand 5.3x faster on android phones",
+
 		"cft_text": "favicon text (blank and refresh to disable)",
 		"cft_fg": "foreground color",
 		"cft_bg": "background color",
@@ -146,7 +149,7 @@ var Ls = {
 		"mt_caac": "convert aac / m4a to opus\">aac",
 		"mt_coth": "convert all others (not mp3) to opus\">oth",
 		"mt_tint": "background level (0-100) on the seekbar$Nto make buffering less distracting",
-		"mt_eq": "enables the equalizer and gain control;$Nboost 0 = unmodified 100% volume$N$Nenabling the equalizer makes gapless albums fully gapless, so leave it on with all the values at zero if you care about that",
+		"mt_eq": "enables the equalizer and gain control;$N$Nboost &lt;code&gt;0&lt;/code&gt; = standard 100% volume (unmodified)$N$Nwidth &lt;code&gt;1 &nbsp;&lt;/code&gt; = standard stereo (unmodified)$Nwidth &lt;code&gt;0.5&lt;/code&gt; = 50% left-right crossfeed$Nwidth &lt;code&gt;0 &nbsp;&lt;/code&gt; = mono$N$Nboost &lt;code&gt;-0.8&lt;/code&gt; &amp; width &lt;code&gt;10&lt;/code&gt; = vocal removal :^)$N$Nenabling the equalizer makes gapless albums fully gapless, so leave it on with all the values at zero (except width = 1) if you care about that",
 
 		"mb_play": "play",
 		"mm_hashplay": "play this audio file?",
@@ -286,8 +289,9 @@ var Ls = {
 
 		"u_https1": "you should",
 		"u_https2": "switch to https",
-		"u_https3": "for much better performance",
+		"u_https3": "for better performance",
 		"u_ancient": 'your browser is impressively ancient -- maybe you should <a href="#" onclick="goto(\'bup\')">use bup instead</a>',
+		"u_nowork": "need firefox 53+ or chrome 57+ or iOS 11+",
 		"u_enpot": 'switch to <a href="#">potato UI</a> (may improve upload speed)',
 		"u_depot": 'switch to <a href="#">fancy UI</a> (may reduce upload speed)',
 		"u_gotpot": 'switching to the potato UI for improved upload speed,\n\nfeel free to disagree and switch back!',
@@ -308,9 +312,11 @@ var Ls = {
 		"u_upping": 'uploading',
 		"u_cuerr": "failed to upload chunk {0} of {1};\nprobably harmless, continuing\n\nfile: {2}",
 		"u_cuerr2": "server rejected upload (chunk {0} of {1});\n\nfile: {2}\n\nerror ",
+		"u_ehstmp": "will retry; see bottom-right",
 		"u_ehsfin": "server rejected the request to finalize upload",
 		"u_ehssrch": "server rejected the request to perform search",
 		"u_ehsinit": "server rejected the request to initiate upload",
+		"u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue",
 		"u_s404": "not found on server",
 		"u_expl": "explain",
 		"u_tu": '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
@@ -437,6 +443,7 @@ var Ls = {
 
 		"ct_thumb": "vis miniatyrbilder istedenfor ikoner$NSnarvei: T",
 		"ct_dots": "vis skjulte filer (gitt at serveren tillater det)",
+		"ct_dir1st": "sorter slik at mapper kommer foran filer",
 		"ct_readme": "vis README.md nedenfor filene",
 
 		"cut_turbo": "forenklet befaring ved opplastning; bør sannsynlig <em>ikke</em> skrus på:$N$Nnyttig dersom du var midt i en svær opplastning som måtte restartes av en eller annen grunn, og du vil komme igang igjen så raskt som overhodet mulig.$N$Nnår denne er skrudd på så forenkles befaringen kraftig; istedenfor å utføre en trygg sjekk på om filene finnes på serveren i god stand, så sjekkes kun om <em>filstørrelsen</em> stemmer. Så dersom en korrupt fil skulle befinne seg på serveren allerede, på samme sted med samme størrelse og navn, så blir det <em>ikke oppdaget</em>.$N$Ndet anbefales å kun benytte denne funksjonen for å komme seg raskt igjennom selve opplastningen, for så å skru den av, og til slutt &quot;laste opp&quot; de samme filene én gang til -- slik at integriteten kan verifiseres",
@@ -447,6 +454,8 @@ var Ls = {
 
 		"cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN",
 
+		"cut_mt": "raskere befaring ved å bruke hele CPU'en$N$Ndenne funksjonen anvender web-workers$Nog krever mer RAM (opptil 512 MiB ekstra)$N$N30% raskere https, 4.5x raskere http,$Nog 5.3x raskere på android-telefoner",
+
 		"cft_text": "ikontekst (blank ut og last siden på nytt for å deaktivere)",
 		"cft_fg": "farge",
 		"cft_bg": "bakgrunnsfarge",
@@ -477,7 +486,7 @@ var Ls = {
 		"mt_caac": "konverter aac / m4a-filer til to opus\">aac",
 		"mt_coth": "konverter alt annet (men ikke mp3) til opus\">andre",
 		"mt_tint": "nivå av bakgrunnsfarge på søkestripa (0-100),$Ngjør oppdateringer mindre distraherende",
-		"mt_eq": "aktiver tonekontroll og forsterker;$Nboost 0 = normal volumskala$N$Nreduserer også dødtid imellom sangfiler",
+		"mt_eq": "aktiver tonekontroll og forsterker;$N$Nboost &lt;code&gt;0&lt;/code&gt; = normal volumskala$N$Nwidth &lt;code&gt;1 &nbsp;&lt;/code&gt; = normal stereo$Nwidth &lt;code&gt;0.5&lt;/code&gt; = 50% blanding venstre-høyre$Nwidth &lt;code&gt;0 &nbsp;&lt;/code&gt; = mono$N$Nboost &lt;code&gt;-0.8&lt;/code&gt; &amp; width &lt;code&gt;10&lt;/code&gt; = instrumental :^)$N$Nreduserer også dødtid imellom sangfiler",
 
 		"mb_play": "lytt",
 		"mm_hashplay": "spill denne sangen?",
@@ -617,8 +626,9 @@ var Ls = {
 
 		"u_https1": "du burde",
 		"u_https2": "bytte til https",
-		"u_https3": "for mye høyere hastighet",
+		"u_https3": "for høyere hastighet",
 		"u_ancient": 'nettleseren din er prehistorisk -- mulig du burde <a href="#" onclick="goto(\'bup\')">bruke bup istedenfor</a>',
+		"u_nowork": "krever firefox 53+, chrome 57+, eller iOS 11+",
 		"u_enpot": 'bytt til <a href="#">enkelt UI</a> (gir sannsynlig raskere opplastning)',
 		"u_depot": 'bytt til <a href="#">snæsent UI</a> (gir sannsynlig tregere opplastning)',
 		"u_gotpot": 'byttet til et enklere UI for å laste opp raskere,\n\ndu kan gjerne bytte tilbake altså!',
@@ -639,9 +649,11 @@ var Ls = {
 		"u_upping": 'sender',
 		"u_cuerr": "kunne ikke laste opp del {0} av {1};\nsikkert harmløst, fortsetter\n\nfil: {2}",
 		"u_cuerr2": "server nektet opplastningen (del {0} av {1});\n\nfile: {2}\n\nerror ",
+		"u_ehstmp": "prøver igjen; se mld nederst",
 		"u_ehsfin": "server nektet forespørselen om å ferdigstille filen",
 		"u_ehssrch": "server nektet forespørselen om å utføre søk",
 		"u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning",
+		"u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...",
 		"u_s404": "ikke funnet på serveren",
 		"u_expl": "forklar",
 		"u_tu": '<p class="warn">ADVARSEL: turbo er på, <span>&nbsp;avbrutte opplastninger vil muligens ikke oppdages og gjenopptas; hold musepekeren over turbo-knappen for mer info</span></p>',
@@ -818,6 +830,7 @@ ebi('op_cfg').innerHTML = (
 	'		<a id="griden" class="tgl btn" href="#" tt="' + L.wt_grid + '">田 the grid</a>\n' +
 	'		<a id="thumbs" class="tgl btn" href="#" tt="' + L.ct_thumb + '">🖼️ thumbs</a>\n' +
 	'		<a id="dotfiles" class="tgl btn" href="#" tt="' + L.ct_dots + '">dotfiles</a>\n' +
+	'		<a id="dir1st" class="tgl btn" href="#" tt="' + L.ct_dir1st + '">📁 first</a>\n' +
 	'		<a id="ireadme" class="tgl btn" href="#" tt="' + L.ct_readme + '">📜 readme</a>\n' +
 	'	</div>\n' +
 	'</div>\n' +
@@ -837,6 +850,7 @@ ebi('op_cfg').innerHTML = (
 	'<div>\n' +
 	'	<h3>' + L.cl_uopts + '</h3>\n' +
 	'	<div>\n' +
+	'		<a id="hashw" class="tgl btn" href="#" tt="' + L.cut_mt + '">mt</a>\n' +
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="' + L.cut_turbo + '">turbo</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="' + L.cut_datechk + '">date-chk</a>\n' +
 	'		<a id="flag_en" class="tgl btn" href="#" tt="' + L.cut_flag + '">💤</a>\n' +
@@ -854,7 +868,7 @@ ebi('op_cfg').innerHTML = (
 	'	</div>\n' +
 	'</div>\n' +
 	'<div><h3>' + L.cl_keytype + '</h3><div id="key_notation"></div></div>\n' +
-	'<div class="fill"><h3>' + L.cl_hiddenc + ' <a href="#" id="hcolsr">' + L.cl_reset + '</h3><div id="hcols"></div></div>'
+	'<div><h3>' + L.cl_hiddenc + ' <a href="#" id="hcolsr">' + L.cl_reset + '</h3><div id="hcols"></div></div>'
 );
 
 
@@ -907,7 +921,7 @@ function opclick(e) {
 	goto(dest);
 
 	var input = QS('.opview.act input:not([type="hidden"])')
-	if (input && !is_touch) {
+	if (input && !TOUCH) {
 		tt.skip = true;
 		input.focus();
 	}
@@ -1677,7 +1691,7 @@ var vbar = (function () {
 		if (e.button === 0)
 			can.onmousemove = null;
 	};
-	if (is_touch) {
+	if (TOUCH) {
 		can.ontouchstart = mousedown;
 		can.ontouchmove = mousemove;
 	}
@@ -1782,7 +1796,7 @@ function playpause(e) {
 		seek_au_mul(x * 1.0 / rect.width);
 	};
 
-	if (!is_touch)
+	if (!TOUCH)
 		bar.onwheel = function (e) {
 			var dist = Math.sign(e.deltaY) * 10;
 			if (Math.abs(e.deltaY) < 30 && !e.deltaMode)
@@ -1822,7 +1836,7 @@ var mpui = (function () {
 		if (++nth > 69) {
 			// android-chrome breaks aspect ratio with unannounced viewport changes
 			nth = 0;
-			if (is_touch) {
+			if (MOBILE) {
 				nth = 1;
 				pbar.onresize();
 				vbar.onresize();
@@ -1890,6 +1904,7 @@ var audio_eq = (function () {
 		"gains": [4, 3, 2, 1, 0, 0, 1, 2, 3, 4],
 		"filters": [],
 		"amp": 0,
+		"chw": 1,
 		"last_au": null,
 		"acst": {}
 	};
@@ -1941,6 +1956,7 @@ var audio_eq = (function () {
 
 	try {
 		r.amp = fcfg_get('au_eq_amp', r.amp);
+		r.chw = fcfg_get('au_eq_chw', r.chw);
 		var gains = jread('au_eq_gain', r.gains);
 		if (r.gains.length == gains.length)
 			r.gains = gains;
@@ -1950,12 +1966,14 @@ var audio_eq = (function () {
 	r.draw = function () {
 		jwrite('au_eq_gain', r.gains);
 		swrite('au_eq_amp', r.amp);
+		swrite('au_eq_chw', r.chw);
 
 		var txt = QSA('input.eq_gain');
 		for (var a = 0; a < r.bands.length; a++)
 			txt[a].value = r.gains[a];
 
 		QS('input.eq_gain[band="amp"]').value = r.amp;
+		QS('input.eq_gain[band="chw"]').value = r.chw;
 	};
 
 	r.stop = function () {
@@ -2025,16 +2043,47 @@ var audio_eq = (function () {
 		for (var a = r.filters.length - 1; a >= 0; a--)
 			r.filters[a].connect(a > 0 ? r.filters[a - 1] : actx.destination);
 
+		if (Math.round(r.chw * 25) != 25) {
+			var split = actx.createChannelSplitter(2),
+				merge = actx.createChannelMerger(2),
+				lg1 = actx.createGain(),
+				lg2 = actx.createGain(),
+				rg1 = actx.createGain(),
+				rg2 = actx.createGain(),
+				vg1 = 1 - (1 - r.chw) / 2,
+				vg2 = 1 - vg1;
+
+			console.log('chw', vg1, vg2);
+
+			merge.connect(r.filters[r.filters.length - 1]);
+			lg1.gain.value = rg2.gain.value = vg1;
+			lg2.gain.value = rg1.gain.value = vg2;
+			lg1.connect(merge, 0, 0);
+			rg1.connect(merge, 0, 0);
+			lg2.connect(merge, 0, 1);
+			rg2.connect(merge, 0, 1);
+
+			split.connect(lg1, 0);
+			split.connect(lg2, 0);
+			split.connect(rg1, 1);
+			split.connect(rg2, 1);
+			r.filters.push(split);
+			mp.acs.channelCountMode = 'explicit';
+		}
+
 		mp.acs.connect(r.filters[r.filters.length - 1]);
 	}
 
 	function eq_step(e) {
 		ev(e);
-		var band = parseInt(this.getAttribute('band')),
+		var sb = this.getAttribute('band'),
+			band = parseInt(sb),
 			step = parseFloat(this.getAttribute('step'));
 
-		if (isNaN(band))
+		if (sb == 'amp')
 			r.amp = Math.round((r.amp + step * 0.2) * 100) / 100;
+		else if (sb == 'chw')
+			r.chw = Math.round((r.chw + step * 0.2) * 100) / 100;
 		else
 			r.gains[band] += step;
 
@@ -2044,15 +2093,18 @@ var audio_eq = (function () {
 	function adj_band(that, step) {
 		var err = false;
 		try {
-			var band = parseInt(that.getAttribute('band')),
+			var sb = that.getAttribute('band'),
+				band = parseInt(sb),
 				vs = that.value,
 				v = parseFloat(vs);
 
 			if (isNaN(v) || v + '' != vs)
 				throw new Error('inval band');
 
-			if (isNaN(band))
+			if (sb == 'amp')
 				r.amp = Math.round((v + step * 0.2) * 100) / 100;
+			else if (sb == 'chw')
+				r.chw = Math.round((v + step * 0.2) * 100) / 100;
 			else
 				r.gains[band] = v + step;
 
@@ -2089,6 +2141,7 @@ var audio_eq = (function () {
 		vs.push([a, hz, r.gains[a]]);
 	}
 	vs.push(["amp", "boost", r.amp]);
+	vs.push(["chw", "width", r.chw]);
 
 	for (var a = 0; a < vs.length; a++) {
 		var b = vs[a][0];
@@ -2423,7 +2476,7 @@ function eval_hash() {
 				if (a)
 					QS(treectl.hidden ? '#path a:nth-last-child(2)' : '#treeul a.hl').focus();
 				else
-					QS(thegrid.en ? '#ggrid a' : '#files tbody a').focus();
+					QS(thegrid.en ? '#ggrid a' : '#files tbody tr[tabindex]').focus();
 			};
 		})(a);
 
@@ -2436,7 +2489,8 @@ function sortfiles(nodes) {
 	if (!nodes.length)
 		return nodes;
 
-	var sopts = jread('fsort', [["href", 1, ""]]);
+	var sopts = jread('fsort', [["href", 1, ""]]),
+		dir1st = sread('dir1st') !== '0';
 
 	try {
 		var is_srch = false;
@@ -2467,14 +2521,10 @@ function sortfiles(nodes) {
 
 					if ((v + '').indexOf('<a ') === 0)
 						v = v.split('>')[1];
-					else if (name == "href" && v) {
-						if (v.split('?')[0].slice(-1) == '/')
-							v = '\t' + v;
-
+					else if (name == "href" && v)
 						v = uricom_dec(v)[0];
-					}
 
-					nodes[b]._sv = v;
+					nodes[b]._sv = v
 				}
 			}
 
@@ -2503,6 +2553,13 @@ function sortfiles(nodes) {
 			if (is_srch)
 				delete nodes[b].ext;
 		}
+		if (dir1st) {
+			var r1 = [], r2 = [];
+			for (var b = 0, bb = nodes.length; b < bb; b++)
+				(nodes[b].href.split('?')[0].slice(-1) == '/' ? r1 : r2).push(nodes[b]);
+
+			nodes = r1.concat(r2);
+		}
 	}
 	catch (ex) {
 		console.log("failed to apply sort config: " + ex);
@@ -3955,6 +4012,9 @@ document.onkeydown = function (e) {
 		}
 	}
 
+	if (k == 'Enter' && ae && (ae.onclick || ae.hasAttribute('tabIndex')))
+		return ev(e) && ae.click() || true;
+
 	if (aet && aet != 'a' && aet != 'tr' && aet != 'pre')
 		return;
 
@@ -4154,7 +4214,7 @@ document.onkeydown = function (e) {
 			clearTimeout(defer_timeout);
 			clearTimeout(search_timeout);
 			search_timeout = setTimeout(do_search,
-				v && v.length < (is_touch ? 4 : 3) ? 1000 : 500);
+				v && v.length < (MOBILE ? 4 : 3) ? 1000 : 500);
 		}
 	}
 
@@ -4393,6 +4453,9 @@ var treectl = (function () {
 	bcfg_bind(r, 'dots', 'dotfiles', false, function (v) {
 		r.goto(get_evpath());
 	});
+	bcfg_bind(r, 'dir1st', 'dir1st', true, function (v) {
+		treectl.gentab(get_evpath(), treectl.lsc);
+	});
 	setwrap(bcfg_bind(r, 'wtree', 'wraptree', true, setwrap));
 	setwrap(bcfg_bind(r, 'parpane', 'parpane', true, onscroll));
 	bcfg_bind(r, 'htree', 'hovertree', false, reload_tree);
@@ -4589,9 +4652,9 @@ var treectl = (function () {
 				return ta[a];
 	};
 
-	r.goto = function (url, push) {
+	r.goto = function (url, push, back) {
 		get_tree("", url, true);
-		r.reqls(url, push, true);
+		r.reqls(url, push, true, back);
 	};
 
 	function get_tree(top, dst, rst) {
@@ -4760,9 +4823,10 @@ var treectl = (function () {
 		thegrid.setvis(true);
 	}
 
-	r.reqls = function (url, hpush, no_tree) {
+	r.reqls = function (url, hpush, no_tree, back) {
 		var xhr = new XHR();
 		xhr.top = url;
+		xhr.back = back
 		xhr.hpush = hpush;
 		xhr.ts = Date.now();
 		xhr.open('GET', xhr.top + '?ls' + (r.dots ? '&dots' : ''), true);
@@ -4830,6 +4894,12 @@ var treectl = (function () {
 		if (res.readme)
 			show_readme(res.readme);
 
+		if (this.hpush && !this.back) {
+			var ofs = ebi('wrap').offsetTop;
+			if (document.documentElement.scrollTop > ofs)
+				document.documentElement.scrollTop = ofs;
+		}
+
 		wintitle();
 		var fun = r.ls_cb;
 		if (fun) {
@@ -4839,6 +4909,7 @@ var treectl = (function () {
 	}
 
 	r.gentab = function (top, res) {
+		r.lsc = res;
 		var nodes = res.dirs.concat(res.files),
 			html = mk_files_header(res.taglist),
 			seen = {};
@@ -4851,7 +4922,6 @@ var treectl = (function () {
 				bhref = tn.href.split('?')[0],
 				fname = uricom_dec(bhref)[0],
 				hname = esc(fname),
-				sortv = (bhref.slice(-1) == '/' ? '\t' : '') + hname,
 				id = 'f-' + ('00000000' + crc32(fname)).slice(-8),
 				lang = showfile.getlang(fname);
 
@@ -4866,8 +4936,8 @@ var treectl = (function () {
 				tn.lead = '<a href="?doc=' + tn.href + '" class="doc' + (lang ? ' bri' : '') +
 					'" hl="' + id + '" name="' + hname + '">-txt-</a>';
 
-			var ln = ['<tr><td>' + tn.lead + '</td><td sortv="' + sortv +
-				'"><a href="' + top + tn.href + '" id="' + id + '">' + hname + '</a>', tn.sz];
+			var ln = ['<tr><td>' + tn.lead + '</td><td><a href="' +
+				top + tn.href + '" id="' + id + '">' + hname + '</a>', tn.sz];
 
 			for (var b = 0; b < res.taglist.length; b++) {
 				var k = res.taglist[b],
@@ -5005,7 +5075,7 @@ var treectl = (function () {
 		if (url.search.indexOf('doc=') + 1 && hbase == cbase)
 			return showfile.show(hbase + showfile.sname(url.search), true);
 
-		r.goto(url.pathname);
+		r.goto(url.pathname, false, true);
 	};
 
 	hist_replace(get_evpath() + window.location.hash);

copyparty/web/splash.html

@@ -36,6 +36,7 @@
 				<tr><td>hash-q</td><td>{{ hashq }}</td></tr>
 				<tr><td>tag-q</td><td>{{ tagq }}</td></tr>
 				<tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+				<tr><td>db-act</td><td id="u">{{ dbwt }}</td></tr>
 			</table>
 		</td><td>
 			<table class="vols">
@@ -50,8 +51,8 @@
 			</table>
 		</td></tr></table>
 		<div class="btns">
-			<a id="d" href="/?stack" tt="shows the state of all active threads">dump stack</a>
-			<a id="e" href="/?reload=cfg" tt="reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes">reload cfg</a>
+			<a id="d" href="/?stack">dump stack</a>
+			<a id="e" href="/?reload=cfg">reload cfg</a>
 		</div>
 		{%- endif %}
 

copyparty/web/splash.js

@@ -23,6 +23,12 @@ var Ls = {
 		"r1": "gå hjem",
 		".s1": "kartlegg",
 		"t1": "handling",
+		"u2": "tid siden noen sist skrev til serveren$N( opplastning / navneendring / ... )$N$N17d = 17 dager$N1h23 = 1 time 23 minutter$N4m56 = 4 minuter 56 sekunder",
+	},
+	"eng": {
+		"d2": "shows the state of all active threads",
+		"e2": "reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes",
+		"u2": "time since the last server write$N( upload / rename / ... )$N$N17d = 17 days$N1h23 = 1 hour 23 minutes$N4m56 = 4 minutes 56 seconds",
 	}
 },
 	d = Ls[sread("lang") || lang];
@@ -40,5 +46,10 @@ for (var k in (d || {})) {
 }
 
 tt.init();
-if (!ebi('c'))
-	QS('input[name="cppwd"]').focus();
+var o = QS('input[name="cppwd"]');
+if (!ebi('c') && o.offsetTop + o.offsetHeight < window.innerHeight)
+	o.focus();
+
+o = ebi('u');
+if (o && /[0-9]+$/.exec(o.innerHTML))
+	o.innerHTML = shumantime(o.innerHTML);

copyparty/web/up2k.js

@@ -16,6 +16,7 @@ function goto_up2k() {
 // usually it's undefined but some chromes throw on invoke
 var up2k = null,
     up2k_hooks = [],
+    hws = [],
     sha_js = window.WebAssembly ? 'hw' : 'ac',  // ff53,c57,sa11
     m = 'will use ' + sha_js + ' instead of native sha512 due to';
 
@@ -205,7 +206,7 @@ function U2pvis(act, btns, uc, st) {
         if (!r.is_act(fo.in))
             return;
 
-        var k = 'f{0}{1}'.format(nfile, field.slice(1)),
+        var k = 'f' + nfile + '' + field.slice(1),
             obj = ebi(k);
 
         obj.innerHTML = field == 'ht' ? (markup[html] || html) : html;
@@ -250,9 +251,7 @@ function U2pvis(act, btns, uc, st) {
             nb = fo.bt * (++fo.nh / fo.cb.length),
             p = r.perc(nb, 0, fobj.size, fobj.t_hashing);
 
-        fo.hp = '{0}%, {1}, {2} MB/s'.format(
-            f2f(p[0], 2), p[1], f2f(p[2], 2)
-        );
+        fo.hp = f2f(p[0], 2) + '%, ' + p[1] + ', ' + f2f(p[2], 2) + ' MB/s';
         if (!r.is_act(fo.in))
             return;
 
@@ -269,14 +268,12 @@ function U2pvis(act, btns, uc, st) {
         fo.bd += delta;
 
         var p = r.perc(fo.bd, fo.bd0, fo.bt, fobj.t_uploading);
-        fo.hp = '{0}%, {1}, {2} MB/s'.format(
-            f2f(p[0], 2), p[1], f2f(p[2], 2)
-        );
+        fo.hp = f2f(p[0], 2) + '%, ' + p[1] + ', ' + f2f(p[2], 2) + ' MB/s';
 
         if (!r.is_act(fo.in))
             return;
 
-        var obj = ebi('f{0}p'.format(fobj.n)),
+        var obj = ebi('f' + fobj.n + 'p'),
             o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
 
         if (!obj) {
@@ -446,8 +443,8 @@ function U2pvis(act, btns, uc, st) {
 
         r.npotato = 0;
         var html = [
-            "<p>files: &nbsp; <b>{0}</b> finished, &nbsp; <b>{1}</b> failed, &nbsp; <b>{2}</b> busy, &nbsp; <b>{3}</b> queued</p>".format(r.ctr.ok, r.ctr.ng, r.ctr.bz, r.ctr.q),
-        ];
+            "<p>files: &nbsp; <b>{0}</b> finished, &nbsp; <b>{1}</b> failed, &nbsp; <b>{2}</b> busy, &nbsp; <b>{3}</b> queued</p>".format(
+                r.ctr.ok, r.ctr.ng, r.ctr.bz, r.ctr.q)];
 
         while (r.head < r.tab.length && has(["ok", "ng"], r.tab[r.head].in))
             r.head++;
@@ -457,7 +454,8 @@ function U2pvis(act, btns, uc, st) {
             act = r.tab[r.head];
 
         if (act)
-            html.push("<p>file {0} of {1} : &nbsp; {2} &nbsp; <code>{3}</code></p>\n<div>{4}</div>".format(r.head + 1, r.tab.length, act.ht, act.hp, act.hn));
+            html.push("<p>file {0} of {1} : &nbsp; {2} &nbsp; <code>{3}</code></p>\n<div>{4}</div>".format(
+                r.head + 1, r.tab.length, act.ht, act.hp, act.hn));
 
         html = html.join('\n');
         if (r.hpotato == html)
@@ -470,7 +468,7 @@ function U2pvis(act, btns, uc, st) {
     function apply_html() {
         var oq = {}, n = 0;
         for (var k in r.hq) {
-            var o = ebi('f{0}p'.format(k));
+            var o = ebi('f' + k + 'p');
             if (!o)
                 continue;
 
@@ -682,8 +680,8 @@ function Donut(uc, st) {
         }
 
         if (++r.tc >= 10) {
-            wintitle("{0}%, {1}s, #{2}, ".format(
-                f2f(v * 100 / t, 1), r.eta, st.files.length - st.nfile.upload), true);
+            wintitle("{0}%, {1}, #{2}, ".format(
+                f2f(v * 100 / t, 1), shumantime(r.eta), st.files.length - st.nfile.upload), true);
             r.tc = 0;
         }
 
@@ -721,6 +719,13 @@ function up2k_init(subtle) {
         "gotallfiles": [gotallfiles]  // hooks
     };
 
+    if (window.WebAssembly) {
+        for (var a = 0; a < Math.min(navigator.hardwareConcurrency || 4, 16); a++)
+            hws.push(new Worker('/.cpr/w.hash.js'));
+
+        console.log(hws.length + " hashers ready");
+    }
+
     function showmodal(msg) {
         ebi('u2notbtn').innerHTML = msg;
         ebi('u2btn').style.display = 'none';
@@ -750,7 +755,7 @@ function up2k_init(subtle) {
             showmodal('<h1>loading ' + fn + '</h1>');
             import_js('/.cpr/deps/' + fn, unmodal);
 
-            if (is_https) {
+            if (HTTPS) {
                 // chrome<37 firefox<34 edge<12 opera<24 safari<7
                 m = L.u_ancient;
                 setmsg('');
@@ -793,7 +798,6 @@ function up2k_init(subtle) {
     var parallel_uploads = icfg_get('nthread'),
         uc = {},
         fdom_ctr = 0,
-        min_filebuf = 0,
         biggest_file = 0;
 
     bcfg_bind(uc, 'multitask', 'multitask', true, null, false);
@@ -804,6 +808,7 @@ function up2k_init(subtle) {
     bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo, false);
     bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null, false);
     bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort, false);
+    bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly, set_hashw, false);
 
     var st = {
         "files": [],
@@ -835,7 +840,13 @@ function up2k_init(subtle) {
             "uploading": 0,
             "busy": 0
         },
+        "eta": {
+            "h": "",
+            "u": "",
+            "t": ""
+        },
         "car": 0,
+        "slow_io": null,
         "modn": 0,
         "modv": 0,
         "mod0": null
@@ -919,8 +930,14 @@ function up2k_init(subtle) {
         catch (ex) { }
 
         ev(e);
-        e.dataTransfer.dropEffect = 'copy';
-        e.dataTransfer.effectAllowed = 'copy';
+        try {
+            e.dataTransfer.dropEffect = 'copy';
+            e.dataTransfer.effectAllowed = 'copy';
+        }
+        catch (ex) {
+            document.body.ondragenter = document.body.ondragleave = document.body.ondragover = null;
+            return modal.alert('your browser does not support drag-and-drop uploading');
+        }
         clmod(ebi('drops'), 'vis', 1);
         var v = this.getAttribute('v');
         if (v)
@@ -1278,12 +1295,26 @@ function up2k_init(subtle) {
             ebi('u2tabw').style.minHeight = utw_minh + 'px';
         }
 
-        if (!nhash)
-            ebi('u2etah').innerHTML = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr.ok + pvis.ctr.ng);
+        if (!nhash) {
+            var h = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr.ok + pvis.ctr.ng);
+            if (st.eta.h !== h) {
+                st.eta.h = ebi('u2etah').innerHTML = h;
+                console.log('{0} hash, {1} up, {2} busy'.format(
+                    f2f(st.time.hashing, 1),
+                    f2f(st.time.uploading, 1),
+                    f2f(st.time.busy, 1)));
+            }
+        }
 
-        if (!nsend && !nhash)
-            ebi('u2etau').innerHTML = ebi('u2etat').innerHTML = (
-                L.u_etadone.format(humansize(st.bytes.uploaded), pvis.ctr.ok + pvis.ctr.ng));
+        if (!nsend && !nhash) {
+            var h = L.u_etadone.format(humansize(st.bytes.uploaded), pvis.ctr.ok + pvis.ctr.ng);
+
+            if (st.eta.u !== h)
+                st.eta.u = ebi('u2etau').innerHTML = h;
+
+            if (st.eta.t !== h)
+                st.eta.t = ebi('u2etat').innerHTML = h;
+        }
 
         if (!st.busy.hash.length && !hashing_permitted())
             nhash = 0;
@@ -1314,19 +1345,21 @@ function up2k_init(subtle) {
         for (var a = 0; a < t.length; a++) {
             var rem = st.bytes.total - t[a][2],
                 bps = t[a][1] / t[a][3],
+                hid = t[a][0],
+                eid = hid.slice(-1),
                 eta = Math.floor(rem / bps);
 
             if (t[a][1] < 1024 || t[a][3] < 0.1) {
-                ebi(t[a][0]).innerHTML = L.u_etaprep;
+                ebi(hid).innerHTML = L.u_etaprep;
                 continue;
             }
 
             donut.eta = eta;
-            if (etaskip)
-                continue;
-
-            ebi(t[a][0]).innerHTML = '{0}, {1}/s, {2}'.format(
+            st.eta[eid] = '{0}, {1}/s, {2}'.format(
                 humansize(rem), humansize(bps, 1), humantime(eta));
+
+            if (!etaskip)
+                ebi(hid).innerHTML = st.eta[eid];
         }
         if (++etaskip > 2)
             etaskip = 0;
@@ -1356,6 +1389,10 @@ function up2k_init(subtle) {
             st.busy.handshake.length)
             return false;
 
+        if (t.n - st.car > 8)
+            // prevent runahead from a stuck upload (slow server hdd)
+            return false;
+
         if ((uc.multitask ? 1 : 0) <
             st.todo.upload.length +
             st.busy.upload.length)
@@ -1642,6 +1679,7 @@ function up2k_init(subtle) {
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
         st.nfile.hash = t.n;
+        t.t_hashing = Date.now();
 
         var bpend = 0,
             nchunk = 0,
@@ -1652,30 +1690,23 @@ function up2k_init(subtle) {
         pvis.setab(t.n, nchunks);
         pvis.move(t.n, 'bz');
 
+        if (nchunks > 1 && hws.length && uc.hashw)
+            return wexec_hash(t, chunksize, nchunks);
+
         var segm_next = function () {
-            if (nchunk >= nchunks || (bpend > chunksize && bpend >= min_filebuf))
+            if (nchunk >= nchunks || bpend)
                 return false;
 
             var reader = new FileReader(),
                 nch = nchunk++,
                 car = nch * chunksize,
-                cdr = car + chunksize,
-                t0 = Date.now();
+                cdr = Math.min(chunksize + car, t.size);
 
-            if (cdr >= t.size)
-                cdr = t.size;
-
-            bpend += cdr - car;
             st.bytes.hashed += cdr - car;
 
             function orz(e) {
-                if (!min_filebuf && nch == 1) {
-                    min_filebuf = 1;
-                    var td = Date.now() - t0;
-                    if (td > 50) {
-                        min_filebuf = 32 * 1024 * 1024;
-                    }
-                }
+                bpend--;
+                segm_next();
                 hash_calc(nch, e.target.result);
             }
             reader.onload = function (e) {
@@ -1703,6 +1734,7 @@ function up2k_init(subtle) {
 
                 toast.err(0, 'y o u   b r o k e    i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err);
             };
+            bpend++;
             reader.readAsArrayBuffer(
                 bobslice.call(t.fobj, car, cdr));
 
@@ -1710,8 +1742,6 @@ function up2k_init(subtle) {
         };
 
         var hash_calc = function (nch, buf) {
-            while (segm_next());
-
             var orz = function (hashbuf) {
                 var hslice = new Uint8Array(hashbuf).subarray(0, 33),
                     b64str = buf2b64(hslice);
@@ -1719,15 +1749,12 @@ function up2k_init(subtle) {
                 hashtab[nch] = b64str;
                 t.hash.push(nch);
                 pvis.hashed(t);
-
-                bpend -= buf.byteLength;
-                if (t.hash.length < nchunks) {
+                if (t.hash.length < nchunks)
                     return segm_next();
-                }
+
                 t.hash = [];
-                for (var a = 0; a < nchunks; a++) {
+                for (var a = 0; a < nchunks; a++)
                     t.hash.push(hashtab[a]);
-                }
 
                 t.t_hashed = Date.now();
 
@@ -1759,11 +1786,117 @@ function up2k_init(subtle) {
                 }
             }, 1);
         };
-
-        t.t_hashing = Date.now();
         segm_next();
     }
 
+    function wexec_hash(t, chunksize, nchunks) {
+        var nchunk = 0,
+            reading = 0,
+            max_readers = 1,
+            opt_readers = 2,
+            free = [],
+            busy = {},
+            nbusy = 0,
+            hashtab = {},
+            mem = (MOBILE ? 128 : 256) * 1024 * 1024;
+
+        for (var a = 0; a < hws.length; a++) {
+            var w = hws[a];
+            free.push(w);
+            w.onmessage = onmsg;
+            mem -= chunksize;
+            if (mem <= 0)
+                break;
+        }
+
+        function go_next() {
+            if (st.slow_io && uc.multitask)
+                // android-chrome filereader latency is ridiculous but scales linearly
+                // (unlike every other platform which instead suffers on parallel reads...)
+                max_readers = opt_readers = free.length;
+
+            if (reading >= max_readers || !free.length || nchunk >= nchunks)
+                return;
+
+            var w = free.pop(),
+                car = nchunk * chunksize,
+                cdr = Math.min(chunksize + car, t.size);
+
+            //console.log('[P ] %d read bgin (%d reading, %d busy)', nchunk, reading + 1, nbusy + 1);
+            w.postMessage([nchunk, t.fobj, car, cdr]);
+            busy[nchunk] = w;
+            nbusy++;
+            reading++;
+            nchunk++;
+        }
+
+        function onmsg(d) {
+            d = d.data;
+            var k = d[0];
+
+            if (k == "panic")
+                return vis_exh(d[1], 'up2k.js', '', '', d[1]);
+
+            if (k == "fail") {
+                pvis.seth(t.n, 1, d[1]);
+                pvis.seth(t.n, 2, d[2]);
+                console.log(d[1], d[2]);
+
+                pvis.move(t.n, 'ng');
+                apop(st.busy.hash, t);
+                st.bytes.finished += t.size;
+                return;
+            }
+
+            if (k == "ferr")
+                return toast.err(0, 'y o u   b r o k e    i t\nfile: ' + esc(t.name + '') + '\nerror: ' + d[1]);
+
+            if (k == "read") {
+                reading--;
+                if (MOBILE && CHROME && st.slow_io === null && d[1] == 1 && d[2] > 1024 * 512) {
+                    var spd = Math.floor(d[2] / d[3]);
+                    st.slow_io = spd < 40 * 1024;
+                    console.log('spd {0}, slow: {1}'.format(spd, st.slow_io));
+                }
+                //console.log('[P ] %d read DONE (%d reading, %d busy)', d[1], reading, nbusy);
+                return go_next();
+            }
+
+            if (k == "done") {
+                var nchunk = d[1],
+                    hslice = d[2],
+                    sz = d[3];
+
+                free.push(busy[nchunk]);
+                delete busy[nchunk];
+                nbusy--;
+
+                //console.log('[P ] %d HASH DONE (%d reading, %d busy)', nchunk, reading, nbusy);
+
+                hashtab[nchunk] = buf2b64(hslice);
+                st.bytes.hashed += sz;
+                t.hash.push(nchunk);
+                pvis.hashed(t);
+
+                if (t.hash.length < nchunks)
+                    return nbusy < opt_readers && go_next();
+
+                t.hash = [];
+                for (var a = 0; a < nchunks; a++)
+                    t.hash.push(hashtab[a]);
+
+                t.t_hashed = Date.now();
+
+                pvis.seth(t.n, 2, L.u_hashdone);
+                pvis.seth(t.n, 1, '📦 wait');
+                apop(st.busy.hash, t);
+                st.todo.handshake.push(t);
+                tasker();
+            }
+        }
+        go_next();
+    }
+
     /////
     ////
     ///   head
@@ -1977,6 +2110,9 @@ function up2k_init(subtle) {
                 tasker();
             }
             else {
+                pvis.seth(t.n, 1, "ERROR");
+                pvis.seth(t.n, 2, L.u_ehstmp);
+
                 var err = "",
                     rsp = (xhr.responseText + ''),
                     ofs = rsp.lastIndexOf('\nURL: ');
@@ -2011,6 +2147,9 @@ function up2k_init(subtle) {
                         t.want_recheck = true;
                     }
                 }
+                if (rsp.indexOf('server HDD is full') + 1)
+                    return toast.err(0, L.u_ehsdf + "\n\n" + rsp.replace(/.*; /, ''));
+
                 if (err != "") {
                     pvis.seth(t.n, 1, "ERROR");
                     pvis.seth(t.n, 2, err);
@@ -2135,8 +2274,9 @@ function up2k_init(subtle) {
             xhr.open('POST', t.purl, true);
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
-            xhr.setRequestHeader("X-Up2k-Stat", "{0}/{1}/{2}/{3} {4}/{5}".format(
-                pvis.ctr.ok, pvis.ctr.ng, pvis.ctr.bz, pvis.ctr.q, btot, btot - bfin));
+            xhr.setRequestHeader("X-Up2k-Stat", "{0}/{1}/{2}/{3} {4}/{5} {6}".format(
+                pvis.ctr.ok, pvis.ctr.ng, pvis.ctr.bz, pvis.ctr.q, btot, btot - bfin,
+                st.eta.t.split(' ').pop()));
             xhr.setRequestHeader('Content-Type', 'application/octet-stream');
             if (xhr.overrideMimeType)
                 xhr.overrideMimeType('Content-Type', 'application/octet-stream');
@@ -2182,7 +2322,7 @@ function up2k_init(subtle) {
     window.addEventListener('resize', onresize);
     onresize();
 
-    if (is_touch) {
+    if (MOBILE) {
         // android-chrome wobbles for a bit; firefox / iOS-safari are OK
         setTimeout(onresize, 20);
         setTimeout(onresize, 100);
@@ -2336,6 +2476,13 @@ function up2k_init(subtle) {
         localStorage.removeItem('u2sort');
     }
 
+    function set_hashw() {
+        if (!window.WebAssembly) {
+            bcfg_set('hashw', false);
+            toast.err(10, L.u_nowork);
+        }
+    }
+
     ebi('nthread_add').onclick = function (e) {
         ev(e);
         bumpthread(1);

copyparty/web/util.js

@@ -7,12 +7,28 @@ if (!window['console'])
 
 
 var wah = '',
-    is_touch = 'ontouchstart' in window,
-    is_https = (window.location + '').indexOf('https:') === 0,
-    IPHONE = is_touch && /iPhone|iPad|iPod/i.test(navigator.userAgent),
+    HALFMAX = 8192 * 8192 * 8192 * 8192,
+    HTTPS = (window.location + '').indexOf('https:') === 0,
+    TOUCH = 'ontouchstart' in window,
+    MOBILE = TOUCH,
+    CHROME = !!window.chrome,
+    IPHONE = TOUCH && /iPhone|iPad|iPod/i.test(navigator.userAgent),
     WINDOWS = navigator.platform ? navigator.platform == 'Win32' : /Windows/.test(navigator.userAgent);
 
 
+try {
+    if (navigator.userAgentData.mobile)
+        MOBILE = true;
+
+    if (navigator.userAgentData.platform == 'Windows')
+        WINDOWS = true;
+
+    if (navigator.userAgentData.brands.some(function (d) { return d.brand == 'Chromium' }))
+        CHROME = true;
+}
+catch (ex) { }
+
+
 var ebi = document.getElementById.bind(document),
     QS = document.querySelector.bind(document),
     QSA = document.querySelectorAll.bind(document),
@@ -459,6 +475,16 @@ function sortTable(table, col, cb) {
         }
         return reverse * (a.localeCompare(b));
     });
+    if (sread('dir1st') !== '0') {
+        var r1 = [], r2 = [];
+        for (var i = 0; i < tr.length; i++) {
+            var cell = tr[vl[i][1]].cells[1],
+                href = cell.getAttribute('sortv') || cell.textContent.trim();
+
+            (href.split('?')[0].slice(-1) == '/' ? r1 : r2).push(vl[i]);
+        }
+        vl = r1.concat(r2);
+    }
     for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
     if (cb) cb();
 }
@@ -642,7 +668,7 @@ function humansize(b, terse) {
 
 function humantime(v) {
     if (v >= 60 * 60 * 24)
-        return v;
+        return shumantime(v);
 
     try {
         return /.*(..:..:..).*/.exec(new Date(v * 1000).toUTCString())[1];
@@ -653,12 +679,39 @@ function humantime(v) {
 }
 
 
+function shumantime(v) {
+    if (v < 10)
+        return f2f(v, 2) + 's';
+    if (v < 60)
+        return f2f(v, 1) + 's';
+
+    v = parseInt(v);
+    var st = [[60 * 60 * 24, 60 * 60, 'd'], [60 * 60, 60, 'h'], [60, 1, 'm']];
+
+    for (var a = 0; a < st.length; a++) {
+        var m1 = st[a][0],
+            m2 = st[a][1],
+            ch = st[a][2];
+
+        if (v < m1)
+            continue;
+
+        var v1 = parseInt(v / m1),
+            v2 = ('0' + parseInt((v % m1) / m2)).slice(-2);
+
+        return v1 + ch + (v1 >= 10 ? '' : v2);
+    }
+}
+
+
 function clamp(v, a, b) {
     return Math.min(Math.max(v, a), b);
 }
 
 
 function has(haystack, needle) {
+    try { return haystack.includes(needle); } catch (ex) { }
+
     for (var a = 0; a < haystack.length; a++)
         if (haystack[a] == needle)
             return true;
@@ -908,7 +961,7 @@ var tt = (function () {
             return r.show.bind(this)();
 
         tev = setTimeout(r.show.bind(this), 800);
-        if (is_touch)
+        if (TOUCH)
             return;
 
         this.addEventListener('mousemove', r.move);
@@ -1495,13 +1548,13 @@ function xhrchk(xhr, prefix, e404) {
     var errtxt = (xhr.response && xhr.response.err) || xhr.responseText,
         fun = toast.err;
 
-    if (xhr.status == 503 && /\bDD(?:wah){0}[o]S [Pp]rote[c]tion|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser/.test(errtxt)) {
+    if (xhr.status == 503 && /[Cc]loud[f]lare|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser/.test(errtxt)) {
         var now = Date.now(), td = now - cf_cha_t;
         if (td < 15000)
             return;
 
         cf_cha_t = now;
-        errtxt = 'Cloudflare DD' + wah + 'oS protection kicked in\n\n<strong>trying to fix it...</strong>';
+        errtxt = 'Clou' + wah + 'dflare protection kicked in\n\n<strong>trying to fix it...</strong>';
         fun = toast.warn;
 
         qsr('#cf_frame');

copyparty/web/w.hash.js

@@ -0,0 +1,77 @@
+"use strict";
+
+
+function hex2u8(txt) {
+    return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); }));
+}
+
+
+var subtle = null;
+try {
+    subtle = crypto.subtle || crypto.webkitSubtle;
+    subtle.digest('SHA-512', new Uint8Array(1)).then(
+        function (x) { },
+        function (x) { load_fb(); }
+    );
+}
+catch (ex) {
+    load_fb();
+}
+function load_fb() {
+    subtle = null;
+    importScripts('/.cpr/deps/sha512.hw.js');
+}
+
+
+onmessage = (d) => {
+    var [nchunk, fobj, car, cdr] = d.data,
+        t0 = Date.now(),
+        reader = new FileReader();
+
+    reader.onload = function (e) {
+        try {
+            //console.log('[ w] %d HASH bgin', nchunk);
+            postMessage(["read", nchunk, cdr - car, Date.now() - t0]);
+            hash_calc(e.target.result);
+        }
+        catch (ex) {
+            postMessage(["panic", ex + '']);
+        }
+    };
+    reader.onerror = function () {
+        var err = reader.error + '';
+
+        if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender
+            err.indexOf('NotFoundError') !== -1  // macos-firefox permissions
+        )
+            return postMessage(["fail", 'OS-error', err + ' @ ' + car]);
+
+        postMessage(["ferr", err]);
+    };
+    //console.log('[ w] %d read bgin', nchunk);
+    reader.readAsArrayBuffer(
+        File.prototype.slice.call(fobj, car, cdr));
+
+
+    var hash_calc = function (buf) {
+        var hash_done = function (hashbuf) {
+            try {
+                var hslice = new Uint8Array(hashbuf).subarray(0, 33);
+                //console.log('[ w] %d HASH DONE', nchunk);
+                postMessage(["done", nchunk, hslice, cdr - car]);
+            }
+            catch (ex) {
+                postMessage(["panic", ex + '']);
+            }
+        };
+
+        if (subtle)
+            subtle.digest('SHA-512', buf).then(hash_done);
+        else {
+            var u8buf = new Uint8Array(buf);
+            hashwasm.sha512(u8buf).then(function (v) {
+                hash_done(hex2u8(v))
+            });
+        }
+    };
+}

docs/changelog.md

@@ -1,3 +1,194 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0810-2135  `v1.3.11`  webworkers
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* multithreaded file hashing! **300%** average speed increase
+  * when uploading files through the browser client, based on web-workers
+    * `4.5x` faster on http from a laptop -- `146` -> `670` MiB/s
+    * ` 30%` faster on https from a laptop -- `552` -> `716` MiB/s
+    * `4.2x` faster on http from android -- `13.5` -> `57.1` MiB/s
+    * `5.3x` faster on https from android -- `13.8` -> `73.3` MiB/s
+    * can be disabled using the `mt` togglebtn in the settings pane, for example if your phone runs out of memory (it eats ~250 MiB extra RAM)
+  * `2.3x` faster [u2cli](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy) (cmd-line client) -- `398` -> `930` MiB/s
+  * `2.4x` faster filesystem indexing on the server
+  * thx to @kipukun for the webworker suggestion!
+
+## bugfixes
+* ux: reset scroll when navigating into a new folder
+* u2cli: better errormsg if the server's tls certificate got rejected
+* js: more futureproof cloudflare-challenge detection (they got a new one recently)
+
+## other changes
+* print warning if the python interpreter was built with an unsafe sqlite
+* u2cli: add helpful messages on how to make it run on python 2.6
+
+**trivia:** due to a [chrome bug](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210), http can sometimes be faster than https now ¯\\\_(ツ)\_/¯
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0803-2340  `v1.3.10`  folders first
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* faster
+  * tag scanner
+  * on windows: uploading to fat32 or smb
+* toggle-button to sort folders before files (default-on)
+  * almost the same as before, but now also when sorting by size / date
+* repeatedly hit `ctrl-c` to force-quit if everything dies
+* new file-indexing guards
+  * `--xdev` / volflag `:c,xdev` stops if it hits another filesystem (bindmount/symlink)
+  * `--xvol` / volflag `:c,xvol` does not follow symlinks pointing outside the volume
+  * only affects file indexing -- does NOT prevent access!
+
+## bugfixes
+* forget uploads that failed to initialize (allows retry in another folder)
+* wrong filekeys in upload response if volume path contained a symlink
+* faster shutdown on `ctrl-c` while hashing huge files
+* ux: fix navpane covering files on horizontal scroll
+
+## other changes
+* include version info in the base64 crash-message
+* ux: make upload errors more visible on mobile
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0727-1407  `v1.3.8`  more async
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* new arg `--df 4` and volflag `:c,df=4g` to guarantee 4 GiB free disk space by rejecting uploads
+* some features no longer block new uploads while they're processing
+  * `-e2v` file integrity checker
+  * `-e2ts` initial tag scanner
+  * hopefully fixes a [deadlock](https://www.youtube.com/watch?v=DkKoMveT_jo&t=3s) someone ran into (but probably doesn't)
+    * (the "deadlock" link is an addictive demoscene banger -- the actual issue is #10)
+* reduced the impact of some features which still do
+  * defer `--re-maxage` reindexing if there was a write (upload/rename/...) recently
+    * `--db-act` sets minimum idle period before reindex can start (default 10sec)
+* bbox / image-viewer: add video hotkeys 0..9 to seek 0%..90%
+* audio-player: add audio crossfeed (left-right channel mixer / vocal isolation)
+* splashpage (`/?h`) shows time since the most recent write
+
+## bugfixes
+* a11y:
+  * enter-key should always trigger onclick
+  * only focus password box if in-bounds
+  * improve skip-to-files
+* prisonparty: volume labeling in root folders
+* other minor stuff
+  * forget deleted shadowed files from the db
+  * be less noisy if a client disconnects mid-reply
+  * up2k.js less eager to thrash slow server HDDs
+
+## other changes
+* show client's upload ETA in server log
+* dump stacks and issue `lsof` on the db if a transaction is stuck
+  * will hopefully help if there's any more deadlocks
+* [up2k-hook-ytid](https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/up2k-hook-ytid.js) (the overengineered up2k.js plugin example) now has an mp4/webm/mkv metadata parser
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0716-1848  `v1.3.7`  faster
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* `up2k.js`: **improved upload speeds!**
+  * **...when there's many small files** (or the browser is slow)
+    * add [potato mode](https://user-images.githubusercontent.com/241032/179336639-8ecc01ea-2662-4cb6-8048-5be3ad599f33.png) -- lightweight UI for faster uploads from slow boxes
+    * enables automatically if it detects a cpu bottleneck (not very accurate)
+  * **...on really fast connections (LAN / fiber)**
+    * batch progress updates to reduce repaints
+  * **...when there is a mix of big and small files**
+    * sort the uploads by size, smallest first, for optimal cpu/network usage
+      * can be overridden to alphabetical order in the settings tab
+      * new arg `--u2sort` changes the default + overrides the override button
+    * improve upload pacing when alphabetical order is enabled
+      * mainly affecting single files that are 300 GiB + 
+* `up2k.js`: add [up2k hooks](https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/up2k-hooks.js)
+  * specify *client-side* rules to reject files as they are dropped into the browser
+  * not a hard-reject since people can use [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) and whatnot, more like a hint
+* `up2k.py`: add file integrity checker
+  * new arg `-e2v` to scan volumes and verify file checksums on startup
+  * `-e2vu` updates the db on mismatch, `-e2vp` panics
+  * uploads are blocked while the scan is running -- might get fixed at some point
+    * for now it prints a warning
+* bbox / image-viewer: doubletap a picture to enter fullscreen mode
+* md-editor: `ctrl-c/x` affects current line if no selection, and `ctrl-e` is fullscreen
+* tag-parser plugins:
+  * add support for passing metadata from one mtp to another (parser dependencies)
+    * the `p` flag in [vidchk](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/vidchk.py) usage makes it run after the base parser, eating its output
+  * add [rclone uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/rclone-upload.py) which optionally and by default depends on vidchk
+
+## bugfixes
+* sfx would crash if it got the same PID as recently (for example across two reboots)
+* audio equalizer on recent chromes
+  * still can't figure out why chrome sometimes drops the mediasession
+* bbox: don't attach click events to videos
+* up2k.py:
+  * more sensible behavior w/ blank files
+  * avoid some extra directory scans when deleting files
+  * faster shutdown on `ctrl-c` during volume indexing
+* warning from the thumbnail cleaner if the volume has no thumbnails
+* `>fixing py2 support` `>2022`
+
+## other changes
+* up2k.js:
+  * sends a summary of the upload queue to [the server log](https://github.com/9001/copyparty#up2k)
+  * shows a toast while loading huge filedrops to indicate it's still alive
+* sfx: disable guru meditation unless running on windows
+  * avoids hanging systemd on certain crashes
+* logs the state of all threads if sqlite hits a timeout
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0706-0029  `v1.3.5`  sup cloudflare
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* detect + recover from cloudflare ddos-protection memes during upload
+  * while carefully avoiding any mention of "DDoS" in the JS because enterprise firewalls do not enjoy that
+* new option `--favico` to specify a default favicon
+  * set to `🎉` by default, which also enables the fancy upload progress donut 👌
+* baguettebox (image/video viewer):
+  * toolbar button `⛶` to enter fullscreen mode (same as hotkey `F`)
+  * tap middle of screen to show/hide toolbar
+  * tap left/right-side of pics to navigate prev/next
+  * hotkeys `[` and `]` to set A-B loop in videos
+    * and [URL parameters](https://a.ocv.me/pub/demo/pics-vids/#gf-e2e482ae&t=4.2-6) for that + [initial seekpoint](https://a.ocv.me/pub/demo/pics-vids/#gf-c04bb0f6&t=26s) (same as the audio player)
+
+## bugfixes
+* when a tag-parser hits the timeout, `pkill` all its descendants too
+  * and a [new mtp flag](https://github.com/9001/copyparty/#file-parser-plugins) to override that; `kt` (kill tree, default), `km` (kill main, old default), `kn` (kill none)
+* cpu-wasting spin while waiting for the final handful of files to finish tag-scraping
+* detection of sparse-files support inside [prisonparty](https://github.com/9001/copyparty/tree/hovudstraum/bin#prisonpartysh) and other strict jails
+* baguettebox (image/video viewer):
+  * crash on swipe during close
+* didn't reset terminal color at the end of `?ls=v`
+* don't try to thumbnail empty files (harmless but dumb)
+
+## other changes
+* ux improvements
+  * hide the uploads table until something happens
+* bump codemirror to 5.65.6
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2022-0627-2057  `v1.3.3`  sdcardfs
 

docs/notes.md

@@ -0,0 +1,10 @@
+# up2k.js
+
+## potato detection
+
+* tsk 0.25/8.4/31.5 bzw 1.27/22.9/18 = 77% (38.4s, 49.7s)
+  * 4c locale #1313, ff-102,deb-11 @ ryzen4500u wifi -> win10
+  * profiling shows 2sec heavy gc every 2sec
+
+* tsk 0.41/4.1/10 bzw 1.41/9.9/7 = 73% (13.3s, 18.2s)
+  * 4c locale #1313, ch-103,deb-11 @ ryzen4500u wifi -> win10

docs/notes.sh

@@ -185,7 +185,7 @@ brew install python@2
 pip install virtualenv
 
 # readme toc
-cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
+cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:;,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
 
 # fix firefox phantom breakpoints,
 # suggestions from bugtracker, doesnt work (debugger is not attachable)

scripts/deps-docker/Dockerfile

@@ -2,9 +2,9 @@ FROM    alpine:3.16
 WORKDIR /z
 ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
         ver_hashwasm=4.9.0 \
-        ver_marked=4.0.17 \
+        ver_marked=4.0.18 \
         ver_mde=2.16.1 \
-        ver_codemirror=5.65.6 \
+        ver_codemirror=5.65.7 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 

scripts/make-pypi-release.sh

@@ -14,10 +14,6 @@ gtar=$(command -v gtar || command -v gnutar) || true
 		realpath() { grealpath "$@"; }
 }
 
-which md5sum 2>/dev/null >/dev/null &&
-	md5sum=md5sum ||
-	md5sum="md5 -r"
-
 mode="$1"
 
 [ -z "$mode" ] &&

scripts/make-sfx.sh

@@ -26,6 +26,11 @@ help() { exec cat <<'EOF'
 #   (browsers will try to use 'Consolas' instead)
 #
 # `no-dd` saves ~2k by removing the mouse cursor
+#
+# ---------------------------------------------------------------------
+#
+# if you are on windows, you can use msys2:
+#   PATH=/c/Users/$USER/AppData/Local/Programs/Python/Python310:"$PATH" ./make-sfx.sh fast
 
 EOF
 }
@@ -190,7 +195,7 @@ tmpdir="$(
 	done
 
 	# remove type hints before build instead
-	(cd copyparty; python3 ../../scripts/strip_hints/a.py; rm uh)
+	(cd copyparty; "$pybin" ../../scripts/strip_hints/a.py; rm uh)
 }
 
 ver=
@@ -397,8 +402,8 @@ sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
 sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
 
 for n in {1..50}; do
-	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | shuf) >list || true
-	s=$(md5sum list | cut -c-16)
+	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | (shuf||gshuf) ) >list || true
+	s=$( (sha1sum||shasum) < list | cut -c-16)
 	grep -q $s "$zdir/h" && continue
 	echo $s >> "$zdir/h"
 	break

scripts/profile.py

@@ -8,7 +8,7 @@ cmd = sys.argv[1]
 if cmd == "cpp":
     from copyparty.__main__ import main
 
-    argv = ["__main__", "-v", "srv::r", "-v", "../../yt:yt:r"]
+    argv = ["__main__", "-vsrv::r:c,e2ds,e2ts"]
     main(argv=argv)
 
 elif cmd == "test":
@@ -29,6 +29,6 @@ else:
 #
 # python -m vmprof -o prof --lines ./scripts/profile.py test
 
-# linux: ~/.local/bin/vmprofshow prof tree | grep -vF '[1m  0.'
-# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+# linux: ~/.local/bin/vmprofshow prof tree | awk '$2>1{n=5} !n{next} 1;{n--} !n{print""}'
+# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree
 #   win: %appdata%\..\Roaming\Python\Python39\Scripts\vmprofshow.exe prof tree

scripts/py2/queue/__init__.py

@@ -0,0 +1,4 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+from Queue import Queue, LifoQueue, PriorityQueue, Empty, Full

scripts/sfx.ls

@@ -77,3 +77,4 @@ copyparty/web/splash.js,
 copyparty/web/ui.css,
 copyparty/web/up2k.js,
 copyparty/web/util.js,
+copyparty/web/w.hash.js,

scripts/sfx.py

@@ -213,11 +213,11 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    h = hashlib.md5()
+    h = hashlib.sha1()
     for block in yieldfile(fn):
         h.update(block)
 
-    return h.hexdigest()
+    return h.hexdigest()[:24]
 
 
 def unpack():

tests/test_httpcli.py

@@ -10,9 +10,10 @@ import pprint
 import tarfile
 import tempfile
 import unittest
-from argparse import Namespace
 
 from tests import util as tu
+from tests.util import Cfg
+
 from copyparty.authsrv import AuthSrv
 from copyparty.httpcli import HttpCli
 
@@ -22,39 +23,6 @@ def hdr(query):
     return h.format(query).encode("utf-8")
 
 
-class Cfg(Namespace):
-    def __init__(self, a=None, v=None, c=None):
-        ka = {}
-
-        ex = "e2d e2ds e2dsa e2t e2ts e2tsr ed emp force_js ihead no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nw"
-        ka.update(**{k: False for k in ex.split()})
-
-        ex = "nih no_rescan no_sendfile no_voldump"
-        ka.update(**{k: True for k in ex.split()})
-
-        ex = "css_browser hist js_browser no_hash no_idx"
-        ka.update(**{k: None for k in ex.split()})
-
-        ex = "re_maxage rproxy rsp_slp s_wr_slp theme themes turbo"
-        ka.update(**{k: 0 for k in ex.split()})
-
-        ex = "doctitle favico html_head mth textfiles"
-        ka.update(**{k: "" for k in ex.split()})
-
-        super(Cfg, self).__init__(
-            a=a or [],
-            v=v or [],
-            c=c,
-            s_wr_sz=512 * 1024,
-            unpost=600,
-            mtp=[],
-            mte="a",
-            lang="eng",
-            logout=573,
-            **ka
-        )
-
-
 class TestHttpCli(unittest.TestCase):
     def setUp(self):
         self.td = tu.get_ramdisk()

tests/test_vfs.py

@@ -8,44 +8,14 @@ import shutil
 import tempfile
 import unittest
 from textwrap import dedent
-from argparse import Namespace
 
 from tests import util as tu
+from tests.util import Cfg
+
 from copyparty.authsrv import AuthSrv, VFS
 from copyparty import util
 
 
-class Cfg(Namespace):
-    def __init__(self, a=None, v=None, c=None):
-        ex = "nw e2d e2ds e2dsa e2t e2ts e2tsr no_logues no_readme no_acode force_js no_robots no_thumb no_athumb no_vthumb"
-        ex = {k: False for k in ex.split()}
-        ex2 = {
-            "mtp": [],
-            "mte": "a",
-            "mth": "",
-            "doctitle": "",
-            "html_head": "",
-            "hist": None,
-            "no_idx": None,
-            "no_hash": None,
-            "js_browser": None,
-            "css_browser": None,
-            "no_voldump": True,
-            "re_maxage": 0,
-            "rproxy": 0,
-            "rsp_slp": 0,
-            "s_wr_slp": 0,
-            "s_wr_sz": 512 * 1024,
-            "lang": "eng",
-            "theme": 0,
-            "themes": 0,
-            "turbo": 0,
-            "logout": 573,
-        }
-        ex.update(ex2)
-        super(Cfg, self).__init__(a=a or [], v=v or [], c=c, **ex)
-
-
 class TestVFS(unittest.TestCase):
     def setUp(self):
         self.td = tu.get_ramdisk()

tests/util.py

@@ -7,6 +7,7 @@ import threading
 import tempfile
 import platform
 import subprocess as sp
+from argparse import Namespace
 
 
 WINDOWS = platform.system() == "Windows"
@@ -89,6 +90,40 @@ def get_ramdisk():
         return subdir(ret)
 
 
+class Cfg(Namespace):
+    def __init__(self, a=None, v=None, c=None):
+        ka = {}
+
+        ex = "e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp xdev xvol ed emp force_js ihead no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nid nih nw"
+        ka.update(**{k: False for k in ex.split()})
+
+        ex = "no_rescan no_sendfile no_voldump"
+        ka.update(**{k: True for k in ex.split()})
+
+        ex = "css_browser hist js_browser no_hash no_idx"
+        ka.update(**{k: None for k in ex.split()})
+
+        ex = "re_maxage rproxy rsp_slp s_wr_slp theme themes turbo df"
+        ka.update(**{k: 0 for k in ex.split()})
+
+        ex = "doctitle favico html_head mth textfiles log_fk"
+        ka.update(**{k: "" for k in ex.split()})
+
+        super(Cfg, self).__init__(
+            a=a or [],
+            v=v or [],
+            c=c,
+            s_wr_sz=512 * 1024,
+            unpost=600,
+            u2sort="s",
+            mtp=[],
+            mte="a",
+            lang="eng",
+            logout=573,
+            **ka
+        )
+
+
 class NullBroker(object):
     def say(*args):
         pass