v0.10.22

idk forgot to pull

.vscode/launch.json

@@ -14,6 +14,8 @@
                 "-emp",
                 "-e2dsa",
                 "-e2ts",
+                "-mtp",
+                ".bpm=f,bin/mtag/audio-bpm.py",
                 "-a",
                 "ed:wark",
                 "-v",

.vscode/launch.py

@@ -0,0 +1,35 @@
+# takes arguments from launch.json
+# is used by no_dbg in tasks.json
+# launches 10x faster than mspython debugpy
+# and is stoppable with ^C
+
+import os
+import sys
+import shlex
+
+sys.path.insert(0, os.getcwd())
+
+import jstyleson
+from copyparty.__main__ import main as copyparty
+
+with open(".vscode/launch.json", "r", encoding="utf-8") as f:
+    tj = f.read()
+
+oj = jstyleson.loads(tj)
+argv = oj["configurations"][0]["args"]
+
+try:
+    sargv = " ".join([shlex.quote(x) for x in argv])
+    print(sys.executable + " -m copyparty " + sargv + "\n")
+except:
+    pass
+
+argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
+try:
+    copyparty(["a"] + argv)
+except SystemExit as ex:
+    if ex.code:
+        raise
+
+print("\n\033[32mokke\033[0m")
+sys.exit(1)

.vscode/tasks.json

@@ -8,8 +8,8 @@
         },
         {
             "label": "no_dbg",
-            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -e2ts -a ed:wark -v srv::r:aed:cnodupe -v dist:dist:r ;exit 1",
-            "type": "shell"
+            "type": "shell",
+            "command": "${config:python.pythonPath} .vscode/launch.py"
         }
     ]
 }

README.md

@@ -9,27 +9,66 @@
 turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
 
 * server runs on anything with `py2.7` or `py3.3+`
-* *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
+* browse/upload with IE4 / netscape4.0 on win3.11 (heh)
+* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+` for full speed
 * code standard: `black`
 
+📷 screenshots: [browser](#the-browser) // [upload](#uploading) // [md-viewer](#markdown-viewer) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [ie4](#browser-support)
+
+
+## readme toc
+
+* top
+    * [quickstart](#quickstart)
+    * [notes](#notes)
+    * [status](#status)
+* [bugs](#bugs)
+    * [general bugs](#general-bugs)
+    * [not my bugs](#not-my-bugs)
+* [the browser](#the-browser)
+    * [tabs](#tabs)
+    * [hotkeys](#hotkeys)
+    * [tree-mode](#tree-mode)
+    * [zip downloads](#zip-downloads)
+    * [uploading](#uploading)
+        * [file-search](#file-search)
+    * [markdown viewer](#markdown-viewer)
+    * [other tricks](#other-tricks)
+* [searching](#searching)
+    * [search configuration](#search-configuration)
+    * [metadata from audio files](#metadata-from-audio-files)
+    * [file parser plugins](#file-parser-plugins)
+    * [complete examples](#complete-examples)
+* [browser support](#browser-support)
+* [client examples](#client-examples)
+* [up2k](#up2k)
+* [dependencies](#dependencies)
+    * [optional gpl stuff](#optional-gpl-stuff)
+* [sfx](#sfx)
+    * [sfx repack](#sfx-repack)
+* [install on android](#install-on-android)
+* [dev env setup](#dev-env-setup)
+* [how to release](#how-to-release)
+* [todo](#todo)
+
 
 ## quickstart
 
 download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and you're all set!
 
-running the sfx without arguments (for example doubleclicking it on Windows) will let anyone access the current folder; see `-h` for help if you want accounts and volumes etc
+running the sfx without arguments (for example doubleclicking it on Windows) will give everyone full access to the current folder; see `-h` for help if you want accounts and volumes etc
 
 you may also want these, especially on servers:
 * [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
-* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for legit https)
+* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
 
 
 ## notes
 
 * iPhone/iPad: use Firefox to download files
-* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
-* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
-* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
+* Android-Firefox: takes a while to select files (their fix for ☝️)
+* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
 * paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
   * because no browsers currently implement the media-query to do this properly orz
 
@@ -48,14 +87,14 @@ you may also want these, especially on servers:
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ folders as zip / tar files
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view
   * ☑ media player
   * ✖ thumbnails
-  * ✖ SPA (browse while uploading)
-    * currently safe using the file-tree on the left only, not folders in the file list
+  * ☑ SPA (browse while uploading)
+    * if you use the file-tree on the left only, not folders in the file list
 * server indexing
   * ☑ locate files by contents
   * ☑ search by name/path/date/size
@@ -71,14 +110,145 @@ summary: it works! you can use it! (but technically not even close to beta)
 
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+
+## general bugs
+
+* all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
+## not my bugs
+
+* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
+  * this is an msys2 bug, the regular windows edition of python is fine
+
+
+# the browser
+
+![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/115978054-65106380-a57d-11eb-98f8-59e3dee73557.png)
+
+
+## tabs
+
+* `[🔎]` search by size, date, path/name, mp3-tags ... see [searching](#searching)
+* `[🚀]` and `[🎈]` are the uploaders, see [uploading](#uploading)
+* `[📂]` mkdir, create directories
+* `[📝]` new-md, create a new markdown document
+* `[📟]` send-msg, either to server-log or into textfiles if `--urlform save`
+* `[⚙️]` client configuration options
+
+
+## hotkeys
+
+the browser has the following hotkeys
+* `I/K` prev/next folder
+* `P` parent folder
+* when playing audio:
+  * `0..9` jump to 10%..90%
+  * `U/O` skip 10sec back/forward
+  * `J/L` prev/next song
+    * `J` also starts playing the folder
+
+
+## tree-mode
+
+by default there's a breadcrumbs path; you can replace this with a tree-browser sidebar thing by clicking the 🌲
+
+click `[-]` and `[+]` to adjust the size, and the `[a]` toggles if the tree should widen dynamically as you go deeper or stay fixed-size
+
+
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+you can also zip a selection of files or folders by clicking them in the browser, that brings up a selection editor and zip button in the bottom right
+
+![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/116008321-372a2e00-a614-11eb-9a4a-4a1fd9074224.png)
+
+## uploading
+
+two upload methods are available in the html client:
+* `🎈 bup`, the basic uploader, supports almost every browser since netscape 4.0
+* `🚀 up2k`, the fancy one
+
+up2k has several advantages:
+* you can drop folders into the browser (files are added recursively)
+* files are processed in chunks, and each chunk is checksummed
+  * uploads resume if they are interrupted (for example by a reboot)
+  * server detects any corruption; the client reuploads affected chunks
+  * the client doesn't upload anything that already exists on the server
+* the last-modified timestamp of the file is preserved
+
+see [up2k](#up2k) for details on how it works
+
+![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/115978061-680b5400-a57d-11eb-9ef6-cbb5f60aeccc.png)
+
+**protip:** you can avoid scaring away users with [docs/minimal-up2k.html](docs/minimal-up2k.html) which makes it look [much simpler](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+the up2k UI is the epitome of polished inutitive experiences:
+* "parallel uploads" specifies how many chunks to upload at the same time
+* `[🏃]` analysis of other files should continue while one is uploading
+* `[💭]` ask for confirmation before files are added to the list
+* `[💤]` sync uploading between other copyparty tabs so only one is active
+* `[🔎]` switch between upload and file-search mode
+
+and then theres the tabs below it,
+* `[ok]` is uploads which completed successfully
+* `[ng]` is the uploads which failed / got rejected (already exists, ...)
+* `[done]` shows a combined list of `[ok]` and `[ng]`, chronological order
+* `[busy]` files which are currently hashing, pending-upload, or uploading
+  * plus up to 3 entries each from `[done]` and `[que]` for context
+* `[que]` is all the files that are still queued
+
+### file-search
+
+![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/116008320-36919780-a614-11eb-803f-04162326a700.png)
+
+in the `[🚀 up2k]` tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere already
+
+files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
+* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else, this is no longer the case but now i've warmed up to the idea too much
+
+adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files
+
+note that since up2k has to read the file twice, `[🎈 bup]` can be up to 2x faster in extreme cases (if your internet connection is faster than the read-speed of your HDD)
+
+up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well thanks to tls also functioning as an integrity check
+
+
+## markdown viewer
+
+![copyparty-md-read-fs8](https://user-images.githubusercontent.com/241032/115978057-66419080-a57d-11eb-8539-d2be843991aa.png)
+
+* the document preview has a max-width which is the same as an A4 paper when printed
+
+
+## other tricks
+
+* you can link a particular timestamp in an audio file by adding it to the URL, such as `&20` / `&20s` / `&1m20` / `&t=1:20` after the `.../#af-c8960dab`
+
 
 # searching
 
+![copyparty-search-fs8](https://user-images.githubusercontent.com/241032/115978060-6772bd80-a57d-11eb-81d3-174e869b72c3.png)
+
 when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
 * make search queries by `size`/`date`/`directory-path`/`filename`, or...
-* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+* drag/drop a local file to see if the same contents exist somewhere on the server, see [file-search](#file-search)
 
 path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
 * path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
@@ -100,19 +270,24 @@ through arguments:
 * `-e2tsr` deletes all existing tags, so a full reindex
 
 the same arguments can be set as volume flags, in addition to `d2d` and `d2t` for disabling:
-* `-v ~/music::ce2dsa:ce2tsr` does a full reindex of everything on startup
-* `-v ~/music::cd2d` disables **all** indexing, even if any `-e2*` are on
-* `-v ~/music::cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
+* `-v ~/music::r:ce2dsa:ce2tsr` does a full reindex of everything on startup
+* `-v ~/music::r:cd2d` disables **all** indexing, even if any `-e2*` are on
+* `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 
 `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
 
+
+## metadata from audio files
+
 `-mte` decides which tags to index and display in the browser (and also the display order), this can be changed per-volume:
-* `-v ~/music::cmte=title,artist` indexes and displays *title* followed by *artist*
+* `-v ~/music::r:cmte=title,artist` indexes and displays *title* followed by *artist*
 
 if you add/remove a tag from `mte` you will need to run with `-e2tsr` once to rebuild the database, otherwise only new files will be affected
 
 `-mtm` can be used to add or redefine a metadata mapping, say you have media files with `foo` and `bar` tags and you want them to display as `qux` in the browser (preferring `foo` if both are present), then do `-mtm qux=foo,bar` and now you can `-mte artist,title,qux`
 
+tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric value
+
 see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
 
 `--no-mutagen` disables mutagen and uses ffprobe instead, which...
@@ -122,6 +297,66 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
 * more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
 
 
+## file parser plugins
+
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (as argument or volume flag), there is a default timeout of 30sec
+
+* `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
+* `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
+* `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
+
+*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` only do non-audio files, or `ad` do all files (d as in dontcare) 
+
+* `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`)
+
+
+## complete examples
+
+* read-only music server with bpm and key scanning  
+  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+
+
+# browser support
+
+![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/118192791-fb31fe00-b446-11eb-9647-898ea8efc1f7.png)
+
+`ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
+
+| feature         | ie6 | ie9 | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+| --------------- | --- | --- | ---- | ---- | ----- | ---- | --- | ---- |
+| browse files    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| basic uploader  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| make directory  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| send message    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| set sort order  |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| zip selection   |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| directory tree  |  -  |  -  | `*1` | yep  | yep   | yep  | yep | yep  |
+| up2k            |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| icons work      |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown editor |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown viewer |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| play mp3/m4a    |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| play ogg/opus   |  -  |  -  |  -   |  -   | yep   | yep  | `*2` | yep |
+
+* internet explorer 6 to 8 behave the same
+* firefox 52 and chrome 49 are the last winxp versions
+* `*1` only public folders (login session is dropped) and no history / back-button
+* `*2` using a wasm decoder which can sometimes get stuck and consumes a bit more power
+
+quick summary of more eccentric web-browsers trying to view a directory index:
+
+| browser | will it blend |
+| ------- | ------------- |
+| **safari** (14.0.3/macos) | is chrome with janky wasm, so playing opus can deadlock the javascript engine |
+| **safari** (14.0.1/iOS)   | same as macos, except it recovers from the deadlocks if you poke it a bit |
+| **links** (2.21/macports) | can browse, login, upload/mkdir/msg |
+| **lynx** (2.8.9/macports) | can browse, login, upload/mkdir/msg |
+| **w3m** (0.5.3/macports)  | can browse, login, upload at 100kB/s, mkdir/msg |
+| **netsurf** (3.10/arch)   | is basically ie6 with much better css (javascript has almost no effect) | 
+| **ie4** and **netscape** 4.0  | can browse (text is yellow on white), upload with `?b=u` |
+| **SerenityOS** (22d13d8)  | hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying |
+
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)
@@ -146,6 +381,22 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
     b512 <movie.mkv
 
 
+# up2k
+
+quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+* the up2k client splits a file into an "optimal" number of chunks
+  * 1 MiB each, unless that becomes more than 256 chunks
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
+* client posts the list of hashes, filename, size, last-modified
+* server creates the `wark`, an identifier for this upload
+  * `sha512( salt + filesize + chunk_hashes )`
+  * and a sparse file is created for the chunks to drop into
+* client uploads each chunk
+  * header entries for the chunk-hash and wark
+  * server writes chunks into place based on the hash
+* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+
+
 # dependencies
 
 * `jinja2` (is built into the SFX)
@@ -158,11 +409,18 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
 * `Pillow` (requires py2.7 or py3.5+)
 
 
+## optional gpl stuff
+
+some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
+
+these are standalone programs and will never be imported / evaluated by copyparty
+
+
 # sfx
 
-currently there are two self-contained binaries:
-* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
-* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
+currently there are two self-contained "binaries":
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere, **recommended**
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos, kinda deprecated
 
 launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
 
@@ -212,6 +470,7 @@ pip install black bandit pylint flake8  # vscode tooling
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies
+* `git tag v1.2.3 && git push origin --tags`
 * create github release with `make-tgz-release.sh`
 * upload to pypi with `make-pypi-release.(sh|bat)`
 * create sfx with `make-sfx.sh`
@@ -221,15 +480,21 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
+* separate sqlite table per tag
+* audio fingerprinting
+* readme.md as epilogue
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* drop onto folders
 * `os.copy_file_range` for up2k cloning
-* up2k partials ui
 * support pillow-simd
-* cache sha512 chunks on client
-* comment field
-* ~~look into android thumbnail cache file format~~ bad idea
+* single sha512 across all up2k chunks? maybe
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||
+
+discarded ideas
+
+* up2k partials ui
+* cache sha512 chunks on client
+* comment field
+* look into android thumbnail cache file format

bin/README.md

@@ -1,4 +1,4 @@
-# copyparty-fuse.py
+# [`copyparty-fuse.py`](copyparty-fuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
 * **supports Linux** -- expect `117 MiB/s` sequential read
@@ -29,7 +29,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse🅱️.py
+# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
 * mount a copyparty server as a local filesystem (read-only)
 * does the same thing except more correct, `samba` approves
 * **supports Linux** -- expect `18 MiB/s` (wait what)
@@ -37,5 +37,11 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse-streaming.py
+# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
 * pretend this doesn't exist
+
+
+
+# [`mtag/`](mtag/)
+* standalone programs which perform misc. file analysis
+* copyparty can Popen programs like these during file indexing to collect additional metadata

bin/copyparty-fuse.py

@@ -1008,6 +1008,12 @@ def main():
         log = null_log
         dbg = null_log
 
+    if ar.a and ar.a.startswith("$"):
+        fn = ar.a[1:]
+        log("reading password from file [{}]".format(fn))
+        with open(fn, "rb") as f:
+            ar.a = f.read().decode("utf-8").strip()
+
     if WINDOWS:
         os.system("rem")
 

bin/mtag/README.md

@@ -0,0 +1,34 @@
+standalone programs which take an audio file as argument
+
+some of these rely on libraries which are not MIT-compatible
+
+* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
+* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
+
+
+# dependencies
+
+run [`install-deps.sh`](install-deps.sh) to build/install most dependencies required by these programs (supports windows/linux/macos)
+
+*alternatively* (or preferably) use packages from your distro instead, then you'll need at least these:
+
+* from distro: `numpy vamp-plugin-sdk beatroot-vamp mixxx-keyfinder ffmpeg`
+* from pypy: `keyfinder vamp`
+
+
+# usage from copyparty
+
+`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
+
+* `f,` makes the detected value replace any existing values
+* the `.` in `.bpm` indicates numeric value
+* assumes the python files are in the folder you're launching copyparty from, replace the filename with a relative/absolute path if that's not the case
+* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
+
+
+## usage with volume-flags
+
+instead of affecting all volumes, you can set the options for just one volume like so:
+```
+copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
+```

bin/mtag/audio-bpm.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import vamp
+import tempfile
+import numpy as np
+import subprocess as sp
+
+from copyparty.util import fsenc
+
+"""
+dep: vamp
+dep: beatroot-vamp
+dep: ffmpeg
+"""
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-ss", "13",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-ac", "1",
+        "-ar", "22050",
+        "-t", "300",
+        "-f", "f32le",
+        tf
+    ])
+    # fmt: on
+
+    with open(tf, "rb") as f:
+        d = np.fromfile(f, dtype=np.float32)
+        try:
+            # 98% accuracy on jcore
+            c = vamp.collect(d, 22050, "beatroot-vamp:beatroot")
+            cl = c["list"]
+        except:
+            # fallback; 73% accuracy
+            plug = "vamp-example-plugins:fixedtempo"
+            c = vamp.collect(d, 22050, plug, parameters={"maxdflen": 40})
+            print(c["list"][0]["label"].split(" ")[0])
+            return
+
+        # throws if detection failed:
+        bpm = float(cl[-1]["timestamp"] - cl[1]["timestamp"])
+        bpm = round(60 * ((len(cl) - 1) / bpm), 2)
+        print(f"{bpm:.2f}")
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import sys
+import keyfinder
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+
+note: cannot fsenc
+"""
+
+
+try:
+    print(keyfinder.key(sys.argv[1]).camelot())
+except:
+    pass

bin/mtag/file-ext.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+
+"""
+example that just prints the file extension
+"""
+
+print(sys.argv[1].split(".")[-1])

bin/mtag/install-deps.sh

@@ -0,0 +1,265 @@
+#!/bin/bash
+set -e
+
+
+# install dependencies for audio-*.py
+#
+# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
+# win64: requires msys2-mingw64 environment
+# macos: requires macports
+#
+# has the following manual dependencies, especially on mac:
+#   https://www.vamp-plugins.org/pack.html
+#
+# installs stuff to the following locations:
+#   ~/pe/
+#   whatever your python uses for --user packages
+#
+# does the following terrible things:
+#   modifies the keyfinder python lib to load the .so in ~/pe
+
+
+linux=1
+
+win=
+[ ! -z "$MSYSTEM" ] || [ -e /msys2.exe ] && {
+	[ "$MSYSTEM" = MINGW64 ] || {
+		echo windows detected, msys2-mingw64 required
+		exit 1
+	}
+	pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	win=1
+	linux=
+}
+
+mac=
+[ $(uname -s) = Darwin ] && {
+	#pybin="$(printf '%s\n' /opt/local/bin/python* | (sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) | (sort -nr || cat) | (sed -E 's/([^ ]*) (.*)/\2\1/' || cat) | grep -E '/(python|pypy)[0-9\.-]*$' | head -n 1)"
+	pybin=/opt/local/bin/python3.9
+	[ -e "$pybin" ] || {
+		echo mac detected, python3 from macports required
+		exit 1
+	}
+	pkgs='ffmpeg python39 py39-wheel'
+	ninst=$(port installed | awk '/^  /{print$1}' | sort | uniq | grep -E '^('"$(echo "$pkgs" | tr ' ' '|')"')$' | wc -l)
+	[ $ninst -eq 3 ] || {
+		sudo port install $pkgs
+	}
+	mac=1
+	linux=
+}
+
+hash -r
+
+[ $mac ] || {
+	command -v python3 && pybin=python3 || pybin=python
+}
+
+$pybin -m pip install --user numpy
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+
+
+need() {
+	command -v $1 >/dev/null || {
+		echo need $1
+		exit 1
+	}
+}
+need cmake
+need ffmpeg
+need $pybin
+#need patchelf
+
+
+td="$(mktemp -d)"
+cln() {
+	rm -rf "$td"
+}
+trap cln EXIT
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	local yolo= ex=
+	[ $1 = "yolo" ] && yolo=1 && ex=k && shift
+	command -v curl >/dev/null && exec curl -${ex}JOL "$@"
+	
+	[ $yolo ] && ex=--no-check-certificate
+	exec wget --trust-server-names $ex "$@"
+}
+export -f dl_files
+
+
+github_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.tarball_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"tarball_url": "/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+gitlab_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.[0].assets.sources[]|select(.format|test("tar.gz")).url' ||
+
+		# fallback to abomination
+		tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	tee links |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+install_keyfinder() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
+	
+	github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
+
+	tar -xf mixxxdj-libkeyfinder-*
+	rm -- *.tar.gz
+	cd mixxxdj-libkeyfinder*
+	
+	h="$HOME"
+	so="lib/libkeyfinder.so"
+	memes=()
+
+	[ $win ] &&
+		so="bin/libkeyfinder.dll" &&
+		h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
+		memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
+	
+	[ $mac ] &&
+		so="lib/libkeyfinder.dylib"
+
+	cmake -DCMAKE_INSTALL_PREFIX="$h/pe/keyfinder" "${memes[@]}" -S . -B build
+	cmake --build build --parallel $(nproc || echo 4)
+	cmake --install build
+
+	libpath="$h/pe/keyfinder/$so"
+	[ $linux ] && [ ! -e "$libpath" ] &&
+		so=lib64/libkeyfinder.so
+	
+	libpath="$h/pe/keyfinder/$so"
+	[ -e "$libpath" ] || {
+		echo "so not found at $sop"
+		exit 1
+	}
+	
+	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
+	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
+	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
+	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
+	$pybin -m pip install --user keyfinder
+
+	pypath="$($pybin -c 'import keyfinder; print(keyfinder.__file__)')"
+	for pyso in "${pypath%/*}"/*.so; do
+		[ -e "$pyso" ] || break
+		patchelf --set-rpath "${libpath%/*}" "$pyso" ||
+			echo "WARNING: patchelf failed (only fatal on musl-based distros)"
+	done
+	
+	mv "$pypath"{,.bak}
+	(
+		printf 'import ctypes\nctypes.cdll.LoadLibrary("%s")\n' "$libpath"
+		cat "$pypath.bak"
+	) >"$pypath"
+
+	echo
+	echo libkeyfinder successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+have_beatroot() {
+	$pybin -c 'import vampyhost, sys; plugs = vampyhost.list_plugins(); sys.exit(0 if "beatroot-vamp:beatroot" in plugs else 1)'
+}
+
+
+install_vamp() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	
+	$pybin -m pip install --user vamp
+
+	have_beatroot || {
+		printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
+		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
+		sha512sum -c <(
+			echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874  -"
+		) <beatroot-vamp-v1.0.tar.gz
+		tar -xf beatroot-vamp-v1.0.tar.gz 
+		cd beatroot-vamp-v1.0
+		make -f Makefile.linux -j4
+		# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
+		mkdir ~/vamp
+		cp -pv beatroot-vamp.* ~/vamp/
+	}
+	
+	have_beatroot &&
+		printf '\033[32mfound the vamp beatroot plugin, nice\033[0m\n' ||
+		printf '\033[31mWARNING: could not find the vamp beatroot plugin, please install it for optimal results\033[0m\n'
+}
+
+
+# not in use because it kinda segfaults, also no windows support
+install_soundtouch() {
+	gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
+	
+	tar -xvf soundtouch-*
+	rm -- *.tar.gz
+	cd soundtouch-*
+	
+	# https://github.com/jrising/pysoundtouch
+	./bootstrap
+	./configure --enable-integer-samples CXXFLAGS="-fPIC" --prefix="$HOME/pe/soundtouch"
+	make -j$(nproc || echo 4)
+	make install
+	
+	CFLAGS=-I$HOME/pe/soundtouch/include/ \
+	LDFLAGS=-L$HOME/pe/soundtouch/lib \
+	$pybin -m pip install --user git+https://github.com/snowxmas/pysoundtouch.git
+	
+	pypath="$($pybin -c 'import importlib; print(importlib.util.find_spec("soundtouch").origin)')"
+	libpath="$(echo "$HOME/pe/soundtouch/lib/")"
+	patchelf --set-rpath "$libpath" "$pypath"
+
+	echo
+	echo soundtouch successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+[ "$1" = keyfinder ] && { install_keyfinder; exit $?; }
+[ "$1" = soundtouch ] && { install_soundtouch; exit $?; }
+[ "$1" = vamp ] && { install_vamp; exit $?; }
+
+echo no args provided, installing keyfinder and vamp
+install_keyfinder
+install_vamp

bin/mtag/sleep.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+
+import time
+import random
+
+v = random.random() * 6
+time.sleep(v)
+print(f"{v:.2f}")

copyparty/__init__.py

@@ -16,12 +16,17 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 
 class EnvParams(object):
     def __init__(self):
         self.mod = os.path.dirname(os.path.realpath(__file__))
+        if self.mod.endswith("__init__"):
+            self.mod = os.path.dirname(self.mod)
+
         if sys.platform == "win32":
             self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
         elif sys.platform == "darwin":

copyparty/__main__.py

@@ -16,12 +16,14 @@ import shutil
 import filecmp
 import locale
 import argparse
+import threading
+import traceback
 from textwrap import dedent
 
 from .__init__ import E, WINDOWS, VT100, PY2
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab
+from .util import py_desc, align_tab, IMPLICATIONS
 
 HAVE_SSL = True
 try:
@@ -53,6 +55,12 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+class Dodge11874(RiceFormatter):
+    def __init__(self, *args, **kwargs):
+        kwargs["width"] = 9003
+        super(Dodge11874, self).__init__(*args, **kwargs)
+
+
 def warn(msg):
     print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
@@ -164,34 +172,19 @@ def configure_ssl_ciphers(al):
         sys.exit(0)
 
 
-def main():
-    time.strptime("19970815", "%Y%m%d")  # python#7980
-    if WINDOWS:
-        os.system("rem")  # enables colors
+def sighandler(sig=None, frame=None):
+    msg = [""] * 5
+    for th in threading.enumerate():
+        msg.append(str(th))
+        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
 
-    desc = py_desc().replace("[", "\033[1;30m[")
+    msg.append("\n")
+    print("\n".join(msg))
 
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
-
-    ensure_locale()
-    if HAVE_SSL:
-        ensure_cert()
-
-    deprecated = [["-e2s", "-e2ds"]]
-    for dk, nk in deprecated:
-        try:
-            idx = sys.argv.index(dk)
-        except:
-            continue
-
-        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
-        sys.argv[idx] = nk
-        time.sleep(2)
 
+def run_argparse(argv, formatter):
     ap = argparse.ArgumentParser(
-        formatter_class=RiceFormatter,
+        formatter_class=formatter,
         prog="copyparty",
         description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
         epilog=dedent(
@@ -203,6 +196,9 @@ def main():
             
             list of cflags:
               "cnodupe" rejects existing files (instead of symlinking them)
+              "ce2d" sets -e2d (all -e2* args can be set using ce2* cflags)
+              "cd2t" disables metadata collection, overrides -e2t*
+              "cd2d" disables all database stuff, overrides -e2*
 
             example:\033[35m
               -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
@@ -247,9 +243,10 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
-    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
-    ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
+    ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 
     ap2 = ap.add_argument_group('database options')
@@ -264,26 +261,64 @@ def main():
     ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
 
     ap2 = ap.add_argument_group('SSL/TLS options')
     ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
     ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", type=str, help="ssl/tls versions to allow")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="ssl/tls versions to allow")
     ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+
+    ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
+    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/", help="dont log URLs matching")
     
-    al = ap.parse_args()
+    return ap.parse_args(args=argv[1:])
     # fmt: on
 
+
+def main(argv=None):
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if WINDOWS:
+        os.system("rem")  # enables colors
+
+    if argv is None:
+        argv = sys.argv
+
+    desc = py_desc().replace("[", "\033[1;30m[")
+
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
+    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+
+    ensure_locale()
+    if HAVE_SSL:
+        ensure_cert()
+
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        argv[idx] = nk
+        time.sleep(2)
+
+    try:
+        al = run_argparse(argv, RiceFormatter)
+    except AssertionError:
+        al = run_argparse(argv, Dodge11874)
+
     # propagate implications
-    for k1, k2 in [
-        ["e2dsa", "e2ds"],
-        ["e2ds", "e2d"],
-        ["e2tsr", "e2ts"],
-        ["e2ts", "e2t"],
-        ["e2t", "e2d"],
-    ]:
+    for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):
             setattr(al, k2, True)
 
@@ -312,6 +347,8 @@ def main():
             + "  (if you crash with codec errors then that is why)"
         )
 
+    # signal.signal(signal.SIGINT, sighandler)
+
     SvcHub(al).run()
 
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 9, 5)
-CODENAME = "the strongest music server"
-BUILD_DT = (2021, 3, 7)
+VERSION = (0, 10, 22)
+CODENAME = "zip it"
+BUILD_DT = (2021, 5, 18)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
 import re
+import os
+import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
-from .util import undot, Pebkac, fsdec, fsenc, statdir, nuprint
+from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint
 
 
 class VFS(object):
@@ -21,6 +23,14 @@ class VFS(object):
         self.nodes = {}  # child nodes
         self.all_vols = {vpath: self}  # flattened recursive
 
+    def __repr__(self):
+        return "VFS({})".format(
+            ", ".join(
+                "{}={!r}".format(k, self.__dict__[k])
+                for k in "realpath vpath uread uwrite flags".split()
+            )
+        )
+
     def _trk(self, vol):
         self.all_vols[vol.vpath] = vol
         return vol
@@ -44,6 +54,7 @@ class VFS(object):
                 self.uwrite,
                 self.flags,
             )
+            self._trk(vn)
             self.nodes[name] = vn
             return self._trk(vn.add(src, dst))
 
@@ -100,7 +111,27 @@ class VFS(object):
         if rem:
             rp += "/" + rem
 
-        return fsdec(os.path.realpath(fsenc(rp)))
+        try:
+            return fsdec(os.path.realpath(fsenc(rp)))
+        except:
+            if not WINDOWS:
+                raise
+
+            # cpython bug introduced in 3.8, still exists in 3.9.1;
+            # some win7sp1 and win10:20H2 boxes cannot realpath a
+            # networked drive letter such as b"n:" or b"n:\\"
+            #
+            # requirements to trigger:
+            #  * bytestring (not unicode str)
+            #  * just the drive letter (subfolders are ok)
+            #  * networked drive (regular disks and vmhgfs are ok)
+            #  * on an enterprise network (idk, cannot repro with samba)
+            #
+            # hits the following exceptions in succession:
+            #  * access denied at L601: "path = _getfinalpathname(path)"
+            #  * "cant concat str to bytes" at L621: "return path + tail"
+            #
+            return os.path.realpath(rp)
 
     def ls(self, rem, uname, scandir, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
@@ -110,7 +141,12 @@ class VFS(object):
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
-                if uname in vn2.uread or "*" in vn2.uread:
+                if (
+                    uname in vn2.uread
+                    or "*" in vn2.uread
+                    or uname in vn2.uwrite
+                    or "*" in vn2.uwrite
+                ):
                     virt_vis[name] = vn2
 
             # no vfs nodes in the list of real inodes
@@ -118,6 +154,78 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+
+                rm = [x for x in rd if x[0] not in flt]
+                [rd.remove(x) for x in rm]
+
+                rm = [x for x in vd.keys() if x not in flt]
+                [vd.pop(x) for x in rm]
+
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)
@@ -138,6 +246,7 @@ class AuthSrv(object):
         self.args = args
         self.log_func = log_func
         self.warn_anonwrite = warn_anonwrite
+        self.line_ctr = 0
 
         if WINDOWS:
             self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -150,12 +259,6 @@ class AuthSrv(object):
     def log(self, msg, c=0):
         self.log_func("auth", msg, c)
 
-    def invert(self, orig):
-        if PY2:
-            return {v: k for k, v in orig.iteritems()}
-        else:
-            return {v: k for k, v in orig.items()}
-
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
         it = iter(iterable)
@@ -169,7 +272,9 @@ class AuthSrv(object):
     def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount):
         vol_src = None
         vol_dst = None
+        self.line_ctr = 0
         for ln in [x.decode("utf-8").strip() for x in fd]:
+            self.line_ctr += 1
             if not ln and vol_src is not None:
                 vol_src = None
                 vol_dst = None
@@ -199,17 +304,45 @@ class AuthSrv(object):
                 mflags[vol_dst] = {}
                 continue
 
-            lvl, uname = ln.split(" ")
-            if lvl in "ra":
-                mread[vol_dst].append(uname)
-            if lvl in "wa":
-                mwrite[vol_dst].append(uname)
-            if lvl == "c":
-                cval = True
-                if "=" in uname:
-                    uname, cval = uname.split("=", 1)
+            if len(ln) > 1:
+                lvl, uname = ln.split(" ")
+            else:
+                lvl = ln
+                uname = "*"
 
-                mflags[vol_dst][uname] = cval
+            self._read_vol_str(
+                lvl, uname, mread[vol_dst], mwrite[vol_dst], mflags[vol_dst]
+            )
+
+    def _read_vol_str(self, lvl, uname, mr, mw, mf):
+        if lvl == "c":
+            cval = True
+            if "=" in uname:
+                uname, cval = uname.split("=", 1)
+
+            self._read_volflag(mf, uname, cval, False)
+            return
+
+        if uname == "":
+            uname = "*"
+
+        if lvl in "ra":
+            mr.append(uname)
+
+        if lvl in "wa":
+            mw.append(uname)
+
+    def _read_volflag(self, flags, name, value, is_list):
+        if name not in ["mtp"]:
+            flags[name] = value
+            return
+
+        if not is_list:
+            value = [value]
+        elif not value:
+            return
+
+        flags[name] = flags.get(name, []) + value
 
     def reload(self):
         """
@@ -232,7 +365,7 @@ class AuthSrv(object):
 
         if self.args.v:
             # list of src:dst:permset:permset:...
-            # permset is [rwa]username
+            # permset is [rwa]username or [c]flag
             for v_str in self.args.v:
                 m = self.re_vol.match(v_str)
                 if not m:
@@ -249,27 +382,17 @@ class AuthSrv(object):
 
                 perms = perms.split(":")
                 for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
-                    if lvl == "c":
-                        cval = True
-                        if "=" in uname:
-                            uname, cval = uname.split("=", 1)
-
-                        mflags[dst][uname] = cval
-                        continue
-
-                    if uname == "":
-                        uname = "*"
-
-                    if lvl in "ra":
-                        mread[dst].append(uname)
-
-                    if lvl in "wa":
-                        mwrite[dst].append(uname)
+                    self._read_vol_str(lvl, uname, mread[dst], mwrite[dst], mflags[dst])
 
         if self.args.c:
             for cfg_fn in self.args.c:
                 with open(cfg_fn, "rb") as f:
-                    self._parse_config_file(f, user, mread, mwrite, mflags, mount)
+                    try:
+                        self._parse_config_file(f, user, mread, mwrite, mflags, mount)
+                    except:
+                        m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
+                        print(m.format(cfg_fn, self.line_ctr))
+                        raise
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
@@ -310,6 +433,8 @@ class AuthSrv(object):
             )
             raise Exception("invalid config")
 
+        all_mte = {}
+        errors = False
         for vol in vfs.all_vols.values():
             if (self.args.e2ds and vol.uwrite) or self.args.e2dsa:
                 vol.flags["e2ds"] = True
@@ -321,10 +446,75 @@ class AuthSrv(object):
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
+            for k1, k2 in IMPLICATIONS:
+                if k1 in vol.flags:
+                    vol.flags[k2] = True
+
             # default tag-list if unset
             if "mte" not in vol.flags:
                 vol.flags["mte"] = self.args.mte
 
+            # append parsers from argv to volume-flags
+            self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
+
+            # d2d drops all database features for a volume
+            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"]]:
+                if not vol.flags.get(grp, False):
+                    continue
+
+                vol.flags["d2t"] = True
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # mt* needs e2t so drop those too
+            for grp, rm in [["e2t", "mt"]]:
+                if vol.flags.get(grp, False):
+                    continue
+
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # verify tags mentioned by -mt[mp] are used by -mte
+            local_mtp = {}
+            local_only_mtp = {}
+            for a in vol.flags.get("mtp", []) + vol.flags.get("mtm", []):
+                a = a.split("=")[0]
+                local_mtp[a] = True
+                local = True
+                for b in self.args.mtp or []:
+                    b = b.split("=")[0]
+                    if a == b:
+                        local = False
+
+                if local:
+                    local_only_mtp[a] = True
+
+            local_mte = {}
+            for a in vol.flags.get("mte", "").split(","):
+                local = True
+                all_mte[a] = True
+                local_mte[a] = True
+                for b in self.args.mte.split(","):
+                    if not a or not b:
+                        continue
+
+                    if a == b:
+                        local = False
+
+            for mtp in local_only_mtp.keys():
+                if mtp not in local_mte:
+                    m = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)'
+                    self.log(m.format(vol.vpath, mtp), 1)
+                    errors = True
+
+        for mtp in self.args.mtp or []:
+            mtp = mtp.split("=")[0]
+            if mtp not in all_mte:
+                m = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volume-flag)'
+                self.log(m.format(mtp), 1)
+                errors = True
+
+        if errors:
+            sys.exit(1)
+
         try:
             v, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == v.realpath:
@@ -337,7 +527,7 @@ class AuthSrv(object):
         with self.mutex:
             self.vfs = vfs
             self.user = user
-            self.iuser = self.invert(user)
+            self.iuser = {v: k for k, v in user.items()}
 
         # import pprint
         # pprint.pprint({"usr": user, "rd": mread, "wr": mwrite, "mnt": mount})

copyparty/broker_mp.py

@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()

copyparty/broker_mpw.py

@@ -73,7 +73,9 @@ class MpWorker(object):
                 if PY2:
                     sck = pickle.loads(sck)  # nosec
 
-                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+                if self.args.log_conn:
+                    self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+                
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:

copyparty/broker_thr.py

@@ -28,7 +28,9 @@ class BrokerThr(object):
     def put(self, want_retval, dest, *args):
         if dest == "httpconn":
             sck, addr = args
-            self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+
             self.httpsrv.accept(sck, addr)
 
         else:

copyparty/httpcli.py

@@ -7,13 +7,16 @@ import gzip
 import time
 import copy
 import json
+import string
 import socket
 import ctypes
 from datetime import datetime
 import calendar
 
-from .__init__ import E, PY2, WINDOWS
+from .__init__ import E, PY2, WINDOWS, ANYWIN
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .szip import StreamZip
+from .star import StreamTar
 
 if not PY2:
     unicode = str
@@ -45,13 +48,17 @@ class HttpCli(object):
         self.log_func(self.log_src, msg, c)
 
     def _check_nonfatal(self, ex):
-        return ex.code < 400 or ex.code == 404
+        return ex.code < 400 or ex.code in [404, 429]
 
     def _assert_safe_rem(self, rem):
         # sanity check to prevent any disasters
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def j2(self, name, **kwargs):
+        tpl = self.conn.hsrv.j2[name]
+        return tpl.render(**kwargs) if kwargs else tpl
+
     def run(self):
         """returns true if connection can be reused"""
         self.keepalive = False
@@ -67,7 +74,7 @@ class HttpCli(object):
                 headerlines.pop(0)
 
             try:
-                self.mode, self.req, _ = headerlines[0].split(" ")
+                self.mode, self.req, self.http_ver = headerlines[0].split(" ")
             except:
                 raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
 
@@ -86,29 +93,22 @@ class HttpCli(object):
             self.headers[k.lower()] = v.strip()
 
         v = self.headers.get("connection", "").lower()
-        self.keepalive = not v.startswith("close")
+        self.keepalive = not v.startswith("close") and self.http_ver != "HTTP/1.0"
 
         v = self.headers.get("x-forwarded-for", None)
         if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
             self.ip = v.split(",")[0]
             self.log_src = self.conn.set_rproxy(self.ip)
 
-        self.uname = "*"
-        if "cookie" in self.headers:
-            cookies = self.headers["cookie"].split(";")
-            for k, v in [x.split("=", 1) for x in cookies]:
-                if k.strip() != "cppwd":
-                    continue
+        if self.args.ihead:
+            keys = self.args.ihead
+            if "*" in keys:
+                keys = list(sorted(self.headers.keys()))
 
-                v = unescape_cookie(v)
-                if v in self.auth.iuser:
-                    self.uname = self.auth.iuser[v]
-
-                break
-
-        if self.uname:
-            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
-            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
+            for k in keys:
+                v = self.headers.get(k)
+                if v is not None:
+                    self.log("[H] {}: \033[33m[{}]".format(k, v), 6)
 
         # split req into vpath + uparam
         uparam = {}
@@ -130,13 +130,35 @@ class HttpCli(object):
                 else:
                     uparam[k.lower()] = False
 
+        self.ouparam = {k: v for k, v in uparam.items()}
+
+        cookies = self.headers.get("cookie") or {}
+        if cookies:
+            cookies = [x.split("=", 1) for x in cookies.split(";") if "=" in x]
+            cookies = {k.strip(): unescape_cookie(v) for k, v in cookies}
+            for kc, ku in [["cppwd", "pw"], ["b", "b"]]:
+                if kc in cookies and ku not in uparam:
+                    uparam[ku] = cookies[kc]
+
         self.uparam = uparam
+        self.cookies = cookies
         self.vpath = unquotep(vpath)
 
+        pwd = uparam.get("pw")
+        self.uname = self.auth.iuser.get(pwd, "*")
+        if self.uname:
+            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
+            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
+
         ua = self.headers.get("user-agent", "")
-        if ua.startswith("rclone/"):
+        self.is_rclone = ua.startswith("rclone/")
+        if self.is_rclone:
             uparam["raw"] = False
             uparam["dots"] = False
+            uparam["b"] = False
+            cookies["b"] = False
+
+        self.do_log = not self.conn.lf_url or not self.conn.lf_url.match(self.req)
 
         try:
             if self.mode in ["GET", "HEAD"]:
@@ -153,14 +175,18 @@ class HttpCli(object):
         except Pebkac as ex:
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
-                self.keepalive = self._check_nonfatal(ex)
-                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
+                if not self._check_nonfatal(ex):
+                    self.keepalive = False
+
+                self.log("{}\033[0m, {}".format(str(ex), self.vpath), 3)
+                msg = "<pre>{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                self.reply(msg.encode("utf-8", "replace"), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
 
     def send_headers(self, length, status=200, mime=None, headers={}):
-        response = ["HTTP/1.1 {} {}".format(status, HTTPCODE[status])]
+        response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
 
         if length is not None:
             response.append("Content-Length: " + unicode(length))
@@ -172,10 +198,8 @@ class HttpCli(object):
         self.out_headers.update(headers)
 
         # default to utf8 html if no content-type is set
-        try:
-            mime = mime or self.out_headers["Content-Type"]
-        except KeyError:
-            mime = "text/html; charset=UTF-8"
+        if not mime:
+            mime = self.out_headers.get("Content-Type", "text/html; charset=UTF-8")
 
         self.out_headers["Content-Type"] = mime
 
@@ -204,18 +228,58 @@ class HttpCli(object):
         self.log(body.rstrip())
         self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
+    def urlq(self, add={}, rm=[]):
+        """
+        generates url query based on uparam (b, pw, all others)
+        removing anything in rm, adding pairs in add
+        """
+
+        if self.is_rclone:
+            return ""
+
+        kv = {
+            k: v
+            for k, v in self.uparam.items()
+            if k not in rm and self.cookies.get(k) != v
+        }
+        kv.update(add)
+        if not kv:
+            return ""
+
+        r = ["{}={}".format(k, quotep(v)) if v else k for k, v in kv.items()]
+        return "?" + "&amp;".join(r)
+
+    def redirect(
+        self, vpath, suf="", msg="aight", flavor="go to", click=True, use302=False
+    ):
+        html = self.j2(
+            "msg",
+            h2='<a href="/{}">{} /{}</a>'.format(
+                quotep(vpath) + suf, flavor, html_escape(vpath, crlf=True) + suf
+            ),
+            pre=msg,
+            click=click,
+        ).encode("utf-8", "replace")
+
+        if use302:
+            h = {"Location": "/" + vpath, "Cache-Control": "no-cache"}
+            self.reply(html, status=302, headers=h)
+        else:
+            self.reply(html)
+
     def handle_get(self):
-        logmsg = "{:4} {}".format(self.mode, self.req)
+        if self.do_log:
+            logmsg = "{:4} {}".format(self.mode, self.req)
 
-        if "range" in self.headers:
-            try:
-                rval = self.headers["range"].split("=", 1)[1]
-            except:
-                rval = self.headers["range"]
+            if "range" in self.headers:
+                try:
+                    rval = self.headers["range"].split("=", 1)[1]
+                except:
+                    rval = self.headers["range"]
 
-            logmsg += " [\033[36m" + rval + "\033[0m]"
+                logmsg += " [\033[36m" + rval + "\033[0m]"
 
-        self.log(logmsg)
+            self.log(logmsg)
 
         # "embedded" resources
         if self.vpath.startswith(".cpr"):
@@ -226,23 +290,27 @@ class HttpCli(object):
             return self.tx_tree()
 
         # conditional redirect to single volumes
-        if self.vpath == "" and not self.uparam:
+        if self.vpath == "" and not self.ouparam:
             nread = len(self.rvol)
             nwrite = len(self.wvol)
             if nread + nwrite == 1 or (self.rvol == self.wvol and nread == 1):
                 if nread == 1:
-                    self.vpath = self.rvol[0]
+                    vpath = self.rvol[0]
                 else:
-                    self.vpath = self.wvol[0]
+                    vpath = self.wvol[0]
 
-                self.absolute_urls = True
+                if self.vpath != vpath:
+                    self.redirect(vpath, flavor="redirecting to", use302=True)
+                    return True
 
-        # go home if verboten
         self.readable, self.writable = self.conn.auth.vfs.can_access(
             self.vpath, self.uname
         )
         if not self.readable and not self.writable:
-            self.log("inaccessible: [{}]".format(self.vpath))
+            if self.vpath:
+                self.log("inaccessible: [{}]".format(self.vpath))
+                raise Pebkac(404)
+
             self.uparam = {"h": False}
 
         if "h" in self.uparam:
@@ -252,7 +320,9 @@ class HttpCli(object):
         return self.tx_browser()
 
     def handle_options(self):
-        self.log("OPTIONS " + self.req)
+        if self.do_log:
+            self.log("OPTIONS " + self.req)
+
         self.send_headers(
             None,
             204,
@@ -312,8 +382,19 @@ class HttpCli(object):
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
                 for buf in reader:
-                    buf = buf.decode("utf-8", "replace")
-                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
+                    orig = buf.decode("utf-8", "replace")
+                    m = "urlform_raw {} @ {}\n  {}\n"
+                    self.log(m.format(len(orig), self.vpath, orig))
+                    try:
+                        plain = unquote(buf.replace(b"+", b" "))
+                        plain = plain.decode("utf-8", "replace")
+                        if buf.startswith(b"msg="):
+                            plain = plain[4:]
+
+                        m = "urlform_dec {} @ {}\n  {}\n"
+                        self.log(m.format(len(plain), self.vpath, plain))
+                    except Exception as ex:
+                        self.log(repr(ex))
 
             if "get" in opt:
                 return self.handle_get()
@@ -388,8 +469,30 @@ class HttpCli(object):
         if act == "tput":
             return self.handle_text_upload()
 
+        if act == "zip":
+            return self.handle_zip_post()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
+    def handle_zip_post(self):
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                break
+
+        if v is None:
+            raise Pebkac(422, "need zip or tar keyword")
+
+        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        items = self.parser.require("files", 1024 * 1024)
+        if not items:
+            raise Pebkac(422, "need files list")
+
+        items = items.replace("\r", "").split("\n")
+        items = [unquotep(x) for x in items if items]
+
+        return self.tx_zip(k, v, vn, rem, items, self.args.ed)
+
     def handle_post_json(self):
         try:
             remains = int(self.headers["content-length"])
@@ -417,15 +520,18 @@ class HttpCli(object):
         if "srch" in self.uparam or "srch" in body:
             return self.handle_search(body)
 
-        # prefer this over undot; no reason to allow traversion
-        if "/" in body["name"]:
-            raise Pebkac(400, "folders verboten")
-
         # up2k-php compat
         for k in "chunkpit.php", "handshake.php":
             if self.vpath.endswith(k):
                 self.vpath = self.vpath[: -len(k)]
 
+        sub = None
+        name = undot(body["name"])
+        if "/" in name:
+            sub, name = name.rsplit("/", 1)
+            self.vpath = "/".join([self.vpath, sub]).strip("/")
+            body["name"] = name
+
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
 
         body["vtop"] = vfs.vpath
@@ -434,12 +540,22 @@ class HttpCli(object):
         body["addr"] = self.ip
         body["vcfg"] = vfs.flags
 
-        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
-        response = x.get()
-        response = json.dumps(response)
+        if sub:
+            try:
+                dst = os.path.join(vfs.realpath, rem)
+                os.makedirs(dst)
+            except:
+                if not os.path.isdir(dst):
+                    raise Pebkac(400, "some file got your folder name")
 
-        self.log(response)
-        self.reply(response.encode("utf-8"), mime="application/json")
+        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
+        ret = x.get()
+        if sub:
+            ret["name"] = "/".join([sub, ret["name"]])
+
+        ret = json.dumps(ret)
+        self.log(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
         return True
 
     def handle_search(self, body):
@@ -450,19 +566,30 @@ class HttpCli(object):
 
         idx = self.conn.get_u2idx()
         t0 = time.time()
+        if idx.p_end:
+            penalty = 0.7
+            t_idle = t0 - idx.p_end
+            if idx.p_dur > 0.7 and t_idle < penalty:
+                m = "rate-limit ({:.1f} sec), cost {:.2f}, idle {:.2f}"
+                raise Pebkac(429, m.format(penalty, idx.p_dur, t_idle))
+
         if "srch" in body:
             # search by up2k hashlist
             vbody = copy.deepcopy(body)
             vbody["hash"] = len(vbody["hash"])
             self.log("qj: " + repr(vbody))
             hits = idx.fsearch(vols, body)
-            self.log("q#: {} ({:.2f}s)".format(repr(hits), time.time() - t0))
-            taglist = []
+            msg = repr(hits)
+            taglist = {}
         else:
             # search by query params
             self.log("qj: " + repr(body))
             hits, taglist = idx.search(vols, body)
-            self.log("q#: {} ({:.2f}s)".format(len(hits), time.time() - t0))
+            msg = len(hits)
+
+        idx.p_end = time.time()
+        idx.p_dur = idx.p_end - t0
+        self.log("q#: {} ({:.2f}s)".format(msg, idx.p_dur))
 
         order = []
         cfg = self.args.mte.split(",")
@@ -545,7 +672,7 @@ class HttpCli(object):
             self.loud_reply(x, status=500)
             return False
 
-        if not WINDOWS and num_left == 0:
+        if not ANYWIN and num_left == 0:
             times = (int(time.time()), int(lastmod))
             self.log("no more chunks, setting times {}".format(times))
             try:
@@ -564,13 +691,16 @@ class HttpCli(object):
 
         if pwd in self.auth.iuser:
             msg = "login ok"
+            dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
+            exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
         else:
             msg = "naw dude"
             pwd = "x"  # nosec
+            exp = "Fri, 15 Aug 1997 01:00:00 GMT"
 
-        h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
-        html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
-        self.reply(html.encode("utf-8"), headers=h)
+        ck = "cppwd={}; Path=/; Expires={}; SameSite=Lax".format(pwd, exp)
+        html = self.j2("msg", h1=msg, h2='<a href="/">ack</a>', redir="/")
+        self.reply(html.encode("utf-8"), headers={"Set-Cookie": ck})
         return True
 
     def handle_mkdir(self):
@@ -599,13 +729,7 @@ class HttpCli(object):
                 raise Pebkac(500, "mkdir failed, check the logs")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        esc_paths = [quotep(vpath), html_escape(vpath)]
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
-            pre="aight",
-            click=True,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(vpath)
         return True
 
     def handle_new_md(self):
@@ -632,14 +756,7 @@ class HttpCli(object):
                 f.write(b"`GRUNNUR`\n")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
-                quotep(vpath), html_escape(vpath)
-            ),
-            pre="aight",
-            click=True,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(vpath, "?edit")
         return True
 
     def handle_plain_upload(self):
@@ -658,7 +775,9 @@ class HttpCli(object):
 
                 if p_file and not nullwrite:
                     fdir = os.path.join(vfs.realpath, rem)
-                    fname = sanitize_fn(p_file)
+                    fname = sanitize_fn(
+                        p_file, bad=[".prologue.html", ".epilogue.html"]
+                    )
 
                     if not os.path.isdir(fsenc(fdir)):
                         raise Pebkac(404, "that folder does not exist")
@@ -678,7 +797,7 @@ class HttpCli(object):
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
 
-                        files.append([sz, sha512_hex])
+                        files.append([sz, sha512_hex, p_file, fname])
                         self.conn.hsrv.broker.put(
                             False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
                         )
@@ -687,12 +806,16 @@ class HttpCli(object):
                 except Pebkac:
                     if fname != os.devnull:
                         fp = os.path.join(fdir, fname)
+                        fp2 = fp
+                        if self.args.dotpart:
+                            fp2 = os.path.join(fdir, "." + fname)
+
                         suffix = ".PARTIAL"
                         try:
-                            os.rename(fsenc(fp), fsenc(fp + suffix))
+                            os.rename(fsenc(fp), fsenc(fp2 + suffix))
                         except:
-                            fp = fp[: -len(suffix)]
-                            os.rename(fsenc(fp), fsenc(fp + suffix))
+                            fp2 = fp2[: -len(suffix) - 1]
+                            os.rename(fsenc(fp), fsenc(fp2 + suffix))
 
                     raise
 
@@ -709,10 +832,13 @@ class HttpCli(object):
             errmsg = "ERROR: " + errmsg
             status = "ERROR"
 
-        msg = "{0} // {1} bytes // {2:.3f} MiB/s\n".format(status, sz_total, spd)
+        msg = "{} // {} bytes // {:.3f} MiB/s\n".format(status, sz_total, spd)
 
-        for sz, sha512 in files:
-            msg += "sha512: {0} // {1} bytes\n".format(sha512[:56], sz)
+        for sz, sha512, ofn, lfn in files:
+            vpath = self.vpath + "/" + lfn
+            msg += 'sha512: {} // {} bytes // <a href="/{}">{}</a>\n'.format(
+                sha512[:56], sz, quotep(vpath), html_escape(ofn, crlf=True)
+            )
             # truncated SHA-512 prevents length extension attacks;
             # using SHA-512/224, optionally SHA-512/256 = :64
 
@@ -720,31 +846,13 @@ class HttpCli(object):
         self.log("{} {}".format(vspd, msg))
 
         if not nullwrite:
-            # TODO this is bad
             log_fn = "up.{:.6f}.txt".format(t0)
             with open(log_fn, "wb") as f:
-                f.write(
-                    (
-                        "\n".join(
-                            unicode(x)
-                            for x in [
-                                ":".join(unicode(x) for x in [self.ip, self.addr[1]]),
-                                msg.rstrip(),
-                            ]
-                        )
-                        + "\n"
-                        + errmsg
-                        + "\n"
-                    ).encode("utf-8")
-                )
+                ft = "{}:{}".format(self.ip, self.addr[1])
+                ft = "{}\n{}\n{}\n".format(ft, msg.rstrip(), errmsg)
+                f.write(ft.encode("utf-8"))
 
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">return to /{}</a>'.format(
-                quotep(self.vpath), html_escape(self.vpath)
-            ),
-            pre=msg,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
         self.parser.drop()
         return True
 
@@ -844,13 +952,14 @@ class HttpCli(object):
         return True
 
     def _chk_lastmod(self, file_ts):
+        date_fmt = "%a, %d %b %Y %H:%M:%S GMT"
         file_dt = datetime.utcfromtimestamp(file_ts)
-        file_lastmod = file_dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
+        file_lastmod = file_dt.strftime(date_fmt)
 
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
-                cli_dt = time.strptime(cli_lastmod, "%a, %d %b %Y %H:%M:%S GMT")
+                cli_dt = time.strptime(cli_lastmod, date_fmt)
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
             except Exception as ex:
@@ -1009,7 +1118,9 @@ class HttpCli(object):
         logmsg += unicode(status) + logtail
 
         if self.mode == "HEAD" or not do_send:
-            self.log(logmsg)
+            if self.do_log:
+                self.log(logmsg)
+
             return True
 
         ret = True
@@ -1023,32 +1134,92 @@ class HttpCli(object):
             logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
 
         spd = self._spd((upper - lower) - remains)
-        self.log("{},  {}".format(logmsg, spd))
+        if self.do_log:
+            self.log("{},  {}".format(logmsg, spd))
+
         return ret
 
+    def tx_zip(self, fmt, uarg, vn, rem, items, dots):
+        if self.args.no_zip:
+            raise Pebkac(400, "not enabled")
+
+        logmsg = "{:4} {} ".format("", self.req)
+        self.keepalive = False
+
+        if not uarg:
+            uarg = ""
+
+        if fmt == "tar":
+            mime = "application/x-tar"
+            packer = StreamTar
+        else:
+            mime = "application/zip"
+            packer = StreamZip
+
+        fn = items[0] if items and items[0] else self.vpath
+        if fn:
+            fn = fn.rstrip("/").split("/")[-1]
+        else:
+            fn = self.headers.get("host", "hey")
+
+        afn = "".join(
+            [x if x in (string.ascii_letters + string.digits) else "_" for x in fn]
+        )
+
+        bascii = unicode(string.ascii_letters + string.digits).encode("utf-8")
+        ufn = fn.encode("utf-8", "xmlcharrefreplace")
+        if PY2:
+            ufn = [unicode(x) if x in bascii else "%{:02x}".format(ord(x)) for x in ufn]
+        else:
+            ufn = [
+                chr(x).encode("utf-8")
+                if x in bascii
+                else "%{:02x}".format(x).encode("ascii")
+                for x in ufn
+            ]
+        ufn = b"".join(ufn).decode("ascii")
+
+        cdis = "attachment; filename=\"{}.{}\"; filename*=UTF-8''{}.{}"
+        cdis = cdis.format(afn, fmt, ufn, fmt)
+        self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis})
+
+        fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
+        # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
+        bsent = 0
+        for buf in bgen.gen():
+            if not buf:
+                break
+
+            try:
+                self.s.sendall(buf)
+                bsent += len(buf)
+            except:
+                logmsg += " \033[31m" + unicode(bsent) + "\033[0m"
+                break
+
+        spd = self._spd(bsent)
+        self.log("{},  {}".format(logmsg, spd))
+        return True
+
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
-        if "edit2" in self.uparam:
-            html_path = "web/mde.html"
-            template = self.conn.tpl_mde
-        else:
-            html_path = "web/md.html"
-            template = self.conn.tpl_md
 
-        html_path = os.path.join(E.mod, html_path)
+        tpl = "mde" if "edit2" in self.uparam else "md"
+        html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
+        template = self.j2(tpl)
 
         st = os.stat(fsenc(fs_path))
-        # sz_md = st.st_size
         ts_md = st.st_mtime
 
         st = os.stat(fsenc(html_path))
         ts_html = st.st_mtime
 
-        # TODO dont load into memory ;_;
-        #   (trivial fix, count the &'s)
-        with open(fsenc(fs_path), "rb") as f:
-            md = f.read().replace(b"&", b"&amp;")
-            sz_md = len(md)
+        sz_md = 0
+        for buf in yieldfile(fs_path):
+            sz_md += len(buf)
+            for c, v in [[b"&", 4], [b"<", 3], [b">", 3]]:
+                sz_md += (len(buf) - len(buf.replace(c, b""))) * v
 
         file_ts = max(ts_md, ts_html)
         file_lastmod, do_send = self._chk_lastmod(file_ts)
@@ -1056,38 +1227,50 @@ class HttpCli(object):
         self.out_headers["Cache-Control"] = "no-cache"
         status = 200 if do_send else 304
 
+        boundary = "\roll\tide"
         targs = {
             "edit": "edit" in self.uparam,
-            "title": html_escape(self.vpath),
+            "title": html_escape(self.vpath, crlf=True),
             "lastmod": int(ts_md * 1000),
             "md_plug": "true" if self.args.emp else "false",
             "md_chk_rate": self.args.mcr,
-            "md": "",
+            "md": boundary,
         }
-        sz_html = len(template.render(**targs).encode("utf-8"))
-        self.send_headers(sz_html + sz_md, status)
+        html = template.render(**targs).encode("utf-8")
+        html = html.split(boundary.encode("utf-8"))
+        if len(html) != 2:
+            raise Exception("boundary appears in " + html_path)
+
+        self.send_headers(sz_md + len(html[0]) + len(html[1]), status)
 
         logmsg += unicode(status)
         if self.mode == "HEAD" or not do_send:
-            self.log(logmsg)
+            if self.do_log:
+                self.log(logmsg)
+
             return True
 
-        # TODO jinja2 can stream this right?
-        targs["md"] = md.decode("utf-8", "replace")
-        html = template.render(**targs).encode("utf-8")
         try:
-            self.s.sendall(html)
+            self.s.sendall(html[0])
+            for buf in yieldfile(fs_path):
+                self.s.sendall(html_bescape(buf))
+
+            self.s.sendall(html[1])
+
         except:
             self.log(logmsg + " \033[31md/c\033[0m")
             return False
 
-        self.log(logmsg + " " + unicode(len(html)))
+        if self.do_log:
+            self.log(logmsg + " " + unicode(len(html)))
+
         return True
 
     def tx_mounts(self):
+        suf = self.urlq(rm=["h"])
         rvol = [x + "/" if x else x for x in self.rvol]
         wvol = [x + "/" if x else x for x in self.wvol]
-        html = self.conn.tpl_mounts.render(this=self, rvol=rvol, wvol=wvol)
+        html = self.j2("splash", this=self, rvol=rvol, wvol=wvol, url_suf=suf)
         self.reply(html.encode("utf-8"))
         return True
 
@@ -1156,7 +1339,7 @@ class HttpCli(object):
                 else:
                     vpath += "/" + node
 
-                vpnodes.append([quotep(vpath) + "/", html_escape(node)])
+                vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
         vn, rem = self.auth.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
@@ -1167,6 +1350,94 @@ class HttpCli(object):
             # print(abspath)
             raise Pebkac(404)
 
+        srv_info = []
+
+        try:
+            if not self.args.nih:
+                srv_info.append(unicode(socket.gethostname()).split(".")[0])
+        except:
+            self.log("#wow #whoa")
+
+        try:
+            # some fuses misbehave
+            if not self.args.nid:
+                if WINDOWS:
+                    bfree = ctypes.c_ulonglong(0)
+                    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
+                        ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+                    )
+                    srv_info.append(humansize(bfree.value) + " free")
+                else:
+                    sv = os.statvfs(abspath)
+                    free = humansize(sv.f_frsize * sv.f_bfree, True)
+                    total = humansize(sv.f_frsize * sv.f_blocks, True)
+
+                    srv_info.append(free + " free")
+                    srv_info.append(total)
+        except:
+            pass
+
+        srv_info = "</span> /// <span>".join(srv_info)
+
+        perms = []
+        if self.readable:
+            perms.append("read")
+        if self.writable:
+            perms.append("write")
+
+        url_suf = self.urlq()
+        is_ls = "ls" in self.uparam
+        ts = ""  # "?{}".format(time.time())
+
+        tpl = "browser"
+        if "b" in self.uparam:
+            tpl = "browser2"
+
+        logues = ["", ""]
+        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
+            fn = os.path.join(abspath, fn)
+            if os.path.exists(fsenc(fn)):
+                with open(fsenc(fn), "rb") as f:
+                    logues[n] = f.read().decode("utf-8")
+
+        ls_ret = {
+            "dirs": [],
+            "files": [],
+            "taglist": [],
+            "srvinf": srv_info,
+            "perms": perms,
+            "logues": logues,
+        }
+        j2a = {
+            "vdir": quotep(self.vpath),
+            "vpnodes": vpnodes,
+            "files": [],
+            "ts": ts,
+            "perms": json.dumps(perms),
+            "taglist": [],
+            "tag_order": [],
+            "have_up2k_idx": ("e2d" in vn.flags),
+            "have_tags_idx": ("e2t" in vn.flags),
+            "have_zip": (not self.args.no_zip),
+            "have_b_u": (self.writable and self.uparam.get("b") == "u"),
+            "url_suf": url_suf,
+            "logues": logues,
+            "title": html_escape(self.vpath, crlf=True),
+            "srv_info": srv_info,
+        }
+        if not self.readable:
+            if is_ls:
+                ret = json.dumps(ls_ret)
+                self.reply(ret.encode("utf-8", "replace"), mime="application/json")
+                return True
+
+            if not os.path.isdir(fsenc(abspath)):
+                raise Pebkac(404)
+
+            html = self.j2(tpl, **j2a)
+            self.reply(html.encode("utf-8", "replace"))
+            return True
+
         if not os.path.isdir(fsenc(abspath)):
             if abspath.endswith(".md") and "raw" not in self.uparam:
                 return self.tx_md(abspath)
@@ -1176,6 +1447,11 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                return self.tx_zip(k, v, vn, rem, [], self.args.ed)
+
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
@@ -1205,8 +1481,6 @@ class HttpCli(object):
         if rem == ".hist":
             hidden = ["up2k."]
 
-        is_ls = "ls" in self.uparam
-
         icur = None
         if "e2t" in vn.flags:
             idx = self.conn.get_u2idx()
@@ -1236,11 +1510,14 @@ class HttpCli(object):
 
             is_dir = stat.S_ISDIR(inf.st_mode)
             if is_dir:
-                margin = "DIR"
                 href += "/"
+                if self.args.no_zip:
+                    margin = "DIR"
+                else:
+                    margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
-                    base, html_escape(hist[fn][2], quote=True), hist[fn][0]
+                    base, html_escape(hist[fn][2], quote=True, crlf=True), hist[fn][0]
                 )
             else:
                 margin = "-"
@@ -1282,98 +1559,38 @@ class HttpCli(object):
                     args = s3enc(idx.mem_cur, rd, fn)
                     r = icur.execute(q, args).fetchone()
 
+                tags = {}
+                f["tags"] = tags
+
                 if not r:
                     continue
 
                 w = r[0][:16]
-                tags = {}
                 q = "select k, v from mt where w = ? and k != 'x'"
                 for k, v in icur.execute(q, (w,)):
                     taglist[k] = True
                     tags[k] = v
 
-                f["tags"] = tags
-
         if icur:
-            taglist = [k for k in self.args.mte.split(",") if k in taglist]
+            taglist = [k for k in vn.flags.get("mte", "").split(",") if k in taglist]
             for f in dirs:
                 f["tags"] = {}
 
-        srv_info = []
-
-        try:
-            if not self.args.nih:
-                srv_info.append(unicode(socket.gethostname()).split(".")[0])
-        except:
-            self.log("#wow #whoa")
-            pass
-
-        try:
-            # some fuses misbehave
-            if not self.args.nid:
-                if WINDOWS:
-                    bfree = ctypes.c_ulonglong(0)
-                    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
-                        ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
-                    )
-                    srv_info.append(humansize(bfree.value) + " free")
-                else:
-                    sv = os.statvfs(abspath)
-                    free = humansize(sv.f_frsize * sv.f_bfree, True)
-                    total = humansize(sv.f_frsize * sv.f_blocks, True)
-
-                    srv_info.append(free + " free")
-                    srv_info.append(total)
-        except:
-            pass
-
-        srv_info = "</span> /// <span>".join(srv_info)
-
-        perms = []
-        if self.readable:
-            perms.append("read")
-        if self.writable:
-            perms.append("write")
-
-        logues = ["", ""]
-        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
-            fn = os.path.join(abspath, fn)
-            if os.path.exists(fsenc(fn)):
-                with open(fsenc(fn), "rb") as f:
-                    logues[n] = f.read().decode("utf-8")
-
         if is_ls:
             [x.pop(k) for k in ["name", "dt"] for y in [dirs, files] for x in y]
-            ret = {
-                "dirs": dirs,
-                "files": files,
-                "srvinf": srv_info,
-                "perms": perms,
-                "logues": logues,
-                "taglist": taglist,
-            }
-            ret = json.dumps(ret)
+            ls_ret["dirs"] = dirs
+            ls_ret["files"] = files
+            ls_ret["taglist"] = taglist
+            ret = json.dumps(ls_ret)
             self.reply(ret.encode("utf-8", "replace"), mime="application/json")
             return True
 
-        ts = ""
-        # ts = "?{}".format(time.time())
+        j2a["files"] = dirs + files
+        j2a["logues"] = logues
+        j2a["taglist"] = taglist
+        if "mte" in vn.flags:
+            j2a["tag_order"] = json.dumps(vn.flags["mte"].split(","))
 
-        dirs.extend(files)
-
-        html = self.conn.tpl_browser.render(
-            vdir=quotep(self.vpath),
-            vpnodes=vpnodes,
-            files=dirs,
-            ts=ts,
-            perms=json.dumps(perms),
-            taglist=taglist,
-            tag_order=json.dumps(self.args.mte.split(",")),
-            have_up2k_idx=("e2d" in vn.flags),
-            have_tags_idx=("e2t" in vn.flags),
-            logues=logues,
-            title=html_escape(self.vpath),
-            srv_info=srv_info,
-        )
+        html = self.j2(tpl, **j2a)
         self.reply(html.encode("utf-8", "replace"))
         return True

copyparty/httpconn.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
 import sys
 import time
@@ -12,23 +13,6 @@ try:
 except:
     HAVE_SSL = False
 
-try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * {} -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-""".format(
-            os.path.basename(sys.executable)
-        )
-    )
-    sys.exit(1)
-
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
@@ -55,16 +39,9 @@ class HttpConn(object):
         self.workload = 0
         self.u2idx = None
         self.log_func = hsrv.log
+        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
         self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
-
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -112,7 +89,9 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
-                self.log(err)
+                if method:
+                    self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 

copyparty/httpsrv.py

@@ -2,10 +2,28 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
 from .__init__ import E, MACOS
 from .httpconn import HttpConn
 from .authsrv import AuthSrv
@@ -30,6 +48,13 @@ class HttpSrv(object):
         self.workload_thr_alive = False
         self.auth = AuthSrv(self.args, self.log)
 
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+        }
+
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
             self.cert_path = cert_path
@@ -38,7 +63,9 @@ class HttpSrv(object):
 
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+        if self.args.log_conn:
+            self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+
         thr = threading.Thread(target=self.thr_client, args=(sck, addr))
         thr.daemon = True
         thr.start()
@@ -66,11 +93,15 @@ class HttpSrv(object):
                 thr.start()
 
         try:
-            self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+
             cli.run()
 
         finally:
-            self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+
             try:
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()

copyparty/mtag.py

@@ -8,7 +8,7 @@ import shutil
 import subprocess as sp
 
 from .__init__ import PY2, WINDOWS
-from .util import fsenc, fsdec
+from .util import fsenc, fsdec, REKOBO_LKEY
 
 if not PY2:
     unicode = str
@@ -151,6 +151,12 @@ class MTag(object):
                 v = v.split("/")[0].strip().lstrip("0")
                 ret[k] = v or 0
 
+        # normalize key notation to rkeobo
+        okey = ret.get("key")
+        if okey:
+            key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
+            ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
+
         return ret
 
     def compare(self, abspath):
@@ -225,7 +231,7 @@ class MTag(object):
         """
         note:
           tags which contain newline will be truncated on first \n,
-          ffmpeg emits \n and spacepads the : to align visually
+          ffprobe emits \n and spacepads the : to align visually
         note:
           the Stream ln always mentions Audio: if audio
           the Stream ln usually has kb/s, is more accurate
@@ -295,7 +301,7 @@ class MTag(object):
                             sec *= 60
                             sec += int(f)
                     except:
-                        self.log("invalid timestr from ffmpeg: [{}]".format(tstr), c=3)
+                        self.log("invalid timestr from ffprobe: [{}]".format(tstr), c=3)
 
                 ret[".dur"] = sec
                 m = ptn_br1.search(ln)
@@ -312,3 +318,30 @@ class MTag(object):
         ret = {k: [0, v] for k, v in ret.items()}
 
         return self.normalize_tags(ret, md)
+
+    def get_bin(self, parsers, abspath):
+        pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
+        pypath = str(os.pathsep.join(pypath))
+        env = os.environ.copy()
+        env["PYTHONPATH"] = pypath
+
+        ret = {}
+        for tagname, (binpath, timeout) in parsers.items():
+            try:
+                cmd = [sys.executable, binpath, abspath]
+                args = {"env": env, "timeout": timeout}
+
+                if WINDOWS:
+                    args["creationflags"] = 0x4000
+                else:
+                    cmd = ["nice"] + cmd
+
+                cmd = [fsenc(x) for x in cmd]
+                v = sp.check_output(cmd, **args).strip()
+                if v:
+                    ret[tagname] = v.decode("utf-8")
+            except:
+                pass
+
+        return ret

copyparty/star.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen)
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/sutil.py

@@ -0,0 +1,25 @@
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/szip.py

@@ -0,0 +1,271 @@
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, ok="/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -68,23 +68,29 @@ class TcpSrv(object):
             self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
 
         while True:
-            self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
+
             if self.num_clients.v >= self.args.nc:
                 time.sleep(0.1)
                 continue
 
-            self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
+
             ready, _, _ = select.select(self.srv, [], [])
             for srv in ready:
                 sck, addr = srv.accept()
                 sip, sport = srv.getsockname()
-                self.log(
-                    "%s %s" % addr,
-                    "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
-                        "-" * 3, sip, sport % 8, sport
-                    ),
-                    c="1;30",
-                )
+                if self.args.log_conn:
+                    self.log(
+                        "%s %s" % addr,
+                        "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                            "-" * 3, sip, sport % 8, sport
+                        ),
+                        c="1;30",
+                    )
+
                 self.num_clients.add()
                 self.hub.broker.put(False, "httpconn", sck, addr)
 

copyparty/u2idx.py

@@ -1,10 +1,13 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
+import time
+import threading
 from datetime import datetime
 
-from .util import u8safe
+from .util import u8safe, s3dec, html_escape, Pebkac
 from .up2k import up2k_wark_from_hashlist
 
 
@@ -19,6 +22,7 @@ class U2idx(object):
     def __init__(self, args, log_func):
         self.args = args
         self.log_func = log_func
+        self.timeout = args.srch_time
 
         if not HAVE_SQLITE3:
             self.log("could not load sqlite3; searchign wqill be disabled")
@@ -28,6 +32,9 @@ class U2idx(object):
         self.mem_cur = sqlite3.connect(":memory:")
         self.mem_cur.execute(r"create table a (b text)")
 
+        self.p_end = None
+        self.p_dur = 0
+
     def log(self, msg, c=0):
         self.log_func("u2idx", msg, c)
 
@@ -43,7 +50,10 @@ class U2idx(object):
         uq = "substr(w,1,16) = ? and w = ?"
         uv = [wark[:16], wark]
 
-        return self.run_query(vols, uq, uv, "", [])[0]
+        try:
+            return self.run_query(vols, uq, uv, {})[0]
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
 
     def get_cur(self, ptop):
         cur = self.cur.get(ptop)
@@ -73,17 +83,64 @@ class U2idx(object):
 
         uq, uv = _sqlize(qobj)
 
-        tq = ""
-        tv = []
         qobj = {}
         if "tags" in body:
             _conv_txt(qobj, body, "tags", "mt.v")
-            tq, tv = _sqlize(qobj)
 
-        return self.run_query(vols, uq, uv, tq, tv)
+        if "adv" in body:
+            _conv_adv(qobj, body, "adv")
 
-    def run_query(self, vols, uq, uv, tq, tv):
-        self.log("qs: {} {} ,  {} {}".format(uq, repr(uv), tq, repr(tv)))
+        try:
+            return self.run_query(vols, uq, uv, qobj)
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
+
+    def run_query(self, vols, uq, uv, targs):
+        self.log("qs: {} {} ,  {}".format(uq, repr(uv), repr(targs)))
+
+        done_flag = []
+        self.active_id = "{:.6f}_{}".format(
+            time.time(), threading.current_thread().ident
+        )
+        thr = threading.Thread(
+            target=self.terminator,
+            args=(
+                self.active_id,
+                done_flag,
+            ),
+        )
+        thr.daemon = True
+        thr.start()
+
+        if not targs:
+            if not uq:
+                q = "select * from up"
+                v = ()
+            else:
+                q = "select * from up where " + uq
+                v = tuple(uv)
+        else:
+            q = "select up.* from up"
+            keycmp = "substr(up.w,1,16)"
+            where = []
+            v = []
+            ctr = 0
+            for tq, tv in sorted(targs.items()):
+                ctr += 1
+                tq = tq.split("\n")[0]
+                keycmp2 = "mt{}.w".format(ctr)
+                q += " inner join mt mt{} on {} = {}".format(ctr, keycmp, keycmp2)
+                keycmp = keycmp2
+                where.append(tq.replace("mt.", keycmp[:-1]))
+                v.append(tv)
+
+            if uq:
+                where.append(uq)
+                v.extend(uv)
+
+            q += " where " + (" and ".join(where))
+
+        # self.log("q2: {} {}".format(q, repr(v)))
 
         ret = []
         lim = 1000
@@ -93,18 +150,7 @@ class U2idx(object):
             if not cur:
                 continue
 
-            if not tq:
-                if not uq:
-                    q = "select * from up"
-                    v = ()
-                else:
-                    q = "select * from up where " + uq
-                    v = tuple(uv)
-            else:
-                # naive assumption: tags first
-                q = "select up.* from up inner join mt on substr(up.w,1,16) = mt.w where {}"
-                q = q.format(" and ".join([tq, uq]) if uq else tq)
-                v = tuple(tv + uv)
+            self.active_cur = cur
 
             sret = []
             c = cur.execute(q, v)
@@ -124,17 +170,35 @@ class U2idx(object):
                 w = hit["w"]
                 del hit["w"]
                 tags = {}
-                q = "select k, v from mt where w = ? and k != 'x'"
-                for k, v in cur.execute(q, (w,)):
+                q2 = "select k, v from mt where w = ? and k != 'x'"
+                for k, v2 in cur.execute(q2, (w,)):
                     taglist[k] = True
-                    tags[k] = v
+                    tags[k] = v2
 
                 hit["tags"] = tags
 
             ret.extend(sret)
 
+        done_flag.append(True)
+        self.active_id = None
+
+        # undupe hits from multiple metadata keys
+        if len(ret) > 1:
+            ret = [ret[0]] + [
+                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+            ]
+
         return ret, list(taglist.keys())
 
+    def terminator(self, identifier, done_flag):
+        for _ in range(self.timeout):
+            time.sleep(1)
+            if done_flag:
+                return
+
+        if identifier == self.active_id:
+            self.active_cur.connection.interrupt()
+
 
 def _open(ptop):
     db_path = os.path.join(ptop, ".hist", "up2k.db")
@@ -190,6 +254,23 @@ def _conv_txt(q, body, k, sql):
         q[qk + "\n" + v] = u8safe(v)
 
 
+def _conv_adv(q, body, k):
+    ptn = re.compile(r"^(\.?[a-z]+) *(==?|!=|<=?|>=?) *(.*)$")
+
+    parts = body[k].split(" ")
+    parts = [x.strip() for x in parts if x.strip()]
+
+    for part in parts:
+        m = ptn.match(part)
+        if not m:
+            p = html_escape(part)
+            raise Pebkac(400, "invalid argument [" + p + "]")
+
+        k, op, v = m.groups()
+        qk = "mt.k = '{}' and mt.v {} ?".format(k, op)
+        q[qk + "\n" + v] = u8safe(v)
+
+
 def _sqlize(qobj):
     keys = []
     values = []

copyparty/up2k.py

@@ -13,9 +13,10 @@ import base64
 import hashlib
 import threading
 import traceback
+import subprocess as sp
 from copy import deepcopy
 
-from .__init__ import WINDOWS
+from .__init__ import WINDOWS, ANYWIN
 from .util import (
     Pebkac,
     Queue,
@@ -28,6 +29,7 @@ from .util import (
     s3enc,
     s3dec,
     statdir,
+    s2hms,
 )
 from .mtag import MTag
 from .authsrv import AuthSrv
@@ -64,7 +66,7 @@ class Up2k(object):
         self.flags = {}
         self.cur = {}
         self.mtag = None
-        self.n_mtag_tags_added = -1
+        self.pending_tags = None
 
         self.mem_cur = None
         self.sqlite_ver = None
@@ -77,7 +79,7 @@ class Up2k(object):
             if self.sqlite_ver < (3, 9):
                 self.no_expr_idx = True
 
-        if WINDOWS:
+        if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
             thr = threading.Thread(target=self._lastmodder)
@@ -99,14 +101,19 @@ class Up2k(object):
             thr.daemon = True
             thr.start()
 
-            thr = threading.Thread(target=self._tagger)
-            thr.daemon = True
-            thr.start()
-
             thr = threading.Thread(target=self._hasher)
             thr.daemon = True
             thr.start()
 
+            if self.mtag:
+                thr = threading.Thread(target=self._tagger)
+                thr.daemon = True
+                thr.start()
+
+                thr = threading.Thread(target=self._run_all_mtp)
+                thr.daemon = True
+                thr.start()
+
     def log(self, msg, c=0):
         self.log_func("up2k", msg + "\033[K", c)
 
@@ -219,6 +226,16 @@ class Up2k(object):
 
             _, flags = self._expr_idx_filter(flags)
 
+            ft = "\033[0;32m{}{:.0}"
+            ff = "\033[0;35m{}{:.0}"
+            fv = "\033[0;36m{}:\033[1;30m{}"
+            a = [
+                (ft if v is True else ff if v is False else fv).format(k, str(v))
+                for k, v in flags.items()
+            ]
+            if a:
+                self.log(" ".join(sorted(a)) + "\033[0m")
+
             reg = {}
             path = os.path.join(ptop, ".hist", "up2k.snap")
             if "e2d" in flags and os.path.exists(path):
@@ -272,9 +289,12 @@ class Up2k(object):
         dbw = [reg[0], 0, time.time()]
         self.pp.n = next(dbw[0].execute("select count(w) from up"))[0]
 
-        # can be symlink so don't `and d.startswith(top)``
-        excl = set([d.realpath for d in all_vols if d != vol])
-        n_add = self._build_dir(dbw, top, excl, top)
+        excl = [
+            vol.realpath + "/" + d.vpath[len(vol.vpath) :].lstrip("/")
+            for d in all_vols
+            if d != vol and (d.vpath.startswith(vol.vpath + "/") or not vol.vpath)
+        ]
+        n_add = self._build_dir(dbw, top, set(excl), top)
         n_rm = self._drop_lost(dbw[0], top)
         if dbw[1]:
             self.log("commit {} new files".format(dbw[1]))
@@ -435,18 +455,7 @@ class Up2k(object):
 
             mpool = False
             if self.mtag.prefer_mt and not self.args.no_mtag_mt:
-                # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
-                # both do crazy runahead so lets reinvent another wheel
-                nw = os.cpu_count() if hasattr(os, "cpu_count") else 4
-                if self.n_mtag_tags_added == -1:
-                    self.log("using {}x {}".format(nw, self.mtag.backend))
-                    self.n_mtag_tags_added = 0
-
-                mpool = Queue(nw)
-                for _ in range(nw):
-                    thr = threading.Thread(target=self._tag_thr, args=(mpool,))
-                    thr.daemon = True
-                    thr.start()
+                mpool = self._start_mpool()
 
             c2 = cur.connection.cursor()
             c3 = cur.connection.cursor()
@@ -457,19 +466,21 @@ class Up2k(object):
                 if c2.execute(q, (w[:16],)).fetchone():
                     continue
 
+                if "mtp" in flags:
+                    q = "insert into mt values (?,'t:mtp','a')"
+                    c2.execute(q, (w[:16],))
+
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
 
                 abspath = os.path.join(ptop, rd, fn)
                 self.pp.msg = "c{} {}".format(n_left, abspath)
-                args = c3, entags, w, abspath
+                args = [entags, w, abspath]
                 if not mpool:
-                    n_tags = self._tag_file(*args)
+                    n_tags = self._tag_file(c3, *args)
                 else:
-                    mpool.put(args)
-                    with self.mutex:
-                        n_tags = self.n_mtag_tags_added
-                        self.n_mtag_tags_added = 0
+                    mpool.put(["mtag"] + args)
+                    n_tags = len(self._flush_mpool(c3))
 
                 n_add += n_tags
                 n_buf += n_tags
@@ -481,17 +492,240 @@ class Up2k(object):
                     last_write = time.time()
                     n_buf = 0
 
-            if mpool:
-                for _ in range(mpool.maxsize):
-                    mpool.put(None)
-
-                mpool.join()
+            self._stop_mpool(mpool, c3)
 
             c3.close()
             c2.close()
 
         return n_add, n_rm, True
 
+    def _flush_mpool(self, wcur):
+        with self.mutex:
+            ret = []
+            for x in self.pending_tags:
+                self._tag_file(wcur, *x)
+                ret.append(x[1])
+
+            self.pending_tags = []
+            return ret
+
+    def _run_all_mtp(self):
+        t0 = time.time()
+        self.mtp_audio = {}
+        self.mtp_force = {}
+        self.mtp_parsers = {}
+        for ptop, flags in self.flags.items():
+            if "mtp" in flags:
+                self._run_one_mtp(ptop)
+
+        td = time.time() - t0
+        msg = "mtp finished in {:.2f} sec ({})"
+        self.log(msg.format(td, s2hms(td, True)))
+
+    def _run_one_mtp(self, ptop):
+        db_path = os.path.join(ptop, ".hist", "up2k.db")
+        sz0 = os.path.getsize(db_path) // 1024
+
+        entags = self.entags[ptop]
+
+        audio = {}  # [r]equire [n]ot [d]ontcare
+        force = {}  # bool
+        timeout = {}  # int
+        parsers = {}
+        for parser in self.flags[ptop]["mtp"]:
+            orig = parser
+            tag, parser = parser.split("=", 1)
+            if tag not in entags:
+                continue
+
+            audio[tag] = "y"
+
+            while True:
+                try:
+                    bp = os.path.expanduser(parser)
+                    if os.path.exists(bp):
+                        parsers[tag] = [bp, timeout.get(tag, 30)]
+                        break
+                except:
+                    pass
+
+                try:
+                    arg, parser = parser.split(",", 1)
+                    arg = arg.lower()
+
+                    if arg.startswith("a"):
+                        audio[tag] = arg[1:]
+                        continue
+
+                    if arg == "f":
+                        force[tag] = True
+                        continue
+
+                    if arg.startswith("t"):
+                        timeout[tag] = int(arg[1:])
+                        continue
+
+                    raise Exception()
+
+                except:
+                    self.log("invalid argument: " + orig, 1)
+                    return
+
+        # todo audio/force => parser attributes
+        self.mtp_audio[ptop] = audio
+        self.mtp_force[ptop] = force
+        self.mtp_parsers[ptop] = parsers
+
+        q = "select count(w) from mt where k = 't:mtp'"
+        with self.mutex:
+            cur = self.cur[ptop]
+            cur = cur.connection.cursor()
+            wcur = cur.connection.cursor()
+            n_left = cur.execute(q).fetchone()[0]
+
+        mpool = self._start_mpool()
+        batch_sz = mpool.maxsize * 3
+        t_prev = time.time()
+        n_prev = n_left
+        n_done = 0
+        to_delete = {}
+        in_progress = {}
+        while True:
+            with self.mutex:
+                q = "select w from mt where k = 't:mtp' limit ?"
+                warks = cur.execute(q, (batch_sz,)).fetchall()
+                warks = [x[0] for x in warks]
+                jobs = []
+                for w in warks:
+                    q = "select rd, fn from up where substr(w,1,16)=? limit 1"
+                    rd, fn = cur.execute(q, (w,)).fetchone()
+                    rd, fn = s3dec(rd, fn)
+                    abspath = os.path.join(ptop, rd, fn)
+
+                    q = "select k from mt where w = ?"
+                    have = cur.execute(q, (w,)).fetchall()
+                    have = [x[0] for x in have]
+
+                    parsers = self._get_parsers(ptop, have)
+                    if not parsers:
+                        to_delete[w] = True
+                        n_left -= 1
+                        continue
+
+                    if w in in_progress:
+                        continue
+
+                    jobs.append([parsers, None, w, abspath])
+                    in_progress[w] = True
+
+            done = self._flush_mpool(wcur)
+
+            with self.mutex:
+                for w in done:
+                    to_delete[w] = True
+                    in_progress.pop(w)
+                    n_done += 1
+
+                for w in to_delete.keys():
+                    q = "delete from mt where w = ? and k = 't:mtp'"
+                    cur.execute(q, (w,))
+
+                to_delete = {}
+
+            if not warks:
+                break
+
+            if not jobs:
+                continue
+
+            try:
+                now = time.time()
+                s = ((now - t_prev) / (n_prev - n_left)) * n_left
+                h, s = divmod(s, 3600)
+                m, s = divmod(s, 60)
+                n_prev = n_left
+                t_prev = now
+            except:
+                h = 1
+                m = 1
+
+            msg = "mtp: {} done, {} left, eta {}h {:02d}m"
+            with self.mutex:
+                msg = msg.format(n_done, n_left, int(h), int(m))
+                self.log(msg, c=6)
+
+            for j in jobs:
+                n_left -= 1
+                mpool.put(j)
+
+            with self.mutex:
+                cur.connection.commit()
+
+        done = self._stop_mpool(mpool, wcur)
+        with self.mutex:
+            for w in done:
+                q = "delete from mt where w = ? and k = 't:mtp'"
+                cur.execute(q, (w,))
+
+            cur.connection.commit()
+            if n_done:
+                self.vac(cur, db_path, n_done, 0, sz0)
+
+            wcur.close()
+            cur.close()
+
+    def _get_parsers(self, ptop, have):
+        try:
+            all_parsers = self.mtp_parsers[ptop]
+        except:
+            return {}
+
+        audio = self.mtp_audio[ptop]
+        force = self.mtp_force[ptop]
+        entags = self.entags[ptop]
+        parsers = {}
+        for k, v in all_parsers.items():
+            if ".dur" in entags:
+                if ".dur" in have:
+                    # is audio, require non-audio?
+                    if audio[k] == "n":
+                        continue
+                # is not audio, require audio?
+                elif audio[k] == "y":
+                    continue
+
+            parsers[k] = v
+
+        parsers = {k: v for k, v in parsers.items() if k in force or k not in have}
+        return parsers
+
+    def _start_mpool(self):
+        # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
+        # both do crazy runahead so lets reinvent another wheel
+        nw = os.cpu_count() if hasattr(os, "cpu_count") else 4
+        if self.pending_tags is None:
+            self.log("using {}x {}".format(nw, self.mtag.backend))
+            self.pending_tags = []
+
+        mpool = Queue(nw)
+        for _ in range(nw):
+            thr = threading.Thread(target=self._tag_thr, args=(mpool,))
+            thr.daemon = True
+            thr.start()
+
+        return mpool
+
+    def _stop_mpool(self, mpool, wcur):
+        if not mpool:
+            return
+
+        for _ in range(mpool.maxsize):
+            mpool.put(None)
+
+        mpool.join()
+        done = self._flush_mpool(wcur)
+        return done
+
     def _tag_thr(self, q):
         while True:
             task = q.get()
@@ -500,24 +734,47 @@ class Up2k(object):
                 return
 
             try:
-                write_cur, entags, wark, abspath = task
-                tags = self.mtag.get(abspath)
+                parser, entags, wark, abspath = task
+                if parser == "mtag":
+                    tags = self.mtag.get(abspath)
+                else:
+                    tags = self.mtag.get_bin(parser, abspath)
+                    vtags = [
+                        "\033[36m{} \033[33m{}".format(k, v) for k, v in tags.items()
+                    ]
+                    self.log("{}\033[0m [{}]".format(" ".join(vtags), abspath))
+
                 with self.mutex:
-                    n = self._tag_file(write_cur, entags, wark, abspath, tags)
-                    self.n_mtag_tags_added += n
+                    self.pending_tags.append([entags, wark, abspath, tags])
             except:
                 ex = traceback.format_exc()
+                if parser == "mtag":
+                    parser = self.mtag.backend
+
                 msg = "{} failed to read tags from {}:\n{}"
-                self.log(msg.format(self.mtag.backend, abspath, ex), c=3)
+                self.log(msg.format(parser, abspath, ex), c=3)
 
             q.task_done()
 
     def _tag_file(self, write_cur, entags, wark, abspath, tags=None):
-        tags = tags or self.mtag.get(abspath)
-        tags = {k: v for k, v in tags.items() if k in entags}
+        if tags is None:
+            tags = self.mtag.get(abspath)
+
+        if entags:
+            tags = {k: v for k, v in tags.items() if k in entags}
+            if not tags:
+                # indicate scanned without tags
+                tags = {"x": 0}
+
         if not tags:
-            # indicate scanned without tags
-            tags = {"x": 0}
+            return 0
+
+        for k in tags.keys():
+            q = "delete from mt where w = ? and ({})".format(
+                " or ".join(["k = ?"] * len(tags))
+            )
+            args = [wark[:16]] + list(tags.keys())
+            write_cur.execute(q, tuple(args))
 
         ret = 0
         for k, v in tags.items():
@@ -529,6 +786,7 @@ class Up2k(object):
 
     def _orz(self, db_path):
         return sqlite3.connect(db_path, check_same_thread=False).cursor()
+        # x.set_trace_callback(trace)
 
     def _open_db(self, db_path):
         existed = os.path.exists(db_path)
@@ -665,7 +923,7 @@ class Up2k(object):
             if cj["ptop"] not in self.registry:
                 raise Pebkac(410, "location unavailable")
 
-        cj["name"] = sanitize_fn(cj["name"])
+        cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
         wark = self._get_wark(cj)
         now = time.time()
@@ -831,6 +1089,8 @@ class Up2k(object):
         with self.mutex:
             job = self.registry[ptop].get(wark, None)
             if not job:
+                known = " ".join([x for x in self.registry[ptop].keys()])
+                self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
@@ -870,8 +1130,9 @@ class Up2k(object):
 
             atomic_move(src, dst)
 
-            if WINDOWS:
-                self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))])
+            if ANYWIN:
+                a = [dst, job["size"], (int(time.time()), int(job["lmod"]))]
+                self.lastmod_q.put(a)
 
             # legit api sware 2 me mum
             if self.idx_wark(
@@ -969,9 +1230,23 @@ class Up2k(object):
         #    raise Exception("aaa")
 
         tnam = job["name"] + ".PARTIAL"
+        if self.args.dotpart:
+            tnam = "." + tnam
+
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f:
             f, job["tnam"] = f["orz"]
+            if (
+                ANYWIN
+                and self.args.sparse
+                and self.args.sparse * 1024 * 1024 <= job["size"]
+            ):
+                fp = os.path.join(pdir, job["tnam"])
+                try:
+                    sp.check_call(["fsutil", "sparse", "setflag", fp])
+                except:
+                    self.log("could not sparse [{}]".format(fp), 3)
+
             f.seek(job["size"] - 1)
             f.write(b"e")
 
@@ -983,13 +1258,19 @@ class Up2k(object):
 
             # self.log("lmod: got {}".format(len(ready)))
             time.sleep(5)
-            for path, times in ready:
+            for path, sz, times in ready:
                 self.log("lmod: setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:
                     self.log("lmod: failed to utime ({}, {})".format(path, times))
 
+                if self.args.sparse and self.args.sparse * 1024 * 1024 <= sz:
+                    try:
+                        sp.check_call(["fsutil", "sparse", "setflag", path, "0"])
+                    except:
+                        self.log("could not unsparse [{}]".format(path), 3)
+
     def _snapshot(self):
         persist_interval = 30  # persist unfinished uploads index every 30 sec
         discard_interval = 21600  # drop unfinished uploads after 6 hours inactivity
@@ -1053,24 +1334,33 @@ class Up2k(object):
     def _tagger(self):
         while True:
             ptop, wark, rd, fn = self.tagq.get()
+            if "e2t" not in self.flags[ptop]:
+                continue
+
             abspath = os.path.join(ptop, rd, fn)
-            self.log("tagging " + abspath)
+            tags = self.mtag.get(abspath)
+            ntags1 = len(tags)
+            parsers = self._get_parsers(ptop, tags)
+            if parsers:
+                tags.update(self.mtag.get_bin(parsers, abspath))
+
             with self.mutex:
                 cur = self.cur[ptop]
                 if not cur:
                     self.log("no cursor to write tags with??", c=1)
                     continue
 
+                # TODO is undef if vol 404 on startup
                 entags = self.entags[ptop]
                 if not entags:
                     self.log("no entags okay.jpg", c=3)
                     continue
 
-                if "e2t" in self.flags[ptop]:
-                    self._tag_file(cur, entags, wark, abspath)
-
+                self._tag_file(cur, entags, wark, abspath, tags)
                 cur.connection.commit()
 
+            self.log("tagged {} ({}+{})".format(abspath, ntags1, len(tags) - ntags1))
+
     def _hasher(self):
         while True:
             ptop, rd, fn = self.hashq.get()

copyparty/util.py

@@ -10,12 +10,13 @@ import select
 import struct
 import hashlib
 import platform
+import traceback
 import threading
 import mimetypes
 import contextlib
 import subprocess as sp  # nosec
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -48,6 +49,7 @@ HTTPCODE = {
     200: "OK",
     204: "No Content",
     206: "Partial Content",
+    302: "Found",
     304: "Not Modified",
     400: "Bad Request",
     403: "Forbidden",
@@ -56,11 +58,58 @@ HTTPCODE = {
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
 }
 
 
+IMPLICATIONS = [
+    ["e2dsa", "e2ds"],
+    ["e2ds", "e2d"],
+    ["e2tsr", "e2ts"],
+    ["e2ts", "e2t"],
+    ["e2t", "e2d"],
+]
+
+
+REKOBO_KEY = {
+    v: ln.split(" ", 1)[0]
+    for ln in """
+1B 6d B
+2B 7d Gb F#
+3B 8d Db C#
+4B 9d Ab G#
+5B 10d Eb D#
+6B 11d Bb A#
+7B 12d F
+8B 1d C
+9B 2d G
+10B 3d D
+11B 4d A
+12B 5d E
+1A 6m Abm G#m
+2A 7m Ebm D#m
+3A 8m Bbm A#m
+4A 9m Fm
+5A 10m Cm
+6A 11m Gm
+7A 12m Dm
+8A 1m Am
+9A 2m Em
+10A 3m Bm
+11A 4m Gbm F#m
+12A 5m Dbm C#m
+""".strip().split(
+        "\n"
+    )
+    for v in ln.strip().split(" ")[1:]
+    if v
+}
+
+REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
+
+
 class Counter(object):
     def __init__(self, v=0):
         self.v = v
@@ -139,6 +188,31 @@ def nuprint(msg):
     uprint("{}\n".format(msg))
 
 
+def rice_tid():
+    tid = threading.current_thread().ident
+    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
+
+
+def trace(*args, **kwargs):
+    t = time.time()
+    stack = "".join(
+        "\033[36m{}\033[33m{}".format(x[0].split(os.sep)[-1][:-3], x[1])
+        for x in traceback.extract_stack()[3:-1]
+    )
+    parts = ["{:.6f}".format(t), rice_tid(), stack]
+
+    if args:
+        parts.append(repr(args))
+
+    if kwargs:
+        parts.append(repr(kwargs))
+
+    msg = "\033[0m ".join(parts)
+    # _tracebuf.append(msg)
+    nuprint(msg)
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -477,6 +551,16 @@ def get_spd(nbyte, t0, t=None):
     return "{} \033[0m{}/s\033[0m".format(s1, s2)
 
 
+def s2hms(s, optional_h=False):
+    s = int(s)
+    h, s = divmod(s, 3600)
+    m, s = divmod(s, 60)
+    if not h and optional_h:
+        return "{}:{:02}".format(m, s)
+
+    return "{}:{:02}:{:02}".format(h, m, s)
+
+
 def undot(path):
     ret = []
     for node in path.split("/"):
@@ -493,11 +577,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
-    fn = fn.replace("\\", "/").split("/")[-1]
+def sanitize_fn(fn, ok="", bad=[]):
+    if "/" not in ok:
+        fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -507,15 +592,16 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
-            fn = fn.replace(bad, good)
+        ]
+        for a, b in [x for x in remap if x[0] not in ok]:
+            fn = fn.replace(a, b)
 
-        bad = ["con", "prn", "aux", "nul"]
+        bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):
             bad += "com{0} lpt{0}".format(n).split(" ")
 
-        if fn.lower() in bad:
-            fn = "_" + fn
+    if fn.lower() in bad:
+        fn = "_" + fn
 
     return fn.strip()
 
@@ -531,17 +617,24 @@ def exclude_dotfiles(filepaths):
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 
@@ -697,6 +790,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()
@@ -837,7 +940,11 @@ def chkcmd(*argv):
 def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
-        return struct.unpack(b"I", f.read(4))[0]
+        rv = f.read(4)
+        try:
+            return struct.unpack(b"I", rv)[0]
+        except:
+            return struct.unpack("I", rv)[0]
 
 
 def py_desc():
@@ -847,7 +954,11 @@ def py_desc():
     if ofs > 0:
         py_ver = py_ver[:ofs]
 
-    bitness = struct.calcsize(b"P") * 8
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
     host_os = platform.system()
     compiler = platform.python_compiler()
 

copyparty/web/browser.css

@@ -6,7 +6,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -68,7 +68,7 @@ a, #files tbody div a:last-child {
 	color: #999;
 	font-weight: normal;
 }
-#files tr+tr:hover {
+#files tr:hover {
 	background: #1c1c1c;
 }
 #files thead th {
@@ -90,8 +90,6 @@ a, #files tbody div a:last-child {
 #files td {
 	margin: 0;
 	padding: 0 .5em;
-}
-#files td {
 	border-bottom: 1px solid #111;
 }
 #files td+td+td {
@@ -182,6 +180,21 @@ a, #files tbody div a:last-child {
 	color: #840;
 	text-shadow: 0 0 .3em #b80;
 }
+#files tbody tr.sel td {
+	color: #fff;
+	background: #925;
+	border-color: #c37;
+}
+#files tr.sel a {
+	color: #fff;
+}
+#files tr.sel a.play {
+	color: #fc5;
+}
+#files tr.sel a.play.act {
+	color: #fff;
+	text-shadow: 0 0 1px #fff;
+}
 #blocked {
 	position: fixed;
 	top: 0;
@@ -238,7 +251,7 @@ a, #files tbody div a:last-child {
 	height: 100%;
 	background: #3c3c3c;
 }
-#wtoggle {
+#wtico {
 	cursor: url(/.cpr/dd/1.png), pointer;
 	animation: cursor 500ms infinite;
 }
@@ -268,6 +281,52 @@ a, #files tbody div a:last-child {
 	padding: .2em 0 0 .07em;
 	color: #fff;
 }
+#wzip, #wnp {
+	display: none;
+	margin-right: .3em;
+	padding-right: .3em;
+	border-right: .1em solid #555;
+}
+#wnp a {
+	position: relative;
+	font-size: .47em;
+	margin: 0 .1em;
+	top: -.4em;
+}
+#wnp a+a {
+	margin-left: .33em;
+}
+#wtoggle,
+#wtoggle * {
+	line-height: 1em;
+}
+#wtoggle.np {
+	width: 5.5em;
+}
+#wtoggle.sel {
+	width: 6.4em;
+}
+#wtoggle.sel #wzip,
+#wtoggle.np #wnp {
+	display: inline-block;
+}
+#wtoggle.sel.np #wnp {
+	display: none;
+}
+#wzip a {
+	font-size: .4em;
+	padding: 0 .3em;
+	margin: -.3em .2em;
+	position: relative;
+	display: inline-block;
+}
+#wzip a+a {
+	margin-left: .8em;
+}
+#wtoggle.sel #wzip #selzip {
+	top: -.6em;
+	padding: .4em .3em;
+}
 #barpos,
 #barbuf {
 	position: absolute;
@@ -311,10 +370,10 @@ a, #files tbody div a:last-child {
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
-@media (min-width: 90em) {
+@media (min-width: 80em) {
 	#barpos,
 	#barbuf {
-		width: calc(100% - 24em);
+		width: calc(100% - 21em);
 		left: 9.8em;
 		top: .7em;
 		height: 1.6em;
@@ -324,6 +383,9 @@ a, #files tbody div a:last-child {
 		bottom: -3.2em;
 		height: 3.2em;
 	}
+	#pvol {
+		max-width: 9em;
+	}
 }
 
 
@@ -375,6 +437,7 @@ a, #files tbody div a:last-child {
 	padding: .3em .6em;
 	border-radius: .3em;
 	border-width: .15em 0;
+	white-space: nowrap;
 }
 .opbox {
 	background: #2d2d2d;
@@ -405,7 +468,7 @@ input[type="checkbox"]:checked+label {
 
 
 
-#op_search table {
+#srch_form {
 	border: 1px solid #3a3a3a;
 	box-shadow: 0 0 1em #222 inset;
 	background: #2d2d2d;
@@ -414,14 +477,25 @@ input[type="checkbox"]:checked+label {
 	margin-bottom: 0;
 	padding: 0 .5em .5em 0;
 }
+#srch_form table {
+	display: inline-block;
+}
 #srch_form td {
 	padding: .6em .6em;
 }
+#srch_form td:first-child {
+	width: 3em;
+	padding-right: .2em;
+	text-align: right;
+}
 #op_search input {
 	margin: 0;
 }
 #srch_q {
 	white-space: pre;
+	color: #f80;
+	height: 1em;
+	margin: .2em 0 -1em 1.6em;
 }
 #files td div span {
 	color: #fff;
@@ -444,19 +518,15 @@ input[type="checkbox"]:checked+label {
 	border-collapse: collapse;
 	width: 100%;
 }
-#files td div a:last-child {
-	width: 100%;
-}
 #wrap {
 	margin-top: 2em;
 }
 #tree {
 	display: none;
-	position: fixed;
+	position: absolute;
 	left: 0;
 	bottom: 0;
 	top: 7em;
-	padding-top: .2em;
 	overflow-y: auto;
 	-ms-scroll-chaining: none;
 	overscroll-behavior-y: none;
@@ -465,9 +535,7 @@ input[type="checkbox"]:checked+label {
 #thx_ff {
 	padding: 5em 0;
 }
-#tree::-webkit-scrollbar-track {
-	background: #333;
-}
+#tree::-webkit-scrollbar-track,
 #tree::-webkit-scrollbar {
 	background: #333;
 }
@@ -482,6 +550,7 @@ input[type="checkbox"]:checked+label {
 	left: -1.7em;
 	width: calc(100% + 1.3em);
 }
+.tglbtn,
 #tree>a+a {
 	padding: .2em .4em;
 	font-size: 1.2em;
@@ -492,9 +561,11 @@ input[type="checkbox"]:checked+label {
 	position: relative;
 	top: -.2em;
 }
+.tglbtn:hover,
 #tree>a+a:hover {
 	background: #805;
 }
+.tglbtn.on,
 #tree>a+a.on {
 	background: #fc4;
 	color: #400;
@@ -503,6 +574,7 @@ input[type="checkbox"]:checked+label {
 #detree {
 	padding: .3em .5em;
 	font-size: 1.5em;
+	line-height: 1.5em;
 }
 #tree ul,
 #tree li {
@@ -584,7 +656,8 @@ input[type="checkbox"]:checked+label {
 #files td.min a {
 	display: none;
 }
-#files tr.play td {
+#files tr.play td,
+#files tr.play div a {
 	background: #fc4;
 	border-color: transparent;
 	color: #400;
@@ -601,7 +674,10 @@ input[type="checkbox"]:checked+label {
 	max-width: none;
 	margin-right: 1.5em;
 }
-#key_notation>span {
+#op_cfg>div>a {
+	line-height: 2em;
+}
+#op_cfg>div>span {
 	display: inline-block;
 	padding: .2em .4em;
 }
@@ -624,6 +700,9 @@ input[type="checkbox"]:checked+label {
 	top: 6em;
 	right: 1.5em;
 }
+#ops:hover #opdesc.off {
+	display: none;
+}
 #opdesc code {
 	background: #3c3c3c;
 	padding: .2em .3em;
@@ -631,4 +710,200 @@ input[type="checkbox"]:checked+label {
 	border-radius: .3em;
 	font-family: monospace, monospace;
 	line-height: 2em;
+}
+#pvol,
+#barbuf,
+#barpos,
+#u2conf label {
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+}
+
+
+
+
+
+html.light {
+	color: #333;
+	background: #eee;
+	text-shadow: none;
+}
+html.light #ops,
+html.light .opbox,
+html.light #srch_form {
+	background: #f7f7f7;
+	box-shadow: 0 0 .3em #ddd;
+	border-color: #f7f7f7;
+}
+html.light #ops a.act {
+	box-shadow: 0 .2em .2em #ccc;
+	background: #fff;
+	border-color: #07a;
+	padding-top: .4em;
+}
+html.light #op_cfg h3 {
+	border-color: #ccc;
+}
+html.light .tglbtn,
+html.light #tree > a + a {
+	color: #666;
+	background: #ddd;
+	box-shadow: none;
+}
+html.light .tglbtn:hover,
+html.light #tree > a + a:hover {
+	background: #caf;
+}
+html.light .tglbtn.on,
+html.light #tree > a + a.on {
+	background: #4a0;
+	color: #fff;
+}
+html.light #srv_info {
+	color: #c83;
+	text-shadow: 1px 1px 0 #fff;
+}
+html.light #srv_info span {
+	color: #000;
+}
+html.light #treeul a+a {
+	background: inherit;
+	color: #06a;
+}
+html.light #treeul a.hl {
+	background: #07a;
+	color: #fff;
+}
+html.light #tree li {
+	border-color: #ddd #fff #f7f7f7 #fff;
+}
+html.light #tree ul {
+	border-color: #ccc;
+}
+html.light a,
+html.light #ops a,
+html.light #files tbody div a:last-child {
+	color: #06a;
+}
+html.light #files tbody {
+	background: #f7f7f7;
+}
+html.light #files {
+	box-shadow: 0 0 .3em #ccc;
+}
+html.light #files thead th {
+	background: #eee;
+}
+html.light #files tr td {
+	border-top: 1px solid #ddd;
+}
+html.light #files td {
+	border-bottom: 1px solid #f7f7f7;
+}
+html.light #files tbody tr:last-child td {
+	border-bottom: .2em solid #ccc;
+}
+html.light #files td:nth-child(2n) {
+	color: #d38;
+}
+html.light #files tr:hover td {
+	background: #fff;
+}
+html.light #files tbody a.play {
+	color: #c0f;
+}
+html.light tr.play td {
+	background: #fc5;
+}
+html.light tr.play a {
+	color: #406;
+}
+html.light #files th:hover .cfg,
+html.light #files th.min .cfg {
+	background: #ccc;
+}
+html.light #files > thead > tr > th.min span {
+	background: linear-gradient(90deg, rgba(204,204,204,0), rgba(204,204,204,0.5) 70%, #ccc);
+}
+html.light #blocked {
+	background: #eee;
+}
+html.light #blk_play a,
+html.light #blk_abrt a {
+	background: #fff;
+	box-shadow: 0 .2em .4em #ddd;
+}
+html.light #widget a {
+	color: #fc5;
+}
+html.light #files tr.sel:hover td {
+	background: #c37;
+}
+html.light #files tr.sel td {
+	color: #fff;
+}
+html.light #files tr.sel a {
+	color: #fff;
+}
+html.light #files tr.sel a.play.act {
+	color: #fb0;
+}
+html.light input[type="checkbox"] + label {
+	color: #333;
+}
+html.light .opview input[type="text"] {
+	background: #fff;
+	color: #333;
+	box-shadow: 0 0 2px #888;
+	border-color: #38d;
+}
+html.light #ops:hover #opdesc {
+	background: #fff;
+	box-shadow: 0 .3em 1em #ccc;
+}
+html.light #opdesc code {
+	background: #060;
+	color: #fff;
+}
+html.light #u2tab a>span,
+html.light #files td div span {
+	color: #000;
+}
+html.light #path {
+	background: #f7f7f7;
+	text-shadow: none;
+	box-shadow: 0 0 .3em #bbb;
+}
+html.light #path a {
+	color: #333;
+}
+html.light #path a:not(:last-child)::after {
+	border-color: #ccc;
+	background: none;
+	border-width: .1em .1em 0 0;
+	margin: -.2em .3em -.2em -.3em;
+}
+html.light #path a:hover {
+	background: none;
+	color: #60a;
+}
+html.light #files tbody div a {
+	color: #d38;
+}
+html.light #files a:hover,
+html.light #files tr.sel a:hover {
+	color: #000;
+	background: #fff;
+}
+html.light #tree {
+	scrollbar-color: #a70 #ddd;
+}
+html.light #tree::-webkit-scrollbar-track,
+html.light #tree::-webkit-scrollbar {
+	background: #ddd;
+}
+#tree::-webkit-scrollbar-thumb {
+	background: #da0;
 }

copyparty/web/browser.html

@@ -13,15 +13,15 @@
 <body>
     <div id="ops">
         <a href="#" data-dest="" data-desc="close submenu">---</a>
-        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
         {%- if have_up2k_idx %}
+        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
         <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
         {%- else %}
         <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
         {%- endif %}
         <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
         <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
-        <a href="#" data-perm="write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
+        <a href="#" data-perm="read write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
         <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
         <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
         <div id="opdesc"></div>
@@ -29,9 +29,9 @@
 
     <div id="op_search" class="opview">
         {%- if have_tags_idx %}
-        <table id="srch_form" class="tags"></table>
+        <div id="srch_form" class="tags"></div>
         {%- else %}
-        <table id="srch_form"></table>
+        <div id="srch_form"></div>
         {%- endif %}
         <div id="srch_q"></div>
     </div>
@@ -39,6 +39,15 @@
     {%- include 'upload.html' %}
 
     <div id="op_cfg" class="opview opbox">
+        <h3>switches</h3>
+        <div>
+            <a id="tooltips" class="tglbtn" href="#">tooltips</a>
+            <a id="lightmode" class="tglbtn" href="#">lightmode</a>
+        </div>
+        {%- if have_zip %}
+        <h3>folder download</h3>
+        <div id="arc_fmt"></div>
+        {%- endif %}
         <h3>key notation</h3>
         <div id="key_notation"></div>
     </div>
@@ -54,7 +63,7 @@
         <a href="#" id="detree">🍞...</a>
         <a href="#" step="2" id="twobytwo">+</a>
         <a href="#" step="-2" id="twig">&ndash;</a>
-        <a href="#" id="dyntree">a</a>
+        <a href="#" class="tglbtn" id="dyntree">a</a>
         <ul id="treeul"></ul>
         <div id="thx_ff">&nbsp;</div>
     </div>
@@ -66,18 +75,18 @@
     <table id="files">
         <thead>
             <tr>
-                <th></th>
-                <th><span>File Name</span></th>
-                <th sort="int"><span>Size</span></th>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
                 {%- for k in taglist %}
                     {%- if k.startswith('.') %}
-                        <th sort="int"><span>{{ k[1:] }}</span></th>
+                        <th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
                     {%- else %}
-                        <th><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
+                        <th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
                     {%- endif %}
                 {%- endfor %}
-                <th><span>T</span></th>
-                <th><span>Date</span></th>
+                <th name="ext"><span>T</span></th>
+                <th name="ts"><span>Date</span></th>
             </tr>
         </thead>
         <tbody>
@@ -105,15 +114,7 @@
     <div id="srv_info"><span>{{ srv_info }}</span></div>
     {%- endif %}
 
-    <div id="widget">
-        <div id="wtoggle">♫</div>
-        <div id="widgeti">
-            <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
-            <canvas id="pvol" width="288" height="38"></canvas>
-            <canvas id="barpos"></canvas>
-            <canvas id="barbuf"></canvas>
-        </div>
-    </div>
+    <div id="widget"></div>
 
     <script>
         var tag_order_cfg = {{ tag_order }};

copyparty/web/browser2.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>{{ title }}</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+    <style>
+        html{font-family:sans-serif}
+        td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
+        a{display:block}
+    </style>
+</head>
+
+<body>
+    {%- if srv_info %}
+    <p><span>{{ srv_info }}</span></p>
+    {%- endif %}
+
+    {%- if have_b_u %}
+    <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+        <input type="hidden" name="act" value="bput" />
+        <input type="file" name="f" multiple /><br />
+        <input type="submit" value="start upload" />
+    </form>
+    <br />
+    {%- endif %}
+
+    {%- if logues[0] %}
+    <div>{{ logues[0] }}</div><br />
+    {%- endif %}
+
+    <table id="files">
+        <thead>
+            <tr>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
+                <th name="ts"><span>Date</span></th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+
+{%- for f in files %}
+    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+{%- endfor %}
+
+        </tbody>
+    </table>
+    
+    {%- if logues[1] %}
+    <div>{{ logues[1] }}</div><br />
+    {%- endif %}
+    
+    <h2><a href="{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
+
+</body>
+</html>

copyparty/web/md.css

@@ -50,6 +50,9 @@ pre code:last-child {
 pre code::before {
 	content: counter(precode);
 	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
 	display: inline-block;
 	text-align: right;
 	font-size: .75em;
@@ -591,12 +594,3 @@ blink {
 		color: #940;
 	}
 }
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-*/

copyparty/web/md.html

@@ -138,16 +138,16 @@ var md_opt = {
         document.documentElement.setAttribute("class", dark ? "dark" : "");
         btn.innerHTML = "go " + (dark ? "light" : "dark");
         if (window.localStorage)
-            localStorage.setItem('darkmode', dark ? 1 : 0);
+            localStorage.setItem('lightmode', dark ? 0 : 1);
     };
     btn.onclick = toggle;
-    if (window.localStorage && localStorage.getItem('darkmode') == 1)
+    if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		toggle();
 })();
 
 	</script>
     <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.full.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}
 	<script src="/.cpr/md2.js"></script>

copyparty/web/md.js

@@ -46,7 +46,7 @@ function statify(obj) {
     var ua = navigator.userAgent;
     if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
         // necessary on ff-68.7 at least
-        var s = document.createElement('style');
+        var s = mknod('style');
         s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
         console.log(s.innerHTML);
         document.head.appendChild(s);
@@ -175,12 +175,12 @@ function md_plug_err(ex, js) {
         msg = "Line " + ln + ", " + msg;
         var lns = js.split('\n');
         if (ln < lns.length) {
-            o = document.createElement('span');
+            o = mknod('span');
             o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
             o.textContent = lns[ln - 1];
         }
     }
-    errbox = document.createElement('div');
+    errbox = mknod('div');
     errbox.setAttribute('id', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;

copyparty/web/md2.css

@@ -1,126 +1,125 @@
 #toc {
-    display: none;
+	display: none;
 }
 #mtw {
-    display: block;
-    position: fixed;
-    left: .5em;
-    bottom: 0;
-    width: calc(100% - 56em);
+	display: block;
+	position: fixed;
+	left: .5em;
+	bottom: 0;
+	width: calc(100% - 56em);
 }
 #mw {
-    left: calc(100% - 55em);
-    overflow-y: auto;
-    position: fixed;
-    bottom: 0;
+	left: calc(100% - 55em);
+	overflow-y: auto;
+	position: fixed;
+	bottom: 0;
 }
 
 
 /* single-screen */
 #mtw.preview,
 #mw.editor {
-    opacity: 0;
-    z-index: 1;
+	opacity: 0;
+	z-index: 1;
 }
 #mw.preview,
 #mtw.editor {
-    z-index: 5;
+	z-index: 5;
 }
 #mtw.single,
 #mw.single {
-    margin: 0;
-    left: 1em;
-    left: max(1em, calc((100% - 56em) / 2));
+	margin: 0;
+	left: 1em;
+	left: max(1em, calc((100% - 56em) / 2));
 }
 #mtw.single {
-    width: 55em;
-    width: min(55em, calc(100% - 2em));
+	width: 55em;
+	width: min(55em, calc(100% - 2em));
 }
 
 
 #mp {
-    position: relative;
+	position: relative;
 }
 #mt, #mtr {
-    width: 100%;
-    height: calc(100% - 1px);
-    color: #444;
-    background: #f7f7f7;
-    border: 1px solid #999;
-    outline: none;
-    padding: 0;
-    margin: 0;
-    font-family: 'consolas', monospace, monospace;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    word-wrap: break-word; /*ie*/
-    overflow-y: scroll;
-    line-height: 1.3em;
-    font-size: .9em;
-    position: relative;
-    scrollbar-color: #eb0 #f7f7f7;
+	width: 100%;
+	height: calc(100% - 1px);
+	color: #444;
+	background: #f7f7f7;
+	border: 1px solid #999;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	font-family: 'consolas', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-word;
+	overflow-wrap: break-word;
+	word-wrap: break-word; /*ie*/
+	overflow-y: scroll;
+	line-height: 1.3em;
+	font-size: .9em;
+	position: relative;
+	scrollbar-color: #eb0 #f7f7f7;
 }
 html.dark #mt {
-    color: #eee;
-    background: #222;
-    border: 1px solid #777;
-    scrollbar-color: #b80 #282828;
+	color: #eee;
+	background: #222;
+	border: 1px solid #777;
+	scrollbar-color: #b80 #282828;
 }
 #mtr {
-    position: absolute;
-    top: 0;
-    left: 0;
+	position: absolute;
+	top: 0;
+	left: 0;
 }
 #save.force-save {
-    color: #400;
-    background: #f97;
-    border-radius: .15em;
+	color: #400;
+	background: #f97;
+	border-radius: .15em;
 }
 html.dark #save.force-save {
-    color: #fca;
-    background: #720;
+	color: #fca;
+	background: #720;
 }
 #save.disabled {
-    opacity: .4;
+	opacity: .4;
 }
 #helpbox,
 #toast {
-    background: #f7f7f7;
-    border-radius: .4em;
-    z-index: 9001;
+	background: #f7f7f7;
+	border-radius: .4em;
+	z-index: 9001;
 }
 #helpbox {
-    display: none;
-    position: fixed;
-    padding: 2em;
-    top: 4em;
-    overflow-y: auto;
-    box-shadow: 0 .5em 2em #777;
-    height: calc(100% - 12em);
-    left: calc(50% - 15em);
-    right: 0;
-    width: 30em;
+	display: none;
+	position: fixed;
+	padding: 2em;
+	top: 4em;
+	overflow-y: auto;
+	box-shadow: 0 .5em 2em #777;
+	height: calc(100% - 12em);
+	left: calc(50% - 15em);
+	right: 0;
+	width: 30em;
 }
 #helpclose {
-    display: block;
+	display: block;
 }
 html.dark #helpbox {
-    box-shadow: 0 .5em 2em #444;
+	box-shadow: 0 .5em 2em #444;
 }
 html.dark #helpbox,
 html.dark #toast {
-    background: #222;
-    border: 1px solid #079;
-    border-width: 1px 0;
+	background: #222;
+	border: 1px solid #079;
+	border-width: 1px 0;
 }
 #toast {
-    font-weight: bold;
-    text-align: center;
-    padding: .6em 0;
-    position: fixed;
-    z-index: 9001;
-    top: 30%;
-    transition: opacity 0.2s ease-in-out;
-    opacity: 1;
+	font-weight: bold;
+	text-align: center;
+	padding: .6em 0;
+	position: fixed;
+	top: 30%;
+	transition: opacity 0.2s ease-in-out;
+	opacity: 1;
 }

copyparty/web/md2.js

@@ -16,7 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = document.createElement('div');
+    var d = mknod('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
@@ -71,7 +71,7 @@ var map_src = [];
 var map_pre = [];
 function genmap(dom, oldmap) {
     var find = nlines;
-    while (oldmap && find --> 0) {
+    while (oldmap && find-- > 0) {
         var tmap = genmapq(dom, '*[data-ln="' + find + '"]');
         if (!tmap || !tmap.length)
             continue;
@@ -94,7 +94,7 @@ var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
     function draw_md() {
-        var t0 = new Date().getTime();
+        var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
 
@@ -110,7 +110,7 @@ var draw_md = (function () {
 
         cls(ebi('save'), 'disabled', src == server_md);
 
-        var t1 = new Date().getTime();
+        var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
     }
 
@@ -252,7 +252,7 @@ function Modpoll() {
         }
 
         console.log('modpoll...');
-        var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
         var xhr = new XMLHttpRequest();
         xhr.modpoll = this;
         xhr.open('GET', url, true);
@@ -399,7 +399,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
@@ -455,7 +455,7 @@ function toast(autoclose, style, width, msg) {
         ok.parentNode.removeChild(ok);
 
     style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
-    ok = document.createElement('div');
+    ok = mknod('div');
     ok.setAttribute('id', 'toast');
     ok.setAttribute('style', style);
     ok.innerHTML = msg;
@@ -1049,7 +1049,7 @@ action_stack = (function () {
         var p1 = from.length,
             p2 = to.length;
 
-        while (p1 --> 0 && p2 --> 0)
+        while (p1-- > 0 && p2-- > 0)
             if (from[p1] != to[p2])
                 break;
 
@@ -1142,14 +1142,3 @@ action_stack = (function () {
         _ref: ref
     }
 })();
-
-/*
-ebi('help').onclick = function () {
-    var c1 = getComputedStyle(dom_src).cssText.split(';');
-    var c2 = getComputedStyle(dom_ref).cssText.split(';');
-    var max = Math.min(c1.length, c2.length);
-    for (var a = 0; a < max; a++)
-        if (c1[a] !== c2[a])
-            console.log(c1[a] + '\n' + c2[a]);
-}
-*/

copyparty/web/mde.css

@@ -8,68 +8,58 @@ html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
 html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
 
 html {
-    line-height: 1.5em;
+	line-height: 1.5em;
 }
 html, body {
-    margin: 0;
-    padding: 0;
-    min-height: 100%;
-    font-family: sans-serif;
-    background: #f7f7f7;
-    color: #333;
+	margin: 0;
+	padding: 0;
+	min-height: 100%;
+	font-family: sans-serif;
+	background: #f7f7f7;
+	color: #333;
 }
 #mn {
-    font-weight: normal;
-    margin: 1.3em 0 .7em 1em;
+	font-weight: normal;
+	margin: 1.3em 0 .7em 1em;
 }
 #mn a {
-    color: #444;
-    margin: 0 0 0 -.2em;
-    padding: 0 0 0 .4em;
-    text-decoration: none;
-    /* ie: */
-    border-bottom: .1em solid #777\9;
-    margin-right: 1em\9;
+	color: #444;
+	margin: 0 0 0 -.2em;
+	padding: 0 0 0 .4em;
+	text-decoration: none;
+	/* ie: */
+	border-bottom: .1em solid #777\9;
+	margin-right: 1em\9;
 }
 #mn a:first-child {
-    padding-left: .5em;
+	padding-left: .5em;
 }
 #mn a:last-child {
-    padding-right: .5em;
+	padding-right: .5em;
 }
 #mn a:not(:last-child):after {
-    content: '';
-    width: 1.05em;
-    height: 1.05em;
-    margin: -.2em .3em -.2em -.4em;
-    display: inline-block;
-    border: 1px solid rgba(0,0,0,0.2);
-    border-width: .2em .2em 0 0;
-    transform: rotate(45deg);
+	content: '';
+	width: 1.05em;
+	height: 1.05em;
+	margin: -.2em .3em -.2em -.4em;
+	display: inline-block;
+	border: 1px solid rgba(0,0,0,0.2);
+	border-width: .2em .2em 0 0;
+	transform: rotate(45deg);
 }
 #mn a:hover {
-    color: #000;
-    text-decoration: underline;
+	color: #000;
+	text-decoration: underline;
 }
 
 html .editor-toolbar>button.disabled {
-    opacity: .35;
-    pointer-events: none;
+	opacity: .35;
+	pointer-events: none;
 }
 html .editor-toolbar>button.save.force-save {
-    background: #f97;
+	background: #f97;
 }
 
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-.cm-header { font-size: .4em !important }
-*/
-
 
 
 
@@ -101,29 +91,29 @@ html .editor-toolbar>button.save.force-save {
 	line-height: 1.1em;
 }
 .mdo a {
-    color: #fff;
-    background: #39b;
-    text-decoration: none;
-    padding: 0 .3em;
-    border: none;
-    border-bottom: .07em solid #079;
+	color: #fff;
+	background: #39b;
+	text-decoration: none;
+	padding: 0 .3em;
+	border: none;
+	border-bottom: .07em solid #079;
 }
 .mdo h2 {
-    color: #fff;
-    background: #555;
-    margin-top: 2em;
-    border-bottom: .22em solid #999;
-    border-top: none;
+	color: #fff;
+	background: #555;
+	margin-top: 2em;
+	border-bottom: .22em solid #999;
+	border-top: none;
 }
 .mdo h1 {
-    color: #fff;
-    background: #444;
-    font-weight: normal;
-    border-top: .4em solid #fb0;
-    border-bottom: .4em solid #777;
-    border-radius: 0 1em 0 1em;
-    margin: 3em 0 1em 0;
-    padding: .5em 0;
+	color: #fff;
+	background: #444;
+	font-weight: normal;
+	border-top: .4em solid #fb0;
+	border-bottom: .4em solid #777;
+	border-radius: 0 1em 0 1em;
+	margin: 3em 0 1em 0;
+	padding: .5em 0;
 }
 h1, h2 {
 	line-height: 1.5em;
@@ -197,14 +187,14 @@ th {
 
 /* mde support */
 .mdo {
-    padding: 1em;
-    background: #f7f7f7;
+	padding: 1em;
+	background: #f7f7f7;
 }
 html.dark .mdo {
-    background: #1c1c1c;
+	background: #1c1c1c;
 }
 .CodeMirror {
-    background: #f7f7f7;
+	background: #f7f7f7;
 }
 
 
@@ -214,108 +204,108 @@ html.dark .mdo {
 /* darkmode */
 html.dark .mdo,
 html.dark .CodeMirror {
-    border-color: #222;
+	border-color: #222;
 }
 html.dark,
 html.dark body,
 html.dark .CodeMirror {
-    background: #222;
-    color: #ccc;
+	background: #222;
+	color: #ccc;
 }
 html.dark .CodeMirror-cursor {
-    border-color: #fff;
+	border-color: #fff;
 }
 html.dark .CodeMirror-selected {
-    box-shadow: 0 0 1px #0cf inset;
+	box-shadow: 0 0 1px #0cf inset;
 }
 html.dark .CodeMirror-selected,
 html.dark .CodeMirror-selectedtext {
-    border-radius: .1em;
-    background: #246;
-    color: #fff;
+	border-radius: .1em;
+	background: #246;
+	color: #fff;
 }
 html.dark .mdo a {
-    background: #057;
+	background: #057;
 }
 html.dark .mdo h1 a, html.dark .mdo h4 a,
 html.dark .mdo h2 a, html.dark .mdo h5 a,
 html.dark .mdo h3 a, html.dark .mdo h6 a {
-    color: inherit;
-    background: none;
+	color: inherit;
+	background: none;
 }
 html.dark pre,
 html.dark code {
-    color: #8c0;
-    background: #1a1a1a;
-    border: .07em solid #333;
+	color: #8c0;
+	background: #1a1a1a;
+	border: .07em solid #333;
 }
 html.dark .mdo ul,
 html.dark .mdo ol {
-    border-color: #444;
+	border-color: #444;
 }
 html.dark .mdo>ul,
 html.dark .mdo>ol {
-    border-color: #555;
+	border-color: #555;
 }
 html.dark strong {
-    color: #fff;
+	color: #fff;
 }
 html.dark p>em,
 html.dark li>em,
 html.dark td>em {
-    color: #f94;
-    border-color: #666;
+	color: #f94;
+	border-color: #666;
 }
 html.dark h1 {
-    background: #383838;
-    border-top: .4em solid #b80;
-    border-bottom: .4em solid #4c4c4c;
+	background: #383838;
+	border-top: .4em solid #b80;
+	border-bottom: .4em solid #4c4c4c;
 }
 html.dark h2 {
-    background: #444;
-    border-bottom: .22em solid #555;
+	background: #444;
+	border-bottom: .22em solid #555;
 }
 html.dark td,
 html.dark th {
-    border-color: #444;
+	border-color: #444;
 }
 html.dark blockquote {
-    background: #282828;
-    border: .07em dashed #444;
+	background: #282828;
+	border: .07em dashed #444;
 }
 
 
 
 html.dark #mn a {
-    color: #ccc;
+	color: #ccc;
 }
 html.dark #mn a:not(:last-child):after {
-    border-color: rgba(255,255,255,0.3);
+	border-color: rgba(255,255,255,0.3);
 }
 html.dark .editor-toolbar {
-    border-color: #2c2c2c;
-    background: #1c1c1c;
+	border-color: #2c2c2c;
+	background: #1c1c1c;
 }
 html.dark .editor-toolbar>i.separator {
-    border-left: 1px solid #444;
-    border-right: 1px solid #111;
+	border-left: 1px solid #444;
+	border-right: 1px solid #111;
 }
 html.dark .editor-toolbar>button {
-    margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
-    color: #aaa;
+	margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
+	color: #aaa;
 }
 
 
 
 html.dark .editor-toolbar>button:hover {
-    color: #333;
+	color: #333;
 }
 html.dark .editor-toolbar>button.active {
-    color: #333;
-    border-color: #ec1;
-    background: #c90;
+	color: #333;
+	border-color: #ec1;
+	background: #c90;
 }
 html.dark .editor-toolbar::after,
 html.dark .editor-toolbar::before {
-    background: none;
+	background: none;
 }

copyparty/web/mde.html

@@ -31,12 +31,12 @@ var md_opt = {
 
 var lightswitch = (function () {
 	var fun = function () {
-		var dark = !!!document.documentElement.getAttribute("class");
+		var dark = !document.documentElement.getAttribute("class");
 		document.documentElement.setAttribute("class", dark ? "dark" : "");
 		if (window.localStorage)
-			localStorage.setItem('darkmode', dark ? 1 : 0);
+			localStorage.setItem('lightmode', dark ? 0 : 1);
 	};
-	if (window.localStorage && localStorage.getItem('darkmode') == 1)
+	if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		fun();
 	
 	return fun;

copyparty/web/mde.js

@@ -71,7 +71,7 @@ var mde = (function () {
 })();
 
 function set_jumpto() {
-    document.querySelector('.editor-preview-side').onclick = jumpto;
+    QS('.editor-preview-side').onclick = jumpto;
 }
 
 function jumpto(ev) {
@@ -94,7 +94,7 @@ function md_changed(mde, on_srv) {
         window.md_saved = mde.value();
 
     var md_now = mde.value();
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
 
     if (md_now == window.md_saved)
         save_btn.classList.add('disabled');
@@ -105,7 +105,7 @@ function md_changed(mde, on_srv) {
 }
 
 function save(mde) {
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
     if (save_btn.classList.contains('disabled')) {
         alert('there is nothing to save');
         return;
@@ -212,7 +212,7 @@ function save_chk() {
     last_modified = this.lastmod;
     md_changed(this.mde, true);
 
-    var ok = document.createElement('div');
+    var ok = mknod('div');
     ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
     ok.innerHTML = 'OK✔️';
     var parent = ebi('m');

copyparty/web/msg.css

@@ -3,7 +3,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -20,8 +20,8 @@ body {
 	padding-bottom: 5em;
 }
 #box {
-    padding: .5em 1em;
-    background: #2c2c2c;
+	padding: .5em 1em;
+	background: #2c2c2c;
 }
 pre {
 	font-family: monospace, monospace;

copyparty/web/splash.html

@@ -13,23 +13,27 @@
     <div id="wrap">
         <p>hello {{ this.uname }}</p>
 
+        {%- if rvol %}
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
+        {%- if wvol %}
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/">
+            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />
@@ -38,7 +42,7 @@
     </div>
     <script>
 
-if (window.localStorage && localStorage.getItem('darkmode') == 1)
+if (window.localStorage && localStorage.getItem('lightmode') != 1)
     document.documentElement.setAttribute("class", "dark");
 
 </script>

copyparty/web/upload.css

@@ -19,6 +19,11 @@
 	color: #f87;
 	padding: .5em;
 }
+#u2err.msg {
+	color: #999;
+	padding: .5em;
+	font-size: .9em;
+}
 #u2btn {
 	color: #eee;
 	background: #555;
@@ -47,6 +52,11 @@
 	margin: -1.5em 0;
 	padding: .8em 0;
 	width: 100%;
+	max-width: 12em;
+	display: inline-block;
+}
+#u2conf #u2btn_cw {
+	text-align: right;
 }
 #u2notbtn {
 	display: none;
@@ -72,6 +82,7 @@
 }
 #u2tab td:nth-child(2) {
 	width: 5em;
+	white-space: nowrap;
 }
 #u2tab td:nth-child(3) {
 	width: 40%;
@@ -80,15 +91,52 @@
 	font-family: sans-serif;
 	width: auto;
 }
-#u2tab tr+tr:hover td {
+#u2tab tbody tr:hover td {
 	background: #222;
 }
+#u2cards {
+	padding: 1em 0 .3em 1em;
+	margin: 1.5em auto -2.5em auto;
+	white-space: nowrap;
+	text-align: center;
+	overflow: hidden;
+}
+#u2cards.w {
+	width: 45em;
+	text-align: left;
+}
+#u2cards a {
+	padding: .2em 1em;
+	border: 1px solid #777;
+	border-width: 0 0 1px 0;
+	background: linear-gradient(to bottom, #333, #222);
+}
+#u2cards a:first-child {
+	border-radius: .4em 0 0 0;
+}
+#u2cards a:last-child {
+	border-radius: 0 .4em 0 0;
+}
+#u2cards a.act {
+	padding-bottom: .5em;
+	border-width: 1px 1px .1em 1px;
+	border-radius: .3em .3em 0 0;
+	margin-left: -1px;
+	background: linear-gradient(to bottom, #464, #333 80%);
+	box-shadow: 0 -.17em .67em #280;
+	border-color: #7c5 #583 #333 #583;
+	position: relative;
+	color: #fd7;
+}
+#u2cards span {
+	color: #fff;
+}
 #u2conf {
 	margin: 1em auto;
 	width: 30em;
 }
 #u2conf.has_btn {
-	width: 46em;
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;
@@ -99,12 +147,16 @@
 	outline: none;
 }
 #u2conf .txtbox {
-	width: 4em;
+	width: 3em;
 	color: #fff;
 	background: #444;
 	border: 1px solid #777;
 	font-size: 1.2em;
 	padding: .15em 0;
+	height: 1.05em;
+}
+#u2conf .txtbox.err {
+	background: #922;
 }
 #u2conf a {
 	color: #fff;
@@ -113,13 +165,12 @@
 	border-radius: .1em;
 	font-size: 1.5em;
 	padding: .1em 0;
-	margin: 0 -.25em;
+	margin: 0 -1px;
 	width: 1.5em;
 	height: 1em;
 	display: inline-block;
 	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
+	bottom: -0.08em;
 }
 #u2conf input+a {
 	background: #d80;
@@ -130,7 +181,6 @@
 	height: 1em;
 	padding: .4em 0;
 	display: block;
-	user-select: none;
 	border-radius: .25em;
 }
 #u2conf input[type="checkbox"] {
@@ -170,12 +220,13 @@
 	text-align: center;
 	overflow: hidden;
 	margin: 0 -2em;
-	height: 0;
 	padding: 0 1em;
+	height: 0;
 	opacity: .1;
-    transition: all 0.14s ease-in-out;
-	border-radius: .4em;
+	transition: all 0.14s ease-in-out;
 	box-shadow: 0 .2em .5em #222;
+	border-radius: .4em;
+	z-index: 1;
 }
 #u2cdesc.show {
 	padding: 1em;
@@ -193,24 +244,6 @@
 .prog {
 	font-family: monospace;
 }
-.prog>div {
-	display: inline-block;
-	position: relative;
-	overflow: hidden;
-	margin: 0;
-	padding: 0;
-	height: 1.1em;
-	margin-bottom: -.15em;
-	box-shadow: -1px -1px 0 inset rgba(255,255,255,0.1);
-}
-.prog>div>div {
-	width: 0%;
-	position: absolute;
-	left: 0;
-	top: 0;
-	bottom: 0;
-	background: #0a0;
-}
 #u2tab a>span {
 	font-weight: bold;
 	font-style: italic;
@@ -221,3 +254,44 @@
 	float: right;
 	margin-bottom: -.3em;
 }
+
+
+
+
+
+html.light #u2btn {
+	box-shadow: .4em .4em 0 #ccc;
+}
+html.light #u2cards span {
+	color: #000;
+}
+html.light #u2cards a {
+	background: linear-gradient(to bottom, #eee, #fff);
+}
+html.light #u2cards a.act {
+	color: #037;
+	background: inherit;
+	box-shadow: 0 -.17em .67em #0ad;
+	border-color: #09c #05a #eee #05a;
+}
+html.light #u2conf .txtbox {
+	background: #fff;
+	color: #444;
+}
+html.light #u2conf .txtbox.err {
+	background: #f96;
+	color: #300;
+}
+html.light #u2cdesc {
+	background: #fff;
+	border: none;
+}
+html.light #op_up2k.srch #u2btn {
+	border-color: #a80;
+}
+html.light #u2foot {
+	color: #000;
+}
+html.light #u2tab tbody tr:hover td {
+	background: #fff;
+}

copyparty/web/upload.html

@@ -1,7 +1,7 @@
 
     <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
@@ -9,7 +9,7 @@
     </div>
 
     <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="mkdir" />
             <input type="text" name="name" size="30">
             <input type="submit" value="mkdir">
@@ -17,15 +17,15 @@
     </div>
 
     <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="new_md" />
             <input type="text" name="name" size="30">
             <input type="submit" value="create doc">
         </form>
     </div>
 
-    <div id="op_msg" class="opview opbox">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
+    <div id="op_msg" class="opview opbox act">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="text" name="msg" size="30">
             <input type="submit" value="send msg">
         </form>
@@ -36,7 +36,7 @@
 
             <table id="u2conf">
                 <tr>
-                    <td>parallel uploads</td>
+                    <td><br />parallel uploads:</td>
                     <td rowspan="2">
                         <input type="checkbox" id="multitask" />
                         <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
@@ -59,9 +59,9 @@
                 </tr>
                 <tr>
                     <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
+                        <a href="#" id="nthread_sub">&ndash;</a><input
+                            class="txtbox" id="nthread" value="2"/><a
+                            href="#" id="nthread_add">+</a><br />&nbsp;
                     </td>
                 </tr>
             </table>
@@ -73,19 +73,31 @@
             <div id="u2btn_ct">
                 <div id="u2btn">
                     <span id="u2bm"></span><br />
-                    drop files here<br />
+                    drag/drop files<br />
+                    and folders here<br />
                     (or click me)
                 </div>
             </div>
 
+            <div id="u2cards">
+                <a href="#" act="ok">ok <span>0</span></a><a
+                href="#" act="ng">ng <span>0</span></a><a
+                href="#" act="done">done <span>0</span></a><a
+                href="#" act="bz" class="act">busy <span>0</span></a><a
+                href="#" act="q">que <span>0</span></a>
+            </div>
+
             <table id="u2tab">
-                <tr>
-                    <td>filename</td>
-                    <td>status</td>
-                    <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
-                </tr>
+                <thead>
+                    <tr>
+                        <td>filename</td>
+                        <td>status</td>
+                        <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
+                    </tr>
+                </thead>
+                <tbody></tbody>
             </table>
 
             <p id="u2foot"></p>
-            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
+            <p id="u2footfoot" data-perm="write">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
     </div>

copyparty/web/util.js

@@ -1,5 +1,15 @@
 "use strict";
 
+if (!window['console'])
+    window['console'] = {
+        "log": function (msg) { }
+    };
+
+
+var clickev = window.Touch ? 'touchstart' : 'click',
+    ANDROID = /(android)/i.test(navigator.userAgent);
+
+
 // error handler for mobile devices
 function hcroak(msg) {
     document.body.innerHTML = msg;
@@ -40,9 +50,11 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
 }
 
 
-function ebi(id) {
-    return document.getElementById(id);
-}
+var ebi = document.getElementById.bind(document),
+    QS = document.querySelector.bind(document),
+    QSA = document.querySelectorAll.bind(document),
+    mknod = document.createElement.bind(document);
+
 
 function ev(e) {
     e = e || window.event;
@@ -80,7 +92,7 @@ if (!String.startsWith) {
 // https://stackoverflow.com/a/950146
 function import_js(url, cb) {
     var head = document.head || document.getElementsByTagName('head')[0];
-    var script = document.createElement('script');
+    var script = mknod('script');
     script.type = 'text/javascript';
     script.src = url;
 
@@ -91,7 +103,107 @@ function import_js(url, cb) {
 }
 
 
-function sortTable(table, col) {
+var crctab = (function () {
+    var c, tab = [];
+    for (var n = 0; n < 256; n++) {
+        c = n;
+        for (var k = 0; k < 8; k++) {
+            c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+        }
+        tab[n] = c;
+    }
+    return tab;
+})();
+
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+        crc = (crc >>> 8) ^ crctab[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return ((crc ^ (-1)) >>> 0).toString(16);
+}
+
+
+function clmod(obj, cls, add) {
+    var re = new RegExp('\\s*\\b' + cls + '\\s*\\b', 'g');
+    if (add == 't')
+        add = !re.test(obj.className);
+
+    obj.className = obj.className.replace(re, ' ') + (add ? ' ' + cls : '');
+}
+
+
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["lead", -1, ""], ["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v)
+                        v = uricom_dec(v)[0]
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map(function (x) { return x; });
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+    }
+    return nodes;
+}
+
+
+function sortTable(table, col, cb) {
     var tb = table.tBodies[0],
         th = table.tHead.rows[0].cells,
         tr = Array.prototype.slice.call(tb.rows, 0),
@@ -100,6 +212,27 @@ function sortTable(table, col) {
         th[a].className = th[a].className.replace(/ *sort-?1 */, " ");
     th[col].className += ' sort' + reverse;
     var stype = th[col].getAttribute('sort');
+    try {
+        var nrules = [], rules = jread("fsort", []);
+        rules.unshift([th[col].getAttribute('name'), reverse, stype || '']);
+        for (var a = 0; a < rules.length; a++) {
+            var add = true;
+            for (var b = 0; b < a; b++)
+                if (rules[a][0] == rules[b][0])
+                    add = false;
+
+            if (add)
+                nrules.push(rules[a]);
+
+            if (nrules.length >= 10)
+                break;
+        }
+        jwrite("fsort", nrules);
+    }
+    catch (ex) {
+        console.log("failed to persist sort rules, resetting: " + ex);
+        jwrite("fsort", null);
+    }
     var vl = [];
     for (var a = 0; a < tr.length; a++) {
         var cell = tr[a].cells[col];
@@ -127,8 +260,9 @@ function sortTable(table, col) {
         return reverse * (a.localeCompare(b));
     });
     for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
+    if (cb) cb();
 }
-function makeSortable(table) {
+function makeSortable(table, cb) {
     var th = table.tHead, i;
     th && (th = th.rows[0]) && (th = th.cells);
     if (th) i = th.length;
@@ -136,15 +270,14 @@ function makeSortable(table) {
     while (--i >= 0) (function (i) {
         th[i].onclick = function (e) {
             ev(e);
-            sortTable(table, i);
+            sortTable(table, i, cb);
         };
     }(i));
 }
 
 
-
 (function () {
-    var ops = document.querySelectorAll('#ops>a');
+    var ops = QSA('#ops>a');
     for (var a = 0; a < ops.length; a++) {
         ops[a].onclick = opclick;
     }
@@ -159,25 +292,25 @@ function opclick(e) {
 
     swrite('opmode', dest || null);
 
-    var input = document.querySelector('.opview.act input:not([type="hidden"])')
+    var input = QS('.opview.act input:not([type="hidden"])')
     if (input)
         input.focus();
 }
 
 
 function goto(dest) {
-    var obj = document.querySelectorAll('.opview.act');
+    var obj = QSA('.opview.act');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
-    obj = document.querySelectorAll('#ops>a');
+    obj = QSA('#ops>a');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     if (dest) {
         var ui = ebi('op_' + dest);
-        ui.classList.add('act');
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+        clmod(ui, 'act', true);
+        QS('#ops>a[data-dest=' + dest + ']').className += " act";
 
         var fn = window['goto_' + dest];
         if (fn)
@@ -193,7 +326,10 @@ function goto(dest) {
     goto();
     var op = sread('opmode');
     if (op !== null && op !== '.')
-        goto(op);
+        try {
+            goto(op);
+        }
+        catch (ex) { }
 })();
 
 
@@ -273,6 +409,7 @@ function unix2iso(ts) {
 
 
 function s2ms(s) {
+    s = Math.floor(s);
     var m = Math.floor(s / 60);
     return m + ":" + ("0" + (s - m * 60)).slice(-2);
 }
@@ -359,8 +496,9 @@ function bcfg_upd_ui(name, val) {
 
     if (o.getAttribute('type') == 'checkbox')
         o.checked = val;
-    else if (o)
-        o.setAttribute('class', val ? 'on' : '');
+    else if (o) {
+        clmod(o, 'on', val);
+    }
 }
 
 

docs/example.conf

@@ -32,9 +32,13 @@ r
 
 # and a folder where anyone can upload
 # but nobody can see the contents
+# and set the e2d flag to enable the uploads database
+# and set the nodupe flag to reject duplicate uploads
 /home/ed/inc
 /dump
 w
+c e2d
+c nodupe
 
 # this entire config file can be replaced with these arguments:
 # -u ed:123 -u k:k -v .::r:aed -v priv:priv:rk:aed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w

docs/minimal-up2k.html

@@ -0,0 +1,32 @@
+<!--
+  save this as .epilogue.html inside a write-only folder to declutter the UI,  makes it look like
+  https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
+-->
+
+<style>
+
+    /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
+
+    #ops, #tree, #path, #wrap>h2:last-child,  /* main tabs and navigators (tree/breadcrumbs) */
+
+    #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
+
+    #u2cards  /* and the upload progress tabs */
+
+    {display: none !important}  /* do it! */
+
+
+
+    /* add some margins because now it's weird */
+    .opview {margin-top: 2.5em}
+    #op_up2k {margin-top: 3em}
+
+    /* and embiggen the upload button */
+    #u2conf #u2btn, #u2btn {padding:1.5em 0}
+
+    /* adjust the button area a bit */
+    #u2conf.has_btn {width: 35em !important; margin: 5em auto}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

docs/music-analysis.sh

@@ -0,0 +1,242 @@
+#!/bin/bash
+echo please dont actually run this as a scriopt
+exit 1
+
+
+# dependency-heavy, not particularly good fit
+pacman -S llvm10
+python3 -m pip install --user librosa
+git clone https://github.com/librosa/librosa.git
+
+
+# correct bpm for tracks with bad tags
+br='
+/Trip Trip Trip\(Hardcore Edit\).mp3/ {v=176}
+/World!!.BIG_SOS/ {v=175}
+/\/08\..*\(BIG_SOS Bootleg\)\.mp3/ {v=175}
+/もってけ！セーラ服.Asterisk DnB/ {v=175}
+/Rondo\(Asterisk DnB Re.mp3/ {v=175}
+/Ray Nautica 175 Edit/ {v=175;x="thunk"}
+/TOKIMEKI Language.Jauz/ {v=174}
+/YUPPUN Hardcore Remix\).mp3/ {v=174;x="keeps drifting"}
+/(èâAâï.î╧ûδ|バーチャリアル.狐耶)J-Core Remix\).mp3/ {v=172;x="hard"}
+/lucky train..Freezer/ {v=170}
+/Alf zero Bootleg ReMix/ {v=170}
+/Prisoner of Love.Kacky/ {v=170}
+/火炎 .Qota/ {v=170}
+/\(hu-zin Bootleg\)\.mp3/ {v=170}
+/15. STRAIGHT BET\(Milynn Bootleg\)\.mp3/ {v=170}
+/\/13.*\(Milynn Bootleg\)\.mp3/ {v=167;x="way hard"}
+/COLOR PLANET .10SAI . nijikon Remix\)\.mp3/ {v=165}
+/11\. (朝はご飯派|Æ⌐é═é▓ö╤öh)\.mp3/ {v=162}
+/09\. Where.s the core/ {v=160}
+/PLANET\(Koushif Jersey Club Bootleg\)remaster.mp3/ {v=160;x="starts ez turns bs"}
+/kened Soul - Madeon x Angel Beats!.mp3/ {v=160}
+/Dear Moments\(Mother Harlot Bootleg\)\.mp3/ {v=150}
+/POWER.Ringos UKG/ {v=140}
+/ブルー・フィールド\(Ringos UKG Remix\).mp3/ {v=135}
+/プラチナジェット.Ringo Remix..mp3/ {v=131.2}
+/Mirrorball Love \(TKM Bootleg Mix\).mp3/ {v=130}
+/Photon Melodies \(TKM Bootleg Mix\).mp3/ {v=128}
+/Trap of Love \(TKM Bootleg Mix\).mp3/ {v=128}
+/One Step \(TKM Bootleg Mix\)\.mp3/ {v=126}
+/04 (トリカムイ岩|âgâèâJâÇâCèΓ).mp3/ {v=125}
+/Get your Wish \(NAWN REMIX\)\.mp3/ {v=95}
+/Flicker .Nitro Fun/ {v=92}
+/\/14\..*suicat Remix/ {v=85.5;x="tricky"}
+/Yanagi Nagi - Harumodoki \(EO Remix\)\.mp3/ {v=150}
+/Azure - Nicology\.mp3/ {v=128;x="off by 5 how"}
+'
+
+
+# afun host, collects/grades the results
+runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | awk '{v=$1;sub(/[^ ]+ /,"")} '"$br"' {printf "%s %s\n",v,$0}' | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk 'function prt(c) {printf "\033[3%sm%s\033[0m\n",c,$0} $8!="div,"{next} $5!~/^[0-9\.]+/{next} {meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;prt(6);next} idiff==1{n1++;prt(3);next} idiff>10{nx++;prt(1);next} {n10++;prt(5)} END {printf "ok: %d   1off: %2s   (%3s)   10off: %2s   (%3s)   fail: %2s\n",n0,n1,n0+n1,n10,n0+n1+n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; }
+
+
+# ok:   8   1off: 62   ( 70)   10off: 86   (156)   fail: 25   # 105 sec,  librosa @ 8c archvm on 3700x w10
+# ok:   4   1off: 59   ( 63)   10off: 65   (128)   fail: 53   # using original tags (bad)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } runfun
+
+
+# ok: 119   1off:  5   (124)   10off:  8   (132)   fail: 49   # 51 sec,  vamp-example-fixedtempo
+# ok: 109   1off:  4   (113)   10off:  9   (122)   fail: 59   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; }; runfun
+
+
+# ok: 102   1off: 61   (163)   10off: 12   (175)   fail:  6   # 61 sec,  vamp-qm-tempotracker
+# ok:  80   1off: 48   (128)   10off: 11   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; }; runfun
+
+
+# ok: 133   1off: 32   (165)   10off: 12   (177)   fail:  3   # 51 sec,  vamp-beatroot
+# ok: 101   1off: 22   (123)   10off: 16   (139)   fail: 39   # bad-tags
+# note: some tracks fully fail to analyze (unlike the others which always provide a guess)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; }; runfun
+
+
+# ok: 124   1off:  9   (133)   10off: 40   (173)   fail:  8   # 231 sec,  essentia/full
+# ok: 109   1off:  8   (117)   10off: 22   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'import essentia; import essentia.standard as es; fe, fef = es.MusicExtractor(lowlevelStats=["mean", "stdev"], rhythmStats=["mean", "stdev"], tonalStats=["mean", "stdev"])("/dev/shm/'$core'.wav"); print("{:.2f}".format(fe["rhythm.bpm"]))')"; }; runfun
+
+
+# ok: 113   1off: 18   (131)   10off: 46   (177)   fail:  4   # 134 sec,  essentia/re2013
+# ok: 101   1off: 15   (116)   10off: 26   (142)   fail: 39   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'from essentia.standard import *; a=MonoLoader(filename="/dev/shm/'$core'.wav")(); bpm,beats,confidence,_,intervals=RhythmExtractor2013(method="multifeature")(a); print("{:.2f}".format(bpm))')"; }; runfun
+
+
+
+########################################################################
+##
+##  key detectyion
+##
+########################################################################
+
+
+
+# console scriptlet reusing keytabs from browser.js
+var m=''; for (var a=0; a<24; a++) m += 's/\\|(' + maps["traktor_sharps"][a].trim() + "|" + maps["rekobo_classic"][a].trim() + "|" + maps["traktor_musical"][a].trim() + "|" + maps["traktor_open"][a].trim() + ')$/|' + maps["rekobo_alnum"][a].trim() + '/;'; console.log(m);
+
+
+# translate to camelot
+re='s/\|(B|B|B|6d)$/|1B/;s/\|(F#|F#|Gb|7d)$/|2B/;s/\|(C#|Db|Db|8d)$/|3B/;s/\|(G#|Ab|Ab|9d)$/|4B/;s/\|(D#|Eb|Eb|10d)$/|5B/;s/\|(A#|Bb|Bb|11d)$/|6B/;s/\|(F|F|F|12d)$/|7B/;s/\|(C|C|C|1d)$/|8B/;s/\|(G|G|G|2d)$/|9B/;s/\|(D|D|D|3d)$/|10B/;s/\|(A|A|A|4d)$/|11B/;s/\|(E|E|E|5d)$/|12B/;s/\|(G#m|Abm|Abm|6m)$/|1A/;s/\|(D#m|Ebm|Ebm|7m)$/|2A/;s/\|(A#m|Bbm|Bbm|8m)$/|3A/;s/\|(Fm|Fm|Fm|9m)$/|4A/;s/\|(Cm|Cm|Cm|10m)$/|5A/;s/\|(Gm|Gm|Gm|11m)$/|6A/;s/\|(Dm|Dm|Dm|12m)$/|7A/;s/\|(Am|Am|Am|1m)$/|8A/;s/\|(Em|Em|Em|2m)$/|9A/;s/\|(Bm|Bm|Bm|3m)$/|10A/;s/\|(F#m|F#m|Gbm|4m)$/|11A/;s/\|(C#m|Dbm|Dbm|5m)$/|12A/;'
+
+
+# runner/wrapper
+runfun() { cores=8; touch run; tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, key.v from mt key join mt dur on key.w = dur.w where key.k = "key" and dur.k = ".dur" order by dur.w' | uniq -w16 | grep -vE '(Off-Key|None)$' | sed -r "s/ //g;$re" | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done| grep mir/cr | tr \| / | while read key fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); [ "$key" = "$py" ] && c=2 || c=5; printf '%4s sec, %4s orig, \033[3%dm%4s py,\033[0m %s\n' $td "$key" $c "$py" "$fn") || break; done & done; time wait 2>/dev/null; }
+
+
+# ok: 26   1off: 10   2off: 1   fail: 3   #  15 sec, keyfinder
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 -c 'import sys; import keyfinder; print(keyfinder.key(sys.argv[1]).camelot())' "/dev/shm/$core.wav")"; }; runfun
+
+
+# https://github.com/MTG/essentia/raw/master/src/examples/tutorial/example_key_by_steps_streaming.py
+# https://essentia.upf.edu/reference/std_Key.html  # edma edmm braw bgate
+sed -ri 's/^(key = Key\().*/\1profileType="bgate")/' example_key_by_steps_streaming.py
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 example_key_by_steps_streaming.py /dev/shm/$core.{wav,yml} 2>/dev/null | sed -r "s/ major//;s/ minor/m/;s/^/|/;$re;s/.//")"; }; runfun
+
+
+
+########################################################################
+##
+##  misc
+##
+########################################################################
+
+
+
+python3 -m pip install --user vamp
+
+import librosa
+d, r = librosa.load('/dev/shm/0.wav')
+d.dtype
+# dtype('float32')
+d.shape
+# (1323000,)
+d
+# array([-1.9614939e-08,  1.8037968e-08, -1.4106059e-08, ...,
+#         1.2024145e-01,  2.7462116e-01,  1.6202132e-01], dtype=float32)
+
+
+
+import vamp
+c = vamp.collect(d, r, "vamp-example-plugins:fixedtempo")
+c
+# {'list': [{'timestamp':  0.005804988, 'duration':  9.999092971, 'label': '110.0 bpm', 'values': array([109.98116], dtype=float32)}]}
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+d
+array([-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+       -0.0267825 , -0.03564296], dtype=float32)
+
+d = np.reshape(d, [1, -1])
+d
+array([[-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+        -0.0267825 , -0.03564296]], dtype=float32)
+
+
+
+import vampyhost
+print("\n".join(vampyhost.list_plugins()))
+
+mvamp:marsyas_bextract_centroid
+mvamp:marsyas_bextract_lpcc
+mvamp:marsyas_bextract_lsp
+mvamp:marsyas_bextract_mfcc
+mvamp:marsyas_bextract_rolloff
+mvamp:marsyas_bextract_scf
+mvamp:marsyas_bextract_sfm
+mvamp:marsyas_bextract_zero_crossings
+mvamp:marsyas_ibt
+mvamp:zerocrossing
+qm-vamp-plugins:qm-adaptivespectrogram
+qm-vamp-plugins:qm-barbeattracker
+qm-vamp-plugins:qm-chromagram
+qm-vamp-plugins:qm-constantq
+qm-vamp-plugins:qm-dwt
+qm-vamp-plugins:qm-keydetector
+qm-vamp-plugins:qm-mfcc
+qm-vamp-plugins:qm-onsetdetector
+qm-vamp-plugins:qm-segmenter
+qm-vamp-plugins:qm-similarity
+qm-vamp-plugins:qm-tempotracker
+qm-vamp-plugins:qm-tonalchange
+qm-vamp-plugins:qm-transcription
+vamp-aubio:aubiomelenergy
+vamp-aubio:aubiomfcc
+vamp-aubio:aubionotes
+vamp-aubio:aubioonset
+vamp-aubio:aubiopitch
+vamp-aubio:aubiosilence
+vamp-aubio:aubiospecdesc
+vamp-aubio:aubiotempo
+vamp-example-plugins:amplitudefollower
+vamp-example-plugins:fixedtempo
+vamp-example-plugins:percussiononsets
+vamp-example-plugins:powerspectrum
+vamp-example-plugins:spectralcentroid
+vamp-example-plugins:zerocrossing
+vamp-rubberband:rubberband
+
+
+
+plug = vampyhost.load_plugin("vamp-example-plugins:fixedtempo", 22050, 0)
+plug.info
+{'apiVersion': 2, 'pluginVersion': 1, 'identifier': 'fixedtempo', 'name': 'Simple Fixed Tempo Estimator', 'description': 'Study a short section of audio and estimate its tempo, assuming the tempo is constant', 'maker': 'Vamp SDK Example Plugins', 'copyright': 'Code copyright 2008 Queen Mary, University of London.  Freely redistributable (BSD license)'}
+plug = vampyhost.load_plugin("qm-vamp-plugins:qm-tempotracker", 22050, 0)
+from pprint import pprint; pprint(plug.parameters)
+
+
+
+for c in plug.parameters: print("{} \033[36m{}  [\033[33m{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], "\033[36m, \033[33m".join(c["valueNames"]), c["valueNames"][int(c["defaultValue"])])) if "valueNames" in c else print("{} \033[36m{}  [\033[33m{}..{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], c["minValue"], c["maxValue"], c["defaultValue"]))
+
+
+
+beatroot-vamp:beatroot
+cl=c["list"]; 60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"])))
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+# 128 bpm, key 5A Cm
+
+import vamp
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40})
+c["list"][0]["label"]
+# 127.6 bpm
+
+c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150})
+print("\n".join([v["label"] for v in c["list"] if v["label"]]))
+v = [float(x["label"].split(' ')[0]) for x in c["list"] if x["label"]]
+v = list(sorted(v))[len(v)//4:-len(v)//4]
+v = sum(v) / len(v)
+# 128.1 bpm
+

docs/notes.sh

@@ -67,6 +67,43 @@ wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:392
 shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*1024*1024)); printf $(tail -c +$((v+1)) "$f" | head -c $((w-v)) | sha512sum | cut -c-64 | sed -r 's/ .*//;s/(..)/\\x\1/g') | base64 -w0 | cut -c-43 | tr '+/' '-_'; v=$w; [ $v -lt $sz ] || break; done; }
 
 
+##
+## poll url for performance issues
+
+command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
+
+
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+
+##
+## sqlite3 stuff
+
+# find dupe metadata keys
+sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = mt2.k and mt1.rowid != mt2.rowid'
+
+# partial reindex by deleting all tags for a list of files
+time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
+cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
+
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
+
+##
+## media
+
+# split track into test files
+e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
+
+-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
+sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
+
+
 ##
 ## vscode
 
@@ -96,6 +133,19 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 brew install python@2
 pip install virtualenv
 
+# readme toc
+cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
+
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
 
 ##
 ## http 206
@@ -121,7 +171,7 @@ Range: bytes=26-         Content-Range: bytes */26
 
 var tsh = [];
 function convert_markdown(md_text, dest_dom) {
-    tsh.push(new Date().getTime());
+    tsh.push(Date.now());
     while (tsh.length > 10)
         tsh.shift();
     if (tsh.length > 1) {

scripts/copyparty-repack.sh

@@ -20,6 +20,7 @@ set -e
 # -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py
 
 
+command -v gnutar && tar() { gnutar "$@"; }
 command -v gtar && tar() { gtar "$@"; }
 command -v gsed && sed() { gsed "$@"; }
 td="$(mktemp -d)"
@@ -29,11 +30,11 @@ pwd
 
 
 dl_text() {
-	command -v curl && exec curl "$@"
+	command -v curl >/dev/null && exec curl "$@"
 	exec wget -O- "$@"
 }
 dl_files() {
-	command -v curl && exec curl -L --remote-name-all "$@"
+	command -v curl >/dev/null && exec curl -L --remote-name-all "$@"
 	exec wget "$@"
 }
 export -f dl_files

scripts/make-sfx.sh

@@ -28,6 +28,13 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	unexpand() { gunexpand "$@"; }
 	command -v grealpath >/dev/null &&
 		realpath() { grealpath "$@"; }
+
+	[ -e /opt/local/bin/bzip2 ] &&
+		bzip2() { /opt/local/bin/bzip2 "$@"; }
+}
+pybin=$(command -v python3 || command -v python) || {
+	echo need python
+	exit 1
 }
 
 [ -e copyparty/__main__.py ] || cd ..
@@ -38,11 +45,17 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	exit 1
 }
 
+use_gz=
+do_sh=1
+do_py=1
 while [ ! -z "$1" ]; do
 	[ "$1" = clean  ] && clean=1  && shift && continue
 	[ "$1" = re     ] && repack=1 && shift && continue
+	[ "$1" = gz     ] && use_gz=1 && shift && continue
 	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
 	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
+	[ "$1" = no-sh  ] && do_sh=   && shift && continue
+	[ "$1" = no-py  ] && do_py=   && shift && continue
 	break
 done
 
@@ -104,7 +117,7 @@ cd sfx
 ver=
 git describe --tags >/dev/null 2>/dev/null && {
 	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
-	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//; s/-g?/./g')";
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//')";
 	t_ver=
 
 	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
@@ -150,7 +163,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -169,10 +182,11 @@ done
 	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
 }
 
+[ $repack ] ||
 find | grep -E '\.py$' |
   grep -vE '__version__' |
   tr '\n' '\0' |
-  xargs -0 python ../scripts/uncomment.py
+  xargs -0 $pybin ../scripts/uncomment.py
 
 f=dep-j2/jinja2/constants.py
 awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
@@ -180,39 +194,74 @@ tmv "$f"
 
 # up2k goes from 28k to 22k laff
 echo entabbening
-find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
+find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
 
+echo gen tarlist
+for d in copyparty dep-j2; do find $d -type f; done |
+sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
+sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
+
+(grep -vE 'gz$' list1; grep -E 'gz$' list1) >list
+
 echo creating tar
 args=(--owner=1000 --group=1000)
 [ "$OSTYPE" = msys ] &&
 	args=()
 
-tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
+tar -cf tar "${args[@]}" --numeric-owner -T list
+
+pc=bzip2
+pe=bz2
+[ $use_gz ] && pc=gzip && pe=gz
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2
-for n in {2..9}; do cp tar t.$n;  xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz
-rm t.*
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2; }
+[ $do_sh ] && { for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
+rm t.* || true
+exts=()
 
+
+[ $do_sh ] && {
+exts+=(.sh)
 echo creating unix sfx
 (
 	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
 	grep -E '^sfx_eof$' -B 9001;
 	cat tar.xz
 ) >$sfx_out.sh
+}
 
-echo creating generic sfx
-python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
-mv sfx.out $sfx_out.py
-chmod 755 $sfx_out.*
+
+[ $do_py ] && {
+	echo creating generic sfx
+
+	py=../scripts/sfx.py
+	suf=
+	[ $use_gz ] && {
+		sed -r 's/"r:bz2"/"r:gz"/' <$py >$py.t
+		py=$py.t
+		suf=-gz
+	}
+
+	$pybin $py --sfx-make tar.bz2 $ver $ts
+	mv sfx.out $sfx_out$suf.py
+	
+	exts+=($suf.py)
+	[ $use_gz ] &&
+		rm $py
+}
+
+
+chmod 755 $sfx_out*
 
 printf "done:\n"
-printf "  %s\n" "$(realpath $sfx_out)."{sh,py}
-# rm -rf *
+for ext in ${exts[@]}; do
+	printf "  %s\n" "$(realpath $sfx_out)"$ext
+done
 
-# tar -tvf ../sfx/tar | sed -r 's/(.* ....-..-.. ..:.. )(.*)/\2 `` \1/' | sort | sed -r 's/(.*) `` (.*)/\2 \1/'| less
-# for n in {1..9}; do tar -tf tar | grep -vE '/$' | sed -r 's/(.*)\.(.*)/\2.\1/' | sort | sed -r 's/([^\.]+)\.(.*)/\2.\1/' | tar -cT- | bzip2 -c$n | wc -c; done 
+# apk add bash python3 tar xz bzip2
+# while true; do ./make-sfx.sh; for f in ..//dist/copyparty-sfx.{sh,py}; do mv $f $f.$(wc -c <$f | awk '{print$1}'); done; done

scripts/sfx.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
-# coding: utf-8
+# coding: latin-1
 from __future__ import print_function, unicode_literals
 
-import os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
+import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform, tempfile, traceback
 import subprocess as sp
 
 """
@@ -27,21 +27,21 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
+WINDOWS = sys.platform in ["win32", "msys"]
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
-cpp = None
 
 
-def eprint(*args, **kwargs):
-    kwargs["file"] = sys.stderr
-    print(*args, **kwargs)
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    print(*a, **ka)
 
 
-def msg(*args, **kwargs):
-    if args:
-        args = ["[SFX]", args[0]] + list(args[1:])
+def msg(*a, **ka):
+    if a:
+        a = ["[SFX]", a[0]] + list(a[1:])
 
-    eprint(*args, **kwargs)
+    eprint(*a, **ka)
 
 
 # skip 1
@@ -156,6 +156,9 @@ def encode(data, size, cksum, ver, ts):
                 skip = True
                 continue
 
+            if ln.strip().startswith("# fmt: "):
+                continue
+
             unpk += ln + "\n"
 
         for k, v in [
@@ -209,11 +212,11 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    hasher = hashlib.md5()
+    h = hashlib.md5()
     for block in yieldfile(fn):
-        hasher.update(block)
+        h.update(block)
 
-    return hasher.hexdigest()
+    return h.hexdigest()
 
 
 def unpack():
@@ -222,9 +225,10 @@ def unpack():
     tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
-    final = os.path.join(top, name)
-    mine = os.path.join(top, withpid)
-    tar = os.path.join(mine, "tar")
+    opj = os.path.join
+    final = opj(top, name)
+    mine = opj(top, withpid)
+    tar = opj(mine, "tar")
 
     try:
         if tag in os.listdir(final):
@@ -233,28 +237,24 @@ def unpack():
     except:
         pass
 
-    nwrite = 0
+    sz = 0
     os.mkdir(mine)
     with open(tar, "wb") as f:
         for buf in get_payload():
-            nwrite += len(buf)
+            sz += len(buf)
             f.write(buf)
 
-    if nwrite != SIZE:
-        t = "\n\n  bad file:\n    expected {} bytes, got {}\n".format(SIZE, nwrite)
-        raise Exception(t)
-
-    cksum = hashfile(tar)
-    if cksum != CKSUM:
-        t = "\n\n  bad file:\n    {} expected,\n    {} obtained\n".format(CKSUM, cksum)
-        raise Exception(t)
+    ck = hashfile(tar)
+    if ck != CKSUM:
+        t = "\n\nexpected {} ({} byte)\nobtained {} ({} byte)\nsfx corrupt"
+        raise Exception(t.format(CKSUM, SIZE, ck, sz))
 
     with tarfile.open(tar, "r:bz2") as tf:
         tf.extractall(mine)
 
     os.remove(tar)
 
-    with open(os.path.join(mine, tag), "wb") as f:
+    with open(opj(mine, tag), "wb") as f:
         f.write(b"h\n")
 
     try:
@@ -272,25 +272,25 @@ def unpack():
     except:
         pass
 
+    for fn in u8(os.listdir(top)):
+        if fn.startswith(name) and fn != withpid:
+            try:
+                old = opj(top, fn)
+                if time.time() - os.path.getmtime(old) > 86400:
+                    shutil.rmtree(old)
+            except:
+                pass
+
     try:
         os.symlink(mine, final)
     except:
         try:
             os.rename(mine, final)
+            return final
         except:
             msg("reloc fail,", mine)
-            return mine
 
-    for fn in u8(os.listdir(top)):
-        if fn.startswith(name) and fn not in [name, withpid]:
-            try:
-                old = os.path.join(top, fn)
-                if time.time() - os.path.getmtime(old) > 10:
-                    shutil.rmtree(old)
-            except:
-                pass
-
-    return final
+    return mine
 
 
 def get_payload():
@@ -307,96 +307,119 @@ def get_payload():
         if ofs < 0:
             raise Exception("could not find archive marker")
 
-        # start reading from the final b"\n"
+        # start at final b"\n"
         fpos = ofs + len(ptn) - 3
-        # msg("tar found at", fpos)
         f.seek(fpos)
         dpos = 0
-        leftovers = b""
+        rem = b""
         while True:
             rbuf = f.read(1024 * 32)
             if rbuf:
-                buf = leftovers + rbuf
+                buf = rem + rbuf
                 ofs = buf.rfind(b"\n")
                 if len(buf) <= 4:
-                    leftovers = buf
+                    rem = buf
                     continue
 
                 if ofs >= len(buf) - 4:
-                    leftovers = buf[ofs:]
+                    rem = buf[ofs:]
                     buf = buf[:ofs]
                 else:
-                    leftovers = b"\n# "
+                    rem = b"\n# "
             else:
-                buf = leftovers
+                buf = rem
 
             fpos += len(buf) + 1
-            buf = (
-                buf.replace(b"\n# ", b"")
-                .replace(b"\n#r", b"\r")
-                .replace(b"\n#n", b"\n")
-            )
-            dpos += len(buf) - 1
+            for a, b in [[b"\n# ", b""], [b"\n#r", b"\r"], [b"\n#n", b"\n"]]:
+                buf = buf.replace(a, b)
 
+            dpos += len(buf) - 1
             yield buf
 
             if not rbuf:
                 break
 
 
-def confirm():
+def utime(top):
+    i = 0
+    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    while WINDOWS:
+        t = int(time.time())
+        if i:
+            msg("utime {}, {}".format(i, t))
+
+        for f in files:
+            os.utime(f, (t, t))
+
+        i += 1
+        time.sleep(78123)
+
+
+def confirm(rv):
     msg()
+    msg("retcode", rv if rv else traceback.format_exc())
     msg("*** hit enter to exit ***")
     try:
         raw_input() if PY2 else input()
     except:
         pass
 
+    sys.exit(rv)
 
-def run(tmp, j2ver):
-    global cpp
 
-    msg("jinja2:", j2ver or "bundled")
+def run(tmp, j2):
+    msg("jinja2:", j2 or "bundled")
     msg("sfxdir:", tmp)
+    msg()
 
-    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    # block systemd-tmpfiles-clean.timer
     try:
         import fcntl
 
         fd = os.open(tmp, os.O_RDONLY)
         fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
-    except:
-        pass
+    except Exception as ex:
+        if not WINDOWS:
+            msg("\033[31mflock:", repr(ex))
+
+    t = threading.Thread(target=utime, args=(tmp,))
+    t.daemon = True
+    t.start()
 
     ld = [tmp, os.path.join(tmp, "dep-j2")]
-    if j2ver:
+    if j2:
         del ld[-1]
 
-    cmd = (
-        "import sys, runpy; "
-        + "".join(['sys.path.insert(0, r"' + x + '"); ' for x in ld])
-        + 'runpy.run_module("copyparty", run_name="__main__")'
-    )
-    cmd = [sys.executable, "-c", cmd] + list(sys.argv[1:])
-
-    cmd = [str(x) for x in cmd]
-    msg("\n", cmd, "\n")
-    cpp = sp.Popen(cmd)
-    try:
-        cpp.wait()
-    except:
-        cpp.wait()
-
-    if cpp.returncode != 0:
-        confirm()
-
-    sys.exit(cpp.returncode)
+    if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
+        run_s(ld)
+    else:
+        run_i(ld)
 
 
-def bye(sig, frame):
-    if cpp is not None:
-        cpp.terminate()
+def run_i(ld):
+    for x in ld:
+        sys.path.insert(0, x)
+
+    from copyparty.__main__ import main as p
+
+    p()
+
+
+def run_s(ld):
+    # fmt: off
+    c = "import sys,runpy;" + "".join(['sys.path.insert(0,r"' + x + '");' for x in ld]) + 'runpy.run_module("copyparty",run_name="__main__")'
+    c = [str(x) for x in [sys.executable, "-c", c] + list(sys.argv[1:])]
+    # fmt: on
+    msg("\n", c, "\n")
+    p = sp.Popen(c)
+
+    def bye(*a):
+        p.send_signal(signal.SIGINT)
+
+    signal.signal(signal.SIGTERM, bye)
+    p.wait()
+
+    raise SystemExit(p.returncode)
 
 
 def main():
@@ -430,16 +453,23 @@ def main():
 
     # skip 0
 
-    signal.signal(signal.SIGTERM, bye)
-
-    tmp = unpack()
+    tmp = os.path.realpath(unpack())
 
     try:
-        from jinja2 import __version__ as j2ver
+        from jinja2 import __version__ as j2
     except:
-        j2ver = None
+        j2 = None
 
-    return run(tmp, j2ver)
+    try:
+        run(tmp, j2)
+    except SystemExit as ex:
+        c = ex.code
+        if c not in [0, -15]:
+            confirm(ex.code)
+    except KeyboardInterrupt:
+        pass
+    except:
+        confirm(0)
 
 
 if __name__ == "__main__":

scripts/speedtest-fs.py

@@ -17,14 +17,15 @@ __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
 
-def get_spd(nbyte, nsec):
+def get_spd(nbyte, nfiles, nsec):
     if not nsec:
-        return "0.000 MB   0.000 sec   0.000 MB/s"
+        return "0.000 MB   0 files   0.000 sec   0.000 MB/s   0.000 f/s"
 
     mb = nbyte / (1024 * 1024.0)
     spd = mb / nsec
+    nspd = nfiles / nsec
 
-    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+    return f"{mb:.3f} MB   {nfiles} files   {nsec:.3f} sec   {spd:.3f} MB/s   {nspd:.3f} f/s"
 
 
 class Inf(object):
@@ -36,6 +37,7 @@ class Inf(object):
         self.mtx_reports = threading.Lock()
 
         self.n_byte = 0
+        self.n_file = 0
         self.n_sec = 0
         self.n_done = 0
         self.t0 = t0
@@ -63,7 +65,8 @@ class Inf(object):
                 continue
 
             msgs = msgs[-64:]
-            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            spd = get_spd(self.n_byte, len(self.reports), self.n_sec)
+            msgs = [f"{spd}   {x}" for x in msgs]
             print("\n".join(msgs))
 
     def report(self, fn, n_byte, n_sec):
@@ -131,8 +134,9 @@ def main():
 
     num_threads = 8
     read_sz = 32 * 1024
+    targs = (q, inf, read_sz)
     for _ in range(num_threads):
-        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr = threading.Thread(target=worker, args=targs)
         thr.daemon = True
         thr.start()
 
@@ -151,14 +155,14 @@ def main():
     log = inf.reports
     log.sort()
     for nbyte, nsec, fn in log[-64:]:
-        print(f"{get_spd(nbyte, nsec)}   {fn}")
+        spd = get_spd(nbyte, len(log), nsec)
+        print(f"{spd}   {fn}")
 
     print()
     print("\n".join(inf.errors))
 
-    print(get_spd(inf.n_byte, t2 - t0))
+    print(get_spd(inf.n_byte, len(log), t2 - t0))
 
 
 if __name__ == "__main__":
     main()
-

tests/run.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import sys
+import runpy
+
+host = sys.argv[1]
+sys.argv = sys.argv[:1] + sys.argv[2:]
+sys.path.insert(0, ".")
+
+
+def rp():
+    runpy.run_module("unittest", run_name="__main__")
+
+
+if host == "vmprof":
+    rp()
+
+elif host == "cprofile":
+    import cProfile
+    import pstats
+
+    log_fn = "cprofile.log"
+    cProfile.run("rp()", log_fn)
+    p = pstats.Stats(log_fn)
+    p.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(64)
+
+
+"""
+python3.9 tests/run.py cprofile -v tests/test_httpcli.py
+
+python3.9 -m pip install --user vmprof
+python3.9 -m vmprof --lines -o vmprof.log tests/run.py vmprof -v tests/test_httpcli.py
+"""

tests/test_httpcli.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import io
+import os
+import time
+import shutil
+import pprint
+import tarfile
+import unittest
+
+from argparse import Namespace
+from copyparty.authsrv import AuthSrv
+from copyparty.httpcli import HttpCli
+
+from tests import util as tu
+
+
+def hdr(query):
+    h = "GET /{} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\n\r\n"
+    return h.format(query).encode("utf-8")
+
+
+class Cfg(Namespace):
+    def __init__(self, a=[], v=[], c=None):
+        super(Cfg, self).__init__(
+            a=a,
+            v=v,
+            c=c,
+            ed=False,
+            no_zip=False,
+            no_scandir=False,
+            no_sendfile=True,
+            nih=True,
+            mtp=[],
+            mte="a",
+            **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        )
+
+
+class TestHttpCli(unittest.TestCase):
+    def test(self):
+        td = os.path.join(tu.get_ramdisk(), "vfs")
+        try:
+            shutil.rmtree(td)
+        except OSError:
+            pass
+
+        os.mkdir(td)
+        os.chdir(td)
+
+        self.dtypes = ["ra", "ro", "rx", "wa", "wo", "wx", "aa", "ao", "ax"]
+        self.can_read = ["ra", "ro", "aa", "ao"]
+        self.can_write = ["wa", "wo", "aa", "ao"]
+        self.fn = "g{:x}g".format(int(time.time() * 3))
+
+        allfiles = []
+        allvols = []
+        for top in self.dtypes:
+            allvols.append(top)
+            allfiles.append("/".join([top, self.fn]))
+            for s1 in self.dtypes:
+                p = "/".join([top, s1])
+                allvols.append(p)
+                allfiles.append(p + "/" + self.fn)
+                allfiles.append(p + "/n/" + self.fn)
+                for s2 in self.dtypes:
+                    p = "/".join([top, s1, "n", s2])
+                    os.makedirs(p)
+                    allvols.append(p)
+                    allfiles.append(p + "/" + self.fn)
+
+        for fp in allfiles:
+            with open(fp, "w") as f:
+                f.write("ok {}\n".format(fp))
+
+        for top in self.dtypes:
+            vcfg = []
+            for vol in allvols:
+                if not vol.startswith(top):
+                    continue
+
+                mode = vol[-2]
+                usr = vol[-1]
+                if usr == "a":
+                    usr = ""
+
+                if "/" not in vol:
+                    vol += "/"
+
+                top, sub = vol.split("/", 1)
+                vcfg.append("{0}/{1}:{1}:{2}{3}".format(top, sub, mode, usr))
+
+            pprint.pprint(vcfg)
+
+            self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
+            self.auth = AuthSrv(self.args, self.log)
+            vfiles = [x for x in allfiles if x.startswith(top)]
+            for fp in vfiles:
+                rok, wok = self.can_rw(fp)
+                furl = fp.split("/", 1)[1]
+                durl = furl.rsplit("/", 1)[0] if "/" in furl else ""
+
+                # file download
+                h, ret = self.curl(furl)
+                res = "ok " + fp in ret
+                print("[{}] {} {} = {}".format(fp, rok, wok, res))
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, furl))
+                    self.fail()
+
+                # file browser: html
+                h, ret = self.curl(durl)
+                res = "'{}'".format(self.fn) in ret
+                print(res)
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, durl))
+                    self.fail()
+
+                # file browser: json
+                url = durl + "?ls"
+                h, ret = self.curl(url)
+                res = '"{}"'.format(self.fn) in ret
+                print(res)
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, url))
+                    self.fail()
+
+                # tar
+                url = durl + "?tar"
+                h, b = self.curl(url, True)
+                # with open(os.path.join(td, "tar"), "wb") as f:
+                #    f.write(b)
+                try:
+                    tar = tarfile.open(fileobj=io.BytesIO(b)).getnames()
+                except:
+                    tar = []
+                tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
+                tar = [[x] + self.can_rw(x) for x in tar]
+                tar_ok = [x[0] for x in tar if x[1]]
+                tar_ng = [x[0] for x in tar if not x[1]]
+                self.assertEqual([], tar_ng)
+
+                if durl.split("/")[-1] in self.can_read:
+                    ref = [x for x in vfiles if self.in_dive(top + "/" + durl, x)]
+                    for f in ref:
+                        print("{}: {}".format("ok" if f in tar_ok else "NG", f))
+                    ref.sort()
+                    tar_ok.sort()
+                    self.assertEqual(ref, tar_ok)
+
+                # stash
+                h, ret = self.put(url)
+                res = h.startswith("HTTP/1.1 200 ")
+                self.assertEqual(res, wok)
+
+    def can_rw(self, fp):
+        # lowest non-neutral folder declares permissions
+        expect = fp.split("/")[:-1]
+        for x in reversed(expect):
+            if x != "n":
+                expect = x
+                break
+
+        return [expect in self.can_read, expect in self.can_write]
+
+    def in_dive(self, top, fp):
+        # archiver bails at first inaccessible subvolume
+        top = top.strip("/").split("/")
+        fp = fp.split("/")
+        for f1, f2 in zip(top, fp):
+            if f1 != f2:
+                return False
+
+        for f in fp[len(top) :]:
+            if f == self.fn:
+                return True
+            if f not in self.can_read and f != "n":
+                return False
+
+        return True
+
+    def put(self, url):
+        buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
+        buf = buf.format(url, len(url) + 4).encode("utf-8")
+        conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
+        HttpCli(conn).run()
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def curl(self, url, binary=False):
+        conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
+        HttpCli(conn).run()
+        if binary:
+            h, b = conn.s._reply.split(b"\r\n\r\n", 1)
+            return [h.decode("utf-8"), b]
+
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def log(self, src, msg, c=0):
+        # print(repr(msg))
+        pass

tests/test_vfs.py

@@ -3,22 +3,24 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import time
 import json
 import shutil
 import tempfile
 import unittest
-import subprocess as sp  # nosec
 
 from textwrap import dedent
 from argparse import Namespace
 from copyparty.authsrv import AuthSrv
 from copyparty import util
 
+from tests import util as tu
+
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
 
 
@@ -49,52 +51,11 @@ class TestVFS(unittest.TestCase):
         real = [x[0] for x in real]
         return fsdir, real, virt
 
-    def runcmd(self, *argv):
-        p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
-        stdout, stderr = p.communicate()
-        stdout = stdout.decode("utf-8")
-        stderr = stderr.decode("utf-8")
-        return [p.returncode, stdout, stderr]
-
-    def chkcmd(self, *argv):
-        ok, sout, serr = self.runcmd(*argv)
-        if ok != 0:
-            raise Exception(serr)
-
-        return sout, serr
-
-    def get_ramdisk(self):
-        for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
-            if os.path.exists(vol):
-                return vol
-
-        if os.path.exists("/Volumes"):
-            devname, _ = self.chkcmd("hdiutil", "attach", "-nomount", "ram://8192")
-            devname = devname.strip()
-            print("devname: [{}]".format(devname))
-            for _ in range(10):
-                try:
-                    _, _ = self.chkcmd(
-                        "diskutil", "eraseVolume", "HFS+", "cptd", devname
-                    )
-                    return "/Volumes/cptd"
-                except Exception as ex:
-                    print(repr(ex))
-                    time.sleep(0.25)
-
-            raise Exception("ramdisk creation failed")
-
-        ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
-        try:
-            os.mkdir(ret)
-        finally:
-            return ret
-
     def log(self, src, msg, c=0):
         pass
 
     def test(self):
-        td = os.path.join(self.get_ramdisk(), "vfs")
+        td = os.path.join(tu.get_ramdisk(), "vfs")
         try:
             shutil.rmtree(td)
         except OSError:
@@ -266,7 +227,7 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(list(v1), list(v2))
 
         # config file parser
-        cfg_path = os.path.join(self.get_ramdisk(), "test.cfg")
+        cfg_path = os.path.join(tu.get_ramdisk(), "test.cfg")
         with open(cfg_path, "wb") as f:
             f.write(
                 dedent(

tests/util.py

@@ -0,0 +1,97 @@
+import os
+import time
+import jinja2
+import tempfile
+import subprocess as sp
+
+from copyparty.util import Unrecv
+
+
+J2_ENV = jinja2.Environment(loader=jinja2.BaseLoader)
+J2_FILES = J2_ENV.from_string("{{ files|join('\n') }}")
+
+
+def runcmd(*argv):
+    p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode("utf-8")
+    stderr = stderr.decode("utf-8")
+    return [p.returncode, stdout, stderr]
+
+
+def chkcmd(*argv):
+    ok, sout, serr = runcmd(*argv)
+    if ok != 0:
+        raise Exception(serr)
+
+    return sout, serr
+
+
+def get_ramdisk():
+    for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
+        if os.path.exists(vol):
+            return vol
+
+    if os.path.exists("/Volumes"):
+        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://32768")
+        devname = devname.strip()
+        print("devname: [{}]".format(devname))
+        for _ in range(10):
+            try:
+                _, _ = chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
+                return "/Volumes/cptd"
+            except Exception as ex:
+                print(repr(ex))
+                time.sleep(0.25)
+
+        raise Exception("ramdisk creation failed")
+
+    ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
+    try:
+        os.mkdir(ret)
+    finally:
+        return ret
+
+
+class NullBroker(object):
+    def put(*args):
+        pass
+
+
+class VSock(object):
+    def __init__(self, buf):
+        self._query = buf
+        self._reply = b""
+        self.sendall = self.send
+
+    def recv(self, sz):
+        ret = self._query[:sz]
+        self._query = self._query[sz:]
+        return ret
+
+    def send(self, buf):
+        self._reply += buf
+        return len(buf)
+
+
+class VHttpSrv(object):
+    def __init__(self):
+        self.broker = NullBroker()
+
+        aliases = ["splash", "browser", "browser2", "msg", "md", "mde"]
+        self.j2 = {x: J2_FILES for x in aliases}
+
+
+class VHttpConn(object):
+    def __init__(self, args, auth, log, buf):
+        self.s = VSock(buf)
+        self.sr = Unrecv(self.s)
+        self.addr = ("127.0.0.1", "42069")
+        self.args = args
+        self.auth = auth
+        self.log_func = log
+        self.log_src = "a"
+        self.hsrv = VHttpSrv()
+        self.nbyte = 0
+        self.workload = 0
+        self.t0 = time.time()