v0.13.14

* extract codec and format info
* add FFprobe as fallback when mutagen fails
* add option to blacklist FFprobe for tags

.vscode/launch.json

@@ -14,12 +14,11 @@
                 "-emp",
                 "-e2dsa",
                 "-e2ts",
-                "-a",
-                "ed:wark",
-                "-v",
-                "srv::r:aed:cnodupe",
-                "-v",
-                "dist:dist:r"
+                "-mtp",
+                ".bpm=f,bin/mtag/audio-bpm.py",
+                "-aed:wark",
+                "-vsrv::r:aed:cnodupe",
+                "-vdist:dist:r"
             ]
         },
         {
@@ -41,5 +40,13 @@
                 "${file}"
             ]
         },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
     ]
 }

.vscode/launch.py

@@ -0,0 +1,45 @@
+# takes arguments from launch.json
+# is used by no_dbg in tasks.json
+# launches 10x faster than mspython debugpy
+# and is stoppable with ^C
+
+import re
+import os
+import sys
+
+print(sys.executable)
+
+import shlex
+import jstyleson
+import subprocess as sp
+
+
+with open(".vscode/launch.json", "r", encoding="utf-8") as f:
+    tj = f.read()
+
+oj = jstyleson.loads(tj)
+argv = oj["configurations"][0]["args"]
+
+try:
+    sargv = " ".join([shlex.quote(x) for x in argv])
+    print(sys.executable + " -m copyparty " + sargv + "\n")
+except:
+    pass
+
+argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
+
+if re.search(" -j ?[0-9]", " ".join(argv)):
+    argv = [sys.executable, "-m", "copyparty"] + argv
+    sp.check_call(argv)
+else:
+    sys.path.insert(0, os.getcwd())
+    from copyparty.__main__ import main as copyparty
+
+    try:
+        copyparty(["a"] + argv)
+    except SystemExit as ex:
+        if ex.code:
+            raise
+
+print("\n\033[32mokke\033[0m")
+sys.exit(1)

.vscode/tasks.json

@@ -8,8 +8,11 @@
         },
         {
             "label": "no_dbg",
-            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -e2ts -a ed:wark -v srv::r:aed:cnodupe -v dist:dist:r ;exit 1",
-            "type": "shell"
+            "type": "shell",
+            "command": "${config:python.pythonPath}",
+            "args": [
+                ".vscode/launch.py"
+            ]
         }
     ]
 }

README.md

@@ -6,161 +6,818 @@
 
 ## summary
 
-turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
+turn your phone or raspi into a portable file server with resumable uploads/downloads using *any* web browser
 
-* server runs on anything with `py2.7` or `py3.3+`
-* *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
+* server only needs `py2.7` or `py3.3+`, all dependencies optional
+* browse/upload with IE4 / netscape4.0 on win3.11 (heh)
+* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+` for full speed
 * code standard: `black`
 
+📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [unpost](#unpost) // [thumbnails](#thumbnails) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [md-viewer](#markdown-viewer) // [ie4](#browser-support)
+
+
+## readme toc
+
+* top
+    * **[quickstart](#quickstart)** - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
+        * [on debian](#on-debian) - recommended additional steps on debian
+    * [notes](#notes) - general notes
+    * [status](#status) - feature summary
+    * [testimonials](#testimonials) - small collection of user feedback
+* [bugs](#bugs)
+    * [general bugs](#general-bugs)
+    * [not my bugs](#not-my-bugs)
+* [accounts and volumes](#accounts-and-volumes) - per-folder, per-user permissions
+* [the browser](#the-browser) - accessing a copyparty server using a web-browser
+    * [tabs](#tabs) - the main tabs in the ui
+    * [hotkeys](#hotkeys) - the browser has the following hotkeys
+    * [navpane](#navpane) - switching between breadcrumbs or navpane
+    * [thumbnails](#thumbnails) - press `g` to toggle grid-view instead of the file listing
+    * [zip downloads](#zip-downloads) - download folders (or file selections) as `zip` or `tar` files
+    * [uploading](#uploading) - web-browsers can upload using `bup` and `up2k`
+        * [file-search](#file-search) - drop files/folders into up2k to see if they exist on the server
+        * [unpost](#unpost) - undo/delete accidental uploads
+    * [file manager](#file-manager) - cut/paste, rename, and delete files/folders (if you have permission)
+    * [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
+    * [markdown viewer](#markdown-viewer) - and there are *two* editors
+    * [other tricks](#other-tricks)
+    * [searching](#searching) - search by size, date, path/name, mp3-tags, ...
+* [server config](#server-config)
+    * [file indexing](#file-indexing)
+    * [upload rules](#upload-rules) - set upload rules using volume flags
+    * [compress uploads](#compress-uploads) - files can be autocompressed on upload
+    * [database location](#database-location) - in-volume (`.hist/up2k.db`, default) or somewhere else
+    * [metadata from audio files](#metadata-from-audio-files) - set `-e2t` to index tags on upload
+    * [file parser plugins](#file-parser-plugins) - provide custom parsers to index additional tags
+    * [complete examples](#complete-examples)
+* [browser support](#browser-support) - TLDR: yes
+* [client examples](#client-examples) - interact with copyparty using non-browser clients
+* [up2k](#up2k) - quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+    * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
+* [performance](#performance) - defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
+* [dependencies](#dependencies) - mandatory deps
+    * [optional dependencies](#optional-dependencies) - install these to enable bonus features
+    * [install recommended deps](#install-recommended-deps)
+    * [optional gpl stuff](#optional-gpl-stuff)
+* [sfx](#sfx) - there are two self-contained "binaries"
+    * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
+* [install on android](#install-on-android)
+* [building](#building)
+    * [dev env setup](#dev-env-setup)
+    * [just the sfx](#just-the-sfx)
+    * [complete release](#complete-release)
+* [todo](#todo) - roughly sorted by priority
+    * [discarded ideas](#discarded-ideas)
+
 
 ## quickstart
 
-download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and you're all set!
+download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
 
-running the sfx without arguments (for example doubleclicking it on Windows) will let anyone access the current folder; see `-h` for help if you want accounts and volumes etc
+running the sfx without arguments (for example doubleclicking it on Windows) will give everyone read/write access to the current folder; see `-h` for help if you want [accounts and volumes](#accounts-and-volumes) etc
+
+some recommended options:
+* `-e2dsa` enables general [file indexing](#file-indexing)
+* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
+* `-v /mnt/music:/music:r:rw,foo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, and read-write for user `foo`, password `bar`
+  * replace `:r:rw,foo` with `:r,foo` to only make the folder readable by `foo` and nobody else
+  * see [accounts and volumes](#accounts-and-volumes) for the syntax and other access levels (`r`ead, `w`rite, `m`ove, `d`elete)
+* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access
 
 you may also want these, especially on servers:
 * [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
-* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for legit https)
+* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
+
+
+### on debian
+
+recommended additional steps on debian  which enable audio metadata and thumbnails (from images and videos):
+
+* as root, run the following:  
+  `apt install python3 python3-pip python3-dev ffmpeg`
+
+* then, as the user which will be running copyparty (so hopefully not root), run this:  
+  `python3 -m pip install --user -U Pillow pillow-avif-plugin`
+
+(skipped `pyheif-pillow-opener` because apparently debian is too old to build it)
 
 
 ## notes
 
-* iPhone/iPad: use Firefox to download files
-* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
-* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
-* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+general notes:
 * paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
   * because no browsers currently implement the media-query to do this properly orz
 
+browser-specific:
+* iPhone/iPad: use Firefox to download files
+* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
+* Android-Firefox: takes a while to select files (their fix for ☝️)
+* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
+* Desktop-Firefox: may stop you from deleting files you've uploaded until you visit `about:memory` and click `Minimize memory usage`
+
 
 ## status
 
+feature summary
+
 * backend stuff
   * ☑ sanic multipart parser
-  * ☑ load balancer (multiprocessing)
+  * ☑ multiprocessing (actual multithreading)
   * ☑ volumes (mountpoints)
-  * ☑ accounts
+  * ☑ [accounts](#accounts-and-volumes)
 * upload
   * ☑ basic: plain multipart, ie6 support
-  * ☑ up2k: js, resumable, multithreaded
+  * ☑ [up2k](#uploading): js, resumable, multithreaded
   * ☑ stash: simple PUT filedropper
+  * ☑ [unpost](#unpost): undo/delete accidental uploads
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ [folders as zip / tar files](#zip-downloads)
   * ☑ FUSE client (read-only)
 * browser
-  * ☑ tree-view
-  * ☑ media player
-  * ✖ thumbnails
-  * ✖ SPA (browse while uploading)
-    * currently safe using the file-tree on the left only, not folders in the file list
+  * ☑ [navpane](#navpane) (directory tree sidebar)
+  * ☑ file manager (cut/paste, delete, [batch-rename](#batch-rename))
+  * ☑ audio player (with OS media controls)
+  * ☑ image gallery with webm player
+  * ☑ [thumbnails](#thumbnails)
+    * ☑ ...of images using Pillow
+    * ☑ ...of videos using FFmpeg
+    * ☑ cache eviction (max-age; maybe max-size eventually)
+  * ☑ SPA (browse while uploading)
+    * if you use the navpane to navigate, not folders in the file list
 * server indexing
-  * ☑ locate files by contents
+  * ☑ [locate files by contents](#file-search)
   * ☑ search by name/path/date/size
-  * ☑ search by ID3-tags etc.
+  * ☑ [search by ID3-tags etc.](#searching)
 * markdown
-  * ☑ viewer
+  * ☑ [viewer](#markdown-viewer)
   * ☑ editor (sure why not)
 
-summary: it works! you can use it! (but technically not even close to beta)
+
+## testimonials
+
+small collection of user feedback
+
+`good enough`, `surprisingly correct`, `certified good software`, `just works`, `why`
 
 
 # bugs
 
-* probably, pls let me know
+* Windows: python 3.7 and older cannot read tags with FFprobe, so use Mutagen or upgrade
+* Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* `--th-ff-jpg` may fix video thumbnails on some FFmpeg versions (macos, some linux)
+
+## general bugs
+
+* all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* probably more, pls let me know
+
+## not my bugs
+
+* Windows: folders cannot be accessed if the name ends with `.`
+  * python or windows bug
+
+* Windows: msys2-python 3.8.6 occasionally throws `RuntimeError: release unlocked lock` when leaving a scoped mutex in up2k
+  * this is an msys2 bug, the regular windows edition of python is fine
+
+* VirtualBox: sqlite throws `Disk I/O Error` when running in a VM and the up2k database is in a vboxsf
+  * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db inside the vm instead
 
 
-# searching
+# accounts and volumes
+
+per-folder, per-user permissions
+* `-a usr:pwd` adds account `usr` with password `pwd`
+* `-v .::r` adds current-folder `.` as the webroot, `r`eadable by anyone
+  * the syntax is `-v src:dst:perm:perm:...` so local-path, url-path, and one or more permissions to set
+  * granting the same permissions to multiple accounts:  
+    `-v .::r,usr1,usr2:rw,usr3,usr4` = usr1/2 read-only, 3/4 read-write
+
+permissions:
+* `r` (read): browse folder contents, download files, download as zip/tar
+* `w` (write): upload files, move files *into* this folder
+* `m` (move): move files/folders *from* this folder
+* `d` (delete): delete files/folders
+
+examples:
+* add accounts named u1, u2, u3 with passwords p1, p2, p3: `-a u1:p1 -a u2:p2 -a u3:p3`
+* make folder `/srv` the root of the filesystem, read-only by anyone: `-v /srv::r`
+* make folder `/mnt/music` available at `/music`, read-only for u1 and u2, read-write for u3: `-v /mnt/music:music:r,u1,u2:rw,u3`
+  * unauthorized users accessing the webroot can see that the `music` folder exists, but cannot open it
+* make folder `/mnt/incoming` available at `/inc`, write-only for u1, read-move for u2: `-v /mnt/incoming:inc:w,u1:rm,u2`
+  * unauthorized users accessing the webroot can see that the `inc` folder exists, but cannot open it
+  * `u1` can open the `inc` folder, but cannot see the contents, only upload new files to it
+  * `u2` can browse it and move files *from* `/inc` into any folder where `u2` has write-access
+
+
+# the browser
+
+accessing a copyparty server using a web-browser
+
+![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/129635359-d6dd9b07-8079-4020-ad77-2bfdb9ebd8d5.png)
+
+
+## tabs
+
+the main tabs in the ui
+* `[🔎]` [search](#searching) by size, date, path/name, mp3-tags ...
+* `[🧯]` [unpost](#unpost): undo/delete accidental uploads
+* `[🚀]` and `[🎈]` are the [uploaders](#uploading)
+* `[📂]` mkdir: create directories
+* `[📝]` new-md: create a new markdown document
+* `[📟]` send-msg: either to server-log or into textfiles if `--urlform save`
+* `[🎺]` audio-player config options
+* `[⚙️]` general client config options
+
+
+## hotkeys
+
+the browser has the following hotkeys  (always qwerty)
+* `B` toggle breadcrumbs / [navpane](#navpane)
+* `I/K` prev/next folder
+* `M` parent folder (or unexpand current)
+* `G` toggle list / [grid view](#thumbnails)
+* `T` toggle thumbnails / icons
+* `ctrl-X` cut selected files/folders
+* `ctrl-V` paste
+* `F2` [rename](#batch-rename) selected file/folder
+* when a file/folder is selected (in not-grid-view):
+  * `Up/Down` move cursor
+  * shift+`Up/Down` select and move cursor
+  * ctrl+`Up/Down` move cursor and scroll viewport
+  * `Space` toggle file selection
+  * `Ctrl-A` toggle select all
+* when playing audio:
+  * `J/L` prev/next song
+  * `U/O` skip 10sec back/forward
+  * `0..9` jump to 0%..90%
+  * `P` play/pause (also starts playing the folder)
+* when viewing images / playing videos:
+  * `J/L, Left/Right` prev/next file
+  * `Home/End` first/last file
+  * `S` toggle selection
+  * `R` rotate clockwise (shift=ccw)
+  * `Esc` close viewer
+  * videos:
+    * `U/O` skip 10sec back/forward
+    * `P/K/Space` play/pause
+    * `F` fullscreen
+    * `C` continue playing next video
+    * `V` loop
+    * `M` mute
+* when the navpane is open:
+  * `A/D` adjust tree width
+* in the [grid view](#thumbnails):
+  * `S` toggle multiselect
+  * shift+`A/D` zoom
+* in the markdown editor:
+  * `^s` save
+  * `^h` header
+  * `^k` autoformat table
+  * `^u` jump to next unicode character
+  * `^e` toggle editor / preview
+  * `^up, ^down` jump paragraphs
+
+
+## navpane
+
+switching between breadcrumbs or navpane
+
+click the `🌲` or pressing the `B` hotkey to toggle between breadcrumbs path (default), or a navpane (tree-browser sidebar thing)
+
+* `[-]` and `[+]` (or hotkeys `A`/`D`) adjust the size
+* `[v]` jumps to the currently open folder
+* `[a]` toggles automatic widening as you go deeper
+
+
+## thumbnails
+
+press `g` to toggle grid-view instead of the file listing,  and `t` toggles icons / thumbnails
+
+![copyparty-thumbs-fs8](https://user-images.githubusercontent.com/241032/129636211-abd20fa2-a953-4366-9423-1c88ebb96ba9.png)
+
+it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how dangerous your users are
+
+images with the following names (see `--th-covers`) become the thumbnail of the folder they're in: `folder.png`, `folder.jpg`, `cover.png`, `cover.jpg`
+
+in the grid/thumbnail view, if the audio player panel is open, songs will start playing when clicked
+
+
+## zip downloads
+
+download folders (or file selections) as `zip` or `tar` files
+
+select which type of archive you want in the `[⚙️] config` tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * `up2k.db` and `dir.txt` is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * this is only to support MS-DOS PKZIP v2.04g (october 1993) and older
+    * how are you accessing copyparty actually
+
+you can also zip a selection of files or folders by clicking them in the browser, that brings up a selection editor and zip button in the bottom right
+
+![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/129635374-e5136e01-470a-49b1-a762-848e8a4c9cdc.png)
+
+
+## uploading
+
+web-browsers can upload using `bup` and `up2k`:
+* `[🎈] bup`, the basic uploader, supports almost every browser since netscape 4.0
+* `[🚀] up2k`, the fancy one
+
+you can undo/delete uploads using `[🧯]` [unpost](#unpost)
+
+up2k has several advantages:
+* you can drop folders into the browser (files are added recursively)
+* files are processed in chunks, and each chunk is checksummed
+  * uploads autoresume if they are interrupted by network issues
+  * uploads resume if you reboot your browser or pc, just upload the same files again
+  * server detects any corruption; the client reuploads affected chunks
+  * the client doesn't upload anything that already exists on the server
+* much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
+* the last-modified timestamp of the file is preserved
+
+see [up2k](#up2k) for details on how it works
+
+![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/129635371-48fc54ca-fa91-48e3-9b1d-ba413e4b68cb.png)
+
+**protip:** you can avoid scaring away users with [docs/minimal-up2k.html](docs/minimal-up2k.html) which makes it look [much simpler](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+the up2k UI is the epitome of polished inutitive experiences:
+* "parallel uploads" specifies how many chunks to upload at the same time
+* `[🏃]` analysis of other files should continue while one is uploading
+* `[💭]` ask for confirmation before files are added to the queue
+* `[💤]` sync uploading between other copyparty browser-tabs so only one is active
+* `[🔎]` switch between upload and [file-search](#file-search) mode
+
+and then theres the tabs below it,
+* `[ok]` is the files which completed successfully
+* `[ng]` is the ones that failed / got rejected (already exists, ...)
+* `[done]` shows a combined list of `[ok]` and `[ng]`, chronological order
+* `[busy]` files which are currently hashing, pending-upload, or uploading
+  * plus up to 3 entries each from `[done]` and `[que]` for context
+* `[que]` is all the files that are still queued
+
+note that since up2k has to read each file twice, `[🎈 bup]` can be up to 2x faster in some extreme cases (huge files combined with internet connection faster than the read-speed of your HDD)
+
+if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
+
+
+### file-search
+
+drop files/folders into up2k to see if they exist on the server
+
+![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/129635361-c79286f0-b8f1-440e-aaf4-6e929428fac9.png)
+
+in the `[🚀 up2k]` tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere
+
+files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
+* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else, this is no longer the case but now i've warmed up to the idea too much
+
+adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files (or just refresh the page)
+
+
+### unpost
+
+undo/delete accidental uploads
+
+![copyparty-unpost-fs8](https://user-images.githubusercontent.com/241032/129635368-3afa6634-c20f-418c-90dc-ec411f3b3897.png)
+
+you can unpost even if you don't have regular move/delete access, however only for files uploaded within the past `--unpost` seconds (default 12 hours) and the server must be running with `-e2d`
+
+
+## file manager
+
+cut/paste, rename, and delete files/folders (if you have permission)
+
+file selection: click somewhere on the line (not the link itsef), then:
+* `space` to toggle
+* `up/down` to move
+* `shift-up/down` to move-and-select
+* `ctrl-shift-up/down` to also scroll
+
+* cut: select some files and `ctrl-x`
+* paste: `ctrl-v` in another folder
+* rename: `F2`
+
+you can move files across browser tabs (cut in one tab, paste in another)
+
+
+## batch rename
+
+select some files and press `F2` to bring up the rename UI
+
+![batch-rename-fs8](https://user-images.githubusercontent.com/241032/128434204-eb136680-3c07-4ec7-92e0-ae86af20c241.png)
+
+quick explanation of the buttons,  
+* `[✅ apply rename]` confirms and begins renaming
+* `[❌ cancel]` aborts and closes the rename window
+* `[↺ reset]` reverts any filename changes back to the original name
+* `[decode]` does a URL-decode on the filename, fixing stuff like `&` and `%20`
+* `[advanced]` toggles advanced mode
+
+advanced mode: rename files based on rules to decide the new names, based on the original name (regex), or based on the tags collected from the file (artist/title/...), or a mix of both
+
+in advanced mode,  
+* `[case]` toggles case-sensitive regex
+* `regex` is the regex pattern to apply to the original filename; any files which don't match will be skipped
+* `format` is the new filename, taking values from regex capturing groups and/or from file tags
+  * very loosely based on foobar2000 syntax
+* `presets` lets you save rename rules for later
+
+available functions:
+* `$lpad(text, length, pad_char)`
+* `$rpad(text, length, pad_char)`
+
+so,
+
+say you have a file named [`meganeko - Eclipse - 07 Sirius A.mp3`](https://www.youtube.com/watch?v=-dtb0vDPruI) (absolutely fantastic album btw) and the tags are: `Album:Eclipse`, `Artist:meganeko`, `Title:Sirius A`, `tn:7`
+
+you could use just regex to rename it:
+* `regex` = `(.*) - (.*) - ([0-9]{2}) (.*)`
+* `format` = `(3). (1) - (4)`
+* `output` = `07. meganeko - Sirius A.mp3`
+
+or you could use just tags:
+* `format` = `$lpad((tn),2,0). (artist) - (title).(ext)`
+* `output` = `7. meganeko - Sirius A.mp3`
+
+or a mix of both:
+* `regex` = ` - ([0-9]{2}) `
+* `format` = `(1). (artist) - (title).(ext)`
+* `output` = `07. meganeko - Sirius A.mp3`
+
+the metadata keys you can use in the format field are the ones in the file-browser table header (whatever is collected with `-mte` and `-mtp`)
+
+
+## markdown viewer
+
+and there are *two* editors
+
+![copyparty-md-read-fs8](https://user-images.githubusercontent.com/241032/115978057-66419080-a57d-11eb-8539-d2be843991aa.png)
+
+* the document preview has a max-width which is the same as an A4 paper when printed
+
+
+## other tricks
+
+* you can link a particular timestamp in an audio file by adding it to the URL, such as `&20` / `&20s` / `&1m20` / `&t=1:20` after the `.../#af-c8960dab`
+
+* if you are using media hotkeys to switch songs and are getting tired of seeing the OSD popup which Windows doesn't let you disable, consider https://ocv.me/dev/?media-osd-bgone.ps1
+
+* click the bottom-left `π` to open a javascript prompt for debugging
+
+* files named `.prologue.html` / `.epilogue.html` will be rendered before/after directory listings unless `--no-logues`
+
+* files named `README.md` / `readme.md` will be rendered after directory listings unless `--no-readme` (but `.epilogue.html` takes precedence)
+
+
+## searching
+
+search by size, date, path/name, mp3-tags, ...
+
+![copyparty-search-fs8](https://user-images.githubusercontent.com/241032/129635365-c0ff2a9f-0ee5-4fc3-8bb6-006033cf67b8.png)
 
 when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
 * make search queries by `size`/`date`/`directory-path`/`filename`, or...
-* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+* drag/drop a local file to see if the same contents exist somewhere on the server, see [file-search](#file-search)
 
 path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
 * path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
 * name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9)
 
-add `-e2ts` to also scan/index tags from music files:
+add the argument `-e2ts` to also scan/index tags from music files, which brings us over to:
 
 
-## search configuration
+# server config
 
-searching relies on two databases, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`). Configuration can be done through arguments, volume flags, or a mix of both.
+## file indexing
+
+file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volume flags, or a mix of both.
 
 through arguments:
 * `-e2d` enables file indexing on upload
-* `-e2ds` scans writable folders on startup
-* `-e2dsa` scans all mounted volumes (including readonly ones)
+* `-e2ds` also scans writable folders for new files on startup
+* `-e2dsa` also scans all mounted volumes (including readonly ones)
 * `-e2t` enables metadata indexing on upload
-* `-e2ts` scans for tags in all files that don't have tags yet
-* `-e2tsr` deletes all existing tags, so a full reindex
+* `-e2ts` also scans for tags in all files that don't have tags yet
+* `-e2tsr` also deletes all existing tags, doing a full reindex
 
 the same arguments can be set as volume flags, in addition to `d2d` and `d2t` for disabling:
-* `-v ~/music::ce2dsa:ce2tsr` does a full reindex of everything on startup
-* `-v ~/music::cd2d` disables **all** indexing, even if any `-e2*` are on
-* `-v ~/music::cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
+* `-v ~/music::r:c,e2dsa:c,e2tsr` does a full reindex of everything on startup
+* `-v ~/music::r:c,d2d` disables **all** indexing, even if any `-e2*` are on
+* `-v ~/music::r:c,d2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 
-`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+note:
+* the parser currently can't handle `c,e2dsa,e2tsr` so you have to `c,e2dsa:c,e2tsr`
+* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those, unless there is a new copyparty version with new parsers and the release note says otherwise
+* the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
+
+to save some time, you can choose to only index filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash` or the volume-flag `:c,dhash`, this has the following consequences:
+* initial indexing is way faster, especially when the volume is on a network disk
+* makes it impossible to [file-search](#file-search)
+* if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
+
+if you set `--no-hash`, you can enable hashing for specific volumes using flag `:c,ehash`
+
+
+## upload rules
+
+set upload rules using volume flags,  some examples:
+
+* `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: b, k, m, g)
+* `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
+* `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
+* `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format
+  * if someone uploads to `/foo/bar` the path would be rewritten to `/foo/bar/2021/08/06/23` for example
+  * but the actual value is not verified, just the structure, so the uploader can choose any values which conform to the format string
+    * just to avoid additional complexity in up2k which is enough of a mess already
+* `:c,lifetime=300` delete uploaded files when they become 5 minutes old
+
+you can also set transaction limits which apply per-IP and per-volume, but these assume `-j 1` (default) otherwise the limits will be off, for example `-j 4` would allow anywhere between 1x and 4x the limits you set depending on which processing node the client gets routed to
+
+* `:c,maxn=250,3600` allows 250 files over 1 hour from each IP (tracked per-volume)
+* `:c,maxb=1g,300` allows 1 GiB total over 5 minutes from each IP (tracked per-volume)
+
+
+## compress uploads
+
+files can be autocompressed on upload,  either on user-request (if config allows) or forced by server-config
+
+* volume flag `gz` allows gz compression
+* volume flag `xz` allows lzma compression
+* volume flag `pk` **forces** compression on all files
+* url parameter `pk` requests compression with server-default algorithm
+* url parameter `gz` or `xz` requests compression with a specific algorithm
+* url parameter `xz` requests xz compression
+
+things to note,
+* the `gz` and `xz` arguments take a single optional argument, the compression level (range 0 to 9)
+* the `pk` volume flag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
+* default compression is gzip level 9
+* all upload methods except up2k are supported
+* the files will be indexed after compression, so dupe-detection and file-search will not work as expected
+
+some examples,
+
+
+## database location
+
+in-volume (`.hist/up2k.db`, default) or somewhere else
+
+copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
+
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+* `--hist ~/.cache/copyparty -v ~/music::r:c,hist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
+
+note:
+* markdown edits are always stored in a local `.hist` subdirectory
+* on windows the volflag path is cyglike, so `/c/temp` means `C:\temp` but use regular paths for `--hist`
+  * you can use cygpaths for volumes too, `-v C:\Users::r` and `-v /c/users::r` both work
+
+
+## metadata from audio files
+
+set `-e2t` to index tags on upload
 
 `-mte` decides which tags to index and display in the browser (and also the display order), this can be changed per-volume:
-* `-v ~/music::cmte=title,artist` indexes and displays *title* followed by *artist*
+* `-v ~/music::r:c,mte=title,artist` indexes and displays *title* followed by *artist*
 
 if you add/remove a tag from `mte` you will need to run with `-e2tsr` once to rebuild the database, otherwise only new files will be affected
 
+but instead of using `-mte`, `-mth` is a better way to hide tags in the browser: these tags will not be displayed by default, but they still get indexed and become searchable, and users can choose to unhide them in the `[⚙️] config` pane
+
 `-mtm` can be used to add or redefine a metadata mapping, say you have media files with `foo` and `bar` tags and you want them to display as `qux` in the browser (preferring `foo` if both are present), then do `-mtm qux=foo,bar` and now you can `-mte artist,title,qux`
 
+tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric value
+
 see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
 
-`--no-mutagen` disables mutagen and uses ffprobe instead, which...
-* is about 20x slower than mutagen
-* catches a few tags that mutagen doesn't
+`--no-mutagen` disables Mutagen and uses FFprobe instead, which...
+* is about 20x slower than Mutagen
+* catches a few tags that Mutagen doesn't
+  * melodic key, video resolution, framerate, pixfmt
 * avoids pulling any GPL code into copyparty
-* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
+* more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve
+
+
+## file parser plugins
+
+provide custom parsers to index additional tags
+
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volume flag), there is a default timeout of 30sec
+
+* `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
+* `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
+* `-v ~/music::r:c,mtp=.bpm=~/bin/audio-bpm.py:c,mtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
+
+*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` only do non-audio files, or `ad` do all files (d as in dontcare) 
+
+* `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`)
+* `-mtp arch,built,ver,orig=an,eexe,edll,~/bin/exe.py` runs `~/bin/exe.py` to get properties about windows-binaries only if file is not audio (`an`) and file extension is exe or dll
+
+
+## complete examples
+
+* read-only music server with bpm and key scanning  
+  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+
+
+# browser support
+
+TLDR: yes
+
+![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/118192791-fb31fe00-b446-11eb-9647-898ea8efc1f7.png)
+
+`ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
+
+| feature         | ie6 | ie9  | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+| --------------- | --- | ---- | ---- | ---- | ----- | ---- | --- | ---- |
+| browse files    | yep | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| thumbnail view  |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| basic uploader  | yep | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| up2k            |  -  |  -   | `*1` | `*1` |  yep  | yep  | yep | yep  |
+| make directory  | yep | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| send message    | yep | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| set sort order  |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| zip selection   |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| file rename     |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| file cut/paste  |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| navpane         |  -  | `*2` | yep  | yep  |  yep  | yep  | yep | yep  |
+| image viewer    |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| video player    |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| markdown editor |  -  |  -   | yep  | yep  |  yep  | yep  | yep | yep  |
+| markdown viewer |  -  |  -   | yep  | yep  |  yep  | yep  | yep | yep  |
+| play mp3/m4a    |  -  | yep  | yep  | yep  |  yep  | yep  | yep | yep  |
+| play ogg/opus   |  -  |  -   |  -   |  -   |  yep  | yep  | `*3` | yep |
+| **= feature =** | ie6 | ie9  | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+
+* internet explorer 6 to 8 behave the same
+* firefox 52 and chrome 49 are the final winxp versions
+* `*1` yes, but extremely slow (ie10: `1 MiB/s`, ie11: `270 KiB/s`)
+* `*2` causes a full-page refresh on each navigation
+* `*3` using a wasm decoder which consumes a bit more power
+
+quick summary of more eccentric web-browsers trying to view a directory index:
+
+| browser | will it blend |
+| ------- | ------------- |
+| **safari** (14.0.3/macos) | is chrome with janky wasm, so playing opus can deadlock the javascript engine |
+| **safari** (14.0.1/iOS)   | same as macos, except it recovers from the deadlocks if you poke it a bit |
+| **links** (2.21/macports) | can browse, login, upload/mkdir/msg |
+| **lynx** (2.8.9/macports) | can browse, login, upload/mkdir/msg |
+| **w3m** (0.5.3/macports)  | can browse, login, upload at 100kB/s, mkdir/msg |
+| **netsurf** (3.10/arch)   | is basically ie6 with much better css (javascript has almost no effect) | 
+| **opera** (11.60/winxp)   | OK: thumbnails, image-viewer, zip-selection, rename/cut/paste. NG: up2k, navpane, markdown, audio |
+| **ie4** and **netscape** 4.0  | can browse (text is yellow on white), upload with `?b=u` |
+| **SerenityOS** (7e98457)  | hits a page fault, works with `?b=u`, file upload not-impl |
 
 
 # client examples
 
+interact with copyparty using non-browser clients
+
 * javascript: dump some state into a file (two separate examples)
   * `await fetch('https://127.0.0.1:3923/', {method:"PUT", body: JSON.stringify(foo)});`
   * `var xhr = new XMLHttpRequest(); xhr.open('POST', 'https://127.0.0.1:3923/msgs?raw'); xhr.send('foo');`
 
 * curl/wget: upload some files (post=file, chunk=stdin)
-  * `post(){ curl -b cppwd=wark http://127.0.0.1:3923/ -F act=bput -F f=@"$1";}`  
+  * `post(){ curl -b cppwd=wark -F act=bput -F f=@"$1" http://127.0.0.1:3923/;}`  
     `post movie.mkv`
-  * `post(){ wget --header='Cookie: cppwd=wark' http://127.0.0.1:3923/?raw --post-file="$1" -O-;}`  
+  * `post(){ wget --header='Cookie: cppwd=wark' --post-file="$1" -O- http://127.0.0.1:3923/?raw;}`  
     `post movie.mkv`
-  * `chunk(){ curl -b cppwd=wark http://127.0.0.1:3923/ -T-;}`  
+  * `chunk(){ curl -b cppwd=wark -T- http://127.0.0.1:3923/;}`  
     `chunk <movie.mkv`
 
 * FUSE: mount a copyparty server as a local filesystem
   * cross-platform python client available in [./bin/](bin/)
   * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
 
+* sharex (screenshot utility): see [./contrib/sharex.sxcu](contrib/#sharexsxcu)
+
 copyparty returns a truncated sha512sum of your PUT/POST as base64; you can generate the same checksum locally to verify uplaods:
 
-    b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|head -c43;}
+    b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|tr '+/' '-_'|head -c44;}
     b512 <movie.mkv
 
+you can provide passwords using cookie 'cppwd=hunter2', as a url query `?pw=hunter2`, or with basic-authentication (either as the username or password)
+
+
+# up2k
+
+quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+* the up2k client splits a file into an "optimal" number of chunks
+  * 1 MiB each, unless that becomes more than 256 chunks
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
+* client posts the list of hashes, filename, size, last-modified
+* server creates the `wark`, an identifier for this upload
+  * `sha512( salt + filesize + chunk_hashes )`
+  * and a sparse file is created for the chunks to drop into
+* client uploads each chunk
+  * header entries for the chunk-hash and wark
+  * server writes chunks into place based on the hash
+* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+
+up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
+
+
+## why chunk-hashes
+
+a single sha512 would be better, right?
+
+this is due to `crypto.subtle` not providing a streaming api (or the option to seed the sha512 hasher with a starting hash)
+
+as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
+
+hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
+
+
+# performance
+
+defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
+
+you can ignore the `cannot efficiently use multiple CPU cores` message, very unlikely to be a problem
+
+below are some tweaks roughly ordered by usefulness:
+
+* `-q` disables logging and can help a bunch, even when combined with `-lo` to redirect logs to file
+* `--http-only` or `--https-only` (unless you want to support both protocols) will reduce the delay before a new connection is established
+* `--hist` pointing to a fast location (ssd) will make directory listings and searches faster when `-e2d` or `-e2t` is set
+* `--no-hash` when indexing a network-disk if you don't care about the actual filehashes and only want the names/tags searchable
+* `-j` enables multiprocessing (actual multithreading) and can make copyparty perform better in cpu-intensive workloads, for example:
+  * huge amount of short-lived connections
+  * really heavy traffic (downloads/uploads)
+  
+  ...however it adds an overhead to internal communication so it might be a net loss, see if it works 4 u
+
+
+# security
+
+some notes on hardening
+
+on public copyparty instances with anonymous upload enabled:
+
+* users can upload html/css/js which will evaluate for other visitors in a few ways,
+  * unless `--no-readme` is set: by uploading/modifying a file named `readme.md`
+  * if `move` access is granted AND none of `--no-logues`, `--no-dot-mv`, `--no-dot-ren` is set: by uploading some .html file and renaming it to `.epilogue.html` (uploading it directly is blocked)
+
+
+## gotchas
+
+behavior that might be unexpected
+
+* users without read-access to a folder can still see the `.prologue.html` / `.epilogue.html` / `README.md` contents, for the purpose of showing a description on how to use the uploader for example
+
 
 # dependencies
 
+mandatory deps:
 * `jinja2` (is built into the SFX)
 
-**optional,** enables music tags:
-* either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
-* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
 
-**optional,** will eventually enable thumbnails:
-* `Pillow` (requires py2.7 or py3.5+)
+## optional dependencies
+
+install these to enable bonus features
+
+enable music tags:
+* either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
+* or `ffprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
+
+enable [thumbnails](#thumbnails) of...
+* **images:** `Pillow` (requires py2.7 or py3.5+)
+* **videos:** `ffmpeg` and `ffprobe` somewhere in `$PATH`
+* **HEIF pictures:** `pyheif-pillow-opener` (requires Linux or a C compiler)
+* **AVIF pictures:** `pillow-avif-plugin`
+
+
+## install recommended deps
+```
+python -m pip install --user -U jinja2 mutagen Pillow
+```
+
+
+## optional gpl stuff
+
+some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
+
+these are standalone programs and will never be imported / evaluated by copyparty, and must be enabled through `-mtp` configs
 
 
 # sfx
 
-currently there are two self-contained binaries:
-* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
-* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
+there are two self-contained "binaries":
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere, **recommended**
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos, kinda deprecated
 
 launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
 
@@ -169,14 +826,18 @@ pls note that `copyparty-sfx.sh` will fail if you rename `copyparty-sfx.py` to `
 
 ## sfx repack
 
-if you don't need all the features you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except for either msys2 or WSL if you're on windows)
-* `724K` original size as of v0.4.0
-* `256K` after `./scripts/make-sfx.sh re no-ogv`
-* `164K` after `./scripts/make-sfx.sh re no-ogv no-cm`
+reduce the size of an sfx by removing features
+
+if you don't need all the features, you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except if you're on windows then you need msys2 or WSL)
+* `525k` size of original sfx.py as of v0.11.30
+* `315k` after `./scripts/make-sfx.sh re no-ogv`
+* `223k` after `./scripts/make-sfx.sh re no-ogv no-cm`
 
 the features you can opt to drop are
-* `ogv`.js, the opus/vorbis decoder which is needed by apple devices to play foss audio files
-* `cm`/easymde, the "fancy" markdown editor
+* `ogv`.js, the opus/vorbis decoder which is needed by apple devices to play foss audio files, saves ~192k
+* `cm`/easymde, the "fancy" markdown editor, saves ~92k
+* `fnt`, source-code-pro, the monospace font, saves ~9k
+* `dd`, the custom mouse cursor for the media player tray tab, saves ~2k
 
 for the `re`pack to work, first run one of the sfx'es once to unpack it
 
@@ -194,24 +855,52 @@ echo $?
 after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
 
 
-# dev env setup
+# building
+
+## dev env setup
+
+mostly optional; if you need a working env for vscode or similar
 
 ```sh
 python3 -m venv .venv
 . .venv/bin/activate
-pip install jinja2  # mandatory deps
-pip install Pillow  # thumbnail deps
+pip install jinja2  # mandatory
+pip install mutagen  # audio metadata
+pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
 pip install black bandit pylint flake8  # vscode tooling
 ```
 
 
-# how to release
+## just the sfx
+
+first grab the web-dependencies from a previous sfx (assuming you don't need to modify something in those):
+
+```sh
+rm -rf copyparty/web/deps
+curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
+python3 x.py -h
+rm x.py
+mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
+```
+
+then build the sfx using any of the following examples:
+
+```sh
+./scripts/make-sfx.sh  # both python and sh editions
+./scripts/make-sfx.sh no-sh gz  # just python with gzip
+```
+
+
+## complete release
+
+also builds the sfx so skip the sfx section above
 
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies
-* create github release with `make-tgz-release.sh`
+* `git tag v1.2.3 && git push origin --tags`
 * upload to pypi with `make-pypi-release.(sh|bat)`
+* create github release with `make-tgz-release.sh`
 * create sfx with `make-sfx.sh`
 
 
@@ -219,15 +908,31 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
+* hls framework for Someone Else to drop code into :^)
+
+
+## discarded ideas
+
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* drop onto folders
+    * not worth the effort, just throw enough conncetions at it
+* single sha512 across all up2k chunks?
+  * crypto.subtle cannot into streaming, would have to use hashwasm, expensive
+* separate sqlite table per tag
+  * performance fixed by skipping some indexes (`+mt.k`)
+* audio fingerprinting
+  * only makes sense if there can be a wasm client and that doesn't exist yet (except for olaf which is agpl hence counts as not existing)
 * `os.copy_file_range` for up2k cloning
+  * almost never hit this path anyways
 * up2k partials ui
-* support pillow-simd
+  * feels like there isn't much point
 * cache sha512 chunks on client
+  * too dangerous
 * comment field
-* ~~look into android thumbnail cache file format~~ bad idea
-* figure out the deal with pixel3a not being connectable as hotspot
-  * pixel3a having unpredictable 3sec latency in general :||||
+  * nah
+* look into android thumbnail cache file format
+  * absolutely not
+* indexedDB for hashes, cfg enable/clear/sz, 2gb avail, ~9k for 1g, ~4k for 100m, 500k items before autoeviction
+  * blank hashlist when up-ok to skip handshake
+    * too many confusing side-effects

bin/README.md

@@ -1,4 +1,4 @@
-# copyparty-fuse.py
+# [`copyparty-fuse.py`](copyparty-fuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
 * **supports Linux** -- expect `117 MiB/s` sequential read
@@ -29,7 +29,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse🅱️.py
+# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
 * mount a copyparty server as a local filesystem (read-only)
 * does the same thing except more correct, `samba` approves
 * **supports Linux** -- expect `18 MiB/s` (wait what)
@@ -37,5 +37,32 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse-streaming.py
+# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
 * pretend this doesn't exist
+
+
+
+# [`mtag/`](mtag/)
+* standalone programs which perform misc. file analysis
+* copyparty can Popen programs like these during file indexing to collect additional metadata
+
+
+# [`dbtool.py`](dbtool.py)
+upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
+
+for that example (upgrading to v0.11.20), first launch the new version of copyparty like usual, let it make a backup of the old db and rebuild the new db until the point where it starts running mtp (colored messages as it adds the mtp tags), that's when you hit CTRL-C and patch in the old mtp tags from the old db instead
+
+so assuming you have `-mtp` parsers to provide the tags `key` and `.bpm`:
+
+```
+cd /mnt/nas/music/.hist
+~/src/copyparty/bin/dbtool.py -ls up2k.db
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -cmp
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy key
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
+```
+
+
+# [`prisonparty.sh`](prisonparty.sh)
+* run copyparty in a chroot, preventing any accidental file access
+* creates bindmounts for /bin, /lib, and so on, see `sysdirs=`

bin/copyparty-fuse-streaming.py

@@ -345,7 +345,7 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, headers={}, **kwargs):
+    def sendreq(self, meth, path, headers, **kwargs):
         if self.password:
             headers["Cookie"] = "=".join(["cppwd", self.password])
 
@@ -354,21 +354,21 @@ class Gateway(object):
             if c.rx_path:
                 raise Exception()
 
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             c.rx = c.getresponse()
             return c
         except:
             tid = threading.current_thread().ident
             dbg(
-                "\033[1;37;44mbad conn {:x}\n  {}\n  {}\033[0m".format(
-                    tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
+                "\033[1;37;44mbad conn {:x}\n  {} {}\n  {}\033[0m".format(
+                    tid, meth, path, c.rx_path if c else "(null)"
                 )
             )
 
         self.closeconn(c)
         c = self.getconn()
         try:
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             c.rx = c.getresponse()
             return c
         except:
@@ -386,7 +386,7 @@ class Gateway(object):
             path = dewin(path)
 
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
-        c = self.sendreq("GET", web_path)
+        c = self.sendreq("GET", web_path, {})
         if c.rx.status != 200:
             self.closeconn(c)
             log(
@@ -440,7 +440,7 @@ class Gateway(object):
             )
         )
 
-        c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        c = self.sendreq("GET", web_path, {"Range": hdr_range})
         if c.rx.status != http.client.PARTIAL_CONTENT:
             self.closeconn(c)
             raise Exception(

bin/copyparty-fuse.py

@@ -54,6 +54,15 @@ MACOS = platform.system() == "Darwin"
 info = log = dbg = None
 
 
+print(
+    "{} v{} @ {}".format(
+        platform.python_implementation(),
+        ".".join([str(x) for x in sys.version_info]),
+        sys.executable,
+    )
+)
+
+
 try:
     from fuse import FUSE, FuseOSError, Operations
 except:
@@ -293,14 +302,14 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, headers={}, **kwargs):
+    def sendreq(self, meth, path, headers, **kwargs):
         tid = get_tid()
         if self.password:
             headers["Cookie"] = "=".join(["cppwd", self.password])
 
         try:
             c = self.getconn(tid)
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             return c.getresponse()
         except:
             dbg("bad conn")
@@ -308,7 +317,7 @@ class Gateway(object):
         self.closeconn(tid)
         try:
             c = self.getconn(tid)
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             return c.getresponse()
         except:
             info("http connection failed:\n" + traceback.format_exc())
@@ -325,7 +334,7 @@ class Gateway(object):
             path = dewin(path)
 
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
-        r = self.sendreq("GET", web_path)
+        r = self.sendreq("GET", web_path, {})
         if r.status != 200:
             self.closeconn()
             log(
@@ -362,7 +371,7 @@ class Gateway(object):
             )
         )
 
-        r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        r = self.sendreq("GET", web_path, {"Range": hdr_range})
         if r.status != http.client.PARTIAL_CONTENT:
             self.closeconn()
             raise Exception(
@@ -1008,6 +1017,12 @@ def main():
         log = null_log
         dbg = null_log
 
+    if ar.a and ar.a.startswith("$"):
+        fn = ar.a[1:]
+        log("reading password from file [{}]".format(fn))
+        with open(fn, "rb") as f:
+            ar.a = f.read().decode("utf-8").strip()
+
     if WINDOWS:
         os.system("rem")
 

bin/dbtool.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import shutil
+import sqlite3
+import argparse
+
+DB_VER1 = 3
+DB_VER2 = 4
+
+
+def die(msg):
+    print("\033[31m\n" + msg + "\n\033[0m")
+    sys.exit(1)
+
+
+def read_ver(db):
+    for tab in ["ki", "kv"]:
+        try:
+            c = db.execute(r"select v from {} where k = 'sver'".format(tab))
+        except:
+            continue
+
+        rows = c.fetchall()
+        if rows:
+            return int(rows[0][0])
+
+    return "corrupt"
+
+
+def ls(db):
+    nfiles = next(db.execute("select count(w) from up"))[0]
+    ntags = next(db.execute("select count(w) from mt"))[0]
+    print(f"{nfiles} files")
+    print(f"{ntags} tags\n")
+
+    print("number of occurences for each tag,")
+    print(" 'x' = file has no tags")
+    print(" 't:mtp' = the mtp flag (file not mtp processed yet)")
+    print()
+    for k, nk in db.execute("select k, count(k) from mt group by k order by k"):
+        print(f"{nk:9} {k}")
+
+
+def compare(n1, d1, n2, d2, verbose):
+    nt = next(d1.execute("select count(w) from up"))[0]
+    n = 0
+    miss = 0
+    for w1, rd, fn in d1.execute("select w, rd, fn from up"):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} files in {n1} against {n2}\033[0m"
+            print(m)
+
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select w from up where rd = ? and fn = ?"
+        hit = d2.execute(q, (rd, fn)).fetchone()
+        if not hit:
+            miss += 1
+            if verbose:
+                print(f"file in {n1} missing in {n2}: [{w1}] {rd}/{fn}")
+
+    print(f" {miss} files in {n1} missing in {n2}\n")
+
+    nt = next(d1.execute("select count(w) from mt"))[0]
+    n = 0
+    miss = {}
+    nmiss = 0
+    for w1, k, v in d1.execute("select * from mt"):
+
+        n += 1
+        if n % 100_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
+            print(m)
+
+        q = "select rd, fn from up where substr(w,1,16) = ?"
+        rd, fn = d1.execute(q, (w1,)).fetchone()
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
+        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if w2:
+            w2 = w2[0]
+
+        v2 = None
+        if w2:
+            v2 = d2.execute(
+                "select v from mt where w = ? and +k = ?", (w2, k)
+            ).fetchone()
+            if v2:
+                v2 = v2[0]
+
+        # if v != v2 and v2 and k in [".bpm", "key"] and n2 == "src":
+        #    print(f"{w} [{rd}/{fn}] {k} = [{v}] / [{v2}]")
+
+        if v2 is not None:
+            if k.startswith("."):
+                try:
+                    diff = abs(float(v) - float(v2))
+                    if diff > float(v) / 0.9:
+                        v2 = None
+                    else:
+                        v2 = v
+                except:
+                    pass
+
+            if v != v2:
+                v2 = None
+
+        if v2 is None:
+            nmiss += 1
+            try:
+                miss[k] += 1
+            except:
+                miss[k] = 1
+
+            if verbose:
+                print(f"missing in {n2}: [{w1}] [{rd}/{fn}] {k} = {v}")
+
+    for k, v in sorted(miss.items()):
+        if v:
+            print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
+
+    print(f"in total, {nmiss} missing tags in {n2}\n")
+
+
+def copy_mtp(d1, d2, tag, rm):
+    nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
+    n = 0
+    ndone = 0
+    for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
+            print(m)
+
+        q = "select rd, fn from up where substr(w,1,16) = ?"
+        rd, fn = d1.execute(q, (w1,)).fetchone()
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
+        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if not w2:
+            continue
+
+        w2 = w2[0]
+        hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
+        if hit:
+            hit = hit[0]
+
+        if hit != v:
+            ndone += 1
+            if hit is not None:
+                d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
+
+            d2.execute("insert into mt values (?,?,?)", (w2, k, v))
+            if rm:
+                d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
+
+    d2.commit()
+    print(f"copied {ndone} {tag} tags over")
+
+
+def main():
+    os.system("")
+    print()
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db", help="database to work on")
+    ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
+
+    ap2 = ap.add_argument_group("informational / read-only stuff")
+    ap2.add_argument("-v", action="store_true", help="verbose")
+    ap2.add_argument("-ls", action="store_true", help="list summary for db")
+    ap2.add_argument("-cmp", action="store_true", help="compare databases")
+
+    ap2 = ap.add_argument_group("options which modify target db")
+    ap2.add_argument("-copy", metavar="TAG", type=str, help="mtp tag to copy over")
+    ap2.add_argument(
+        "-rm-mtp-flag",
+        action="store_true",
+        help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
+    )
+    ap2.add_argument("-vac", action="store_true", help="optimize DB")
+
+    ar = ap.parse_args()
+
+    for v in [ar.db, ar.src]:
+        if v and not os.path.exists(v):
+            die("database must exist")
+
+    db = sqlite3.connect(ar.db)
+    ds = sqlite3.connect(ar.src) if ar.src else None
+
+    # revert journals
+    for d, p in [[db, ar.db], [ds, ar.src]]:
+        if not d:
+            continue
+
+        pj = "{}-journal".format(p)
+        if not os.path.exists(pj):
+            continue
+
+        d.execute("create table foo (bar int)")
+        d.execute("drop table foo")
+
+    if ar.copy:
+        db.close()
+        shutil.copy2(ar.db, "{}.bak.dbtool.{:x}".format(ar.db, int(time.time())))
+        db = sqlite3.connect(ar.db)
+
+    for d, n in [[ds, "src"], [db, "dst"]]:
+        if not d:
+            continue
+
+        ver = read_ver(d)
+        if ver == "corrupt":
+            die("{} database appears to be corrupt, sorry")
+
+        if ver < DB_VER1 or ver > DB_VER2:
+            m = f"{n} db is version {ver}, this tool only supports versions between {DB_VER1} and {DB_VER2}, please upgrade it with copyparty first"
+            die(m)
+
+    if ar.ls:
+        ls(db)
+
+    if ar.cmp:
+        if not ds:
+            die("need src db to compare against")
+
+        compare("src", ds, "dst", db, ar.v)
+        compare("dst", db, "src", ds, ar.v)
+
+    if ar.copy:
+        copy_mtp(ds, db, ar.copy, ar.rm_mtp_flag)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/README.md

@@ -0,0 +1,41 @@
+standalone programs which take an audio file as argument
+
+some of these rely on libraries which are not MIT-compatible
+
+* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
+* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
+* [media-hash.py](./media-hash.py) generates checksums for audio and video streams; uses FFmpeg (LGPL or GPL)
+
+
+# dependencies
+
+run [`install-deps.sh`](install-deps.sh) to build/install most dependencies required by these programs (supports windows/linux/macos)
+
+*alternatively* (or preferably) use packages from your distro instead, then you'll need at least these:
+
+* from distro: `numpy vamp-plugin-sdk beatroot-vamp mixxx-keyfinder ffmpeg`
+* from pypy: `keyfinder vamp`
+
+
+# usage from copyparty
+
+`copyparty -e2dsa -e2ts` followed by any combination of these:
+* `-mtp key=f,audio-key.py`
+* `-mtp .bpm=f,audio-bpm.py`
+* `-mtp ahash,vhash=f,media-hash.py`
+
+* `f,` makes the detected value replace any existing values
+* the `.` in `.bpm` indicates numeric value
+* assumes the python files are in the folder you're launching copyparty from, replace the filename with a relative/absolute path if that's not the case
+* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
+
+
+## usage with volume-flags
+
+instead of affecting all volumes, you can set the options for just one volume like so:
+
+`copyparty -v /mnt/nas/music:/music:r:c,e2dsa:c,e2ts` immediately followed by any combination of these:
+
+* `:c,mtp=key=f,audio-key.py`
+* `:c,mtp=.bpm=f,audio-bpm.py`
+* `:c,mtp=ahash,vhash=f,media-hash.py`

bin/mtag/audio-bpm.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import vamp
+import tempfile
+import numpy as np
+import subprocess as sp
+
+from copyparty.util import fsenc
+
+"""
+dep: vamp
+dep: beatroot-vamp
+dep: ffmpeg
+"""
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-ss", "13",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-ac", "1",
+        "-ar", "22050",
+        "-t", "300",
+        "-f", "f32le",
+        tf
+    ])
+    # fmt: on
+
+    with open(tf, "rb") as f:
+        d = np.fromfile(f, dtype=np.float32)
+        try:
+            # 98% accuracy on jcore
+            c = vamp.collect(d, 22050, "beatroot-vamp:beatroot")
+            cl = c["list"]
+        except:
+            # fallback; 73% accuracy
+            plug = "vamp-example-plugins:fixedtempo"
+            c = vamp.collect(d, 22050, plug, parameters={"maxdflen": 40})
+            print(c["list"][0]["label"].split(" ")[0])
+            return
+
+        # throws if detection failed:
+        bpm = float(cl[-1]["timestamp"] - cl[1]["timestamp"])
+        bpm = round(60 * ((len(cl) - 1) / bpm), 2)
+        print(f"{bpm:.2f}")
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass  # mute
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key-slicing.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+import re
+import os
+import sys
+import tempfile
+import subprocess as sp
+
+import keyfinder
+
+from copyparty.util import fsenc
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+
+note: this is a janky edition of the regular audio-key.py,
+  slicing the files at 20sec intervals and keeping 5sec from each,
+  surprisingly accurate but still garbage (446 ok, 69 bad, 13% miss)
+
+  it is fast tho
+"""
+
+
+def get_duration():
+    # TODO provide ffprobe tags to mtp as json
+
+    # fmt: off
+    dur = sp.check_output([
+        "ffprobe",
+        "-hide_banner",
+        "-v", "fatal",
+        "-show_streams",
+        "-show_format",
+        fsenc(sys.argv[1])
+    ])
+    # fmt: on
+
+    dur = dur.decode("ascii", "replace").split("\n")
+    dur = [x.split("=")[1] for x in dur if x.startswith("duration=")]
+    dur = [float(x) for x in dur if re.match(r"^[0-9\.,]+$", x)]
+    return list(sorted(dur))[-1] if dur else None
+
+
+def get_segs(dur):
+    # keep first 5s of each 20s,
+    # keep entire last segment
+    ofs = 0
+    segs = []
+    while True:
+        seg = [ofs, 5]
+        segs.append(seg)
+        if dur - ofs < 20:
+            seg[-1] = int(dur - seg[0])
+            break
+
+        ofs += 20
+
+    return segs
+
+
+def slice(tf):
+    dur = get_duration()
+    dur = min(dur, 600)  # max 10min
+    segs = get_segs(dur)
+
+    # fmt: off
+    cmd = [
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-y"
+    ]
+
+    for seg in segs:
+        cmd.extend([
+            "-ss", str(seg[0]),
+            "-i", fsenc(sys.argv[1])
+        ])
+    
+    filt = ""
+    for n, seg in enumerate(segs):
+        filt += "[{}:a:0]atrim=duration={}[a{}]; ".format(n, seg[1], n)
+    
+    prev = "a0"
+    for n in range(1, len(segs)):
+        nxt = "b{}".format(n)
+        filt += "[{}][a{}]acrossfade=d=0.5[{}]; ".format(prev, n, nxt)
+        prev = nxt
+
+    cmd.extend([
+        "-filter_complex", filt[:-2],
+        "-map", "[{}]".format(nxt),
+        "-sample_fmt", "s16",
+        tf
+    ])
+    # fmt: on
+
+    # print(cmd)
+    sp.check_call(cmd)
+
+
+def det(tf):
+    slice(tf)
+    print(keyfinder.key(tf).camelot())
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    finally:
+        os.unlink(tf)
+        pass
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import tempfile
+import subprocess as sp
+import keyfinder
+
+from copyparty.util import fsenc
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+"""
+
+
+# tried trimming the first/last 5th, bad idea,
+# misdetects 9a law field (Sphere Caliber) as 10b,
+# obvious when mixing 9a ghostly parapara ship
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-t", "300",
+        "-sample_fmt", "s16",
+        tf
+    ])
+    # fmt: on
+
+    print(keyfinder.key(tf).camelot())
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass  # mute
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/exe.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+
+import sys
+import time
+import json
+import pefile
+
+"""
+retrieve exe info,
+example for multivalue providers
+"""
+
+
+def unk(v):
+    return "unk({:04x})".format(v)
+
+
+class PE2(pefile.PE):
+    def __init__(self, *a, **ka):
+        for k in [
+            # -- parse_data_directories:
+            "parse_import_directory",
+            "parse_export_directory",
+            # "parse_resources_directory",
+            "parse_debug_directory",
+            "parse_relocations_directory",
+            "parse_directory_tls",
+            "parse_directory_load_config",
+            "parse_delay_import_directory",
+            "parse_directory_bound_imports",
+            # -- full_load:
+            "parse_rich_header",
+        ]:
+            setattr(self, k, self.noop)
+
+        super(PE2, self).__init__(*a, **ka)
+
+    def noop(*a, **ka):
+        pass
+
+
+try:
+    pe = PE2(sys.argv[1], fast_load=False)
+except:
+    sys.exit(0)
+
+arch = pe.FILE_HEADER.Machine
+if arch == 0x14C:
+    arch = "x86"
+elif arch == 0x8664:
+    arch = "x64"
+else:
+    arch = unk(arch)
+
+try:
+    buildtime = time.gmtime(pe.FILE_HEADER.TimeDateStamp)
+    buildtime = time.strftime("%Y-%m-%d_%H:%M:%S", buildtime)
+except:
+    buildtime = "invalid"
+
+ui = pe.OPTIONAL_HEADER.Subsystem
+if ui == 2:
+    ui = "GUI"
+elif ui == 3:
+    ui = "cmdline"
+else:
+    ui = unk(ui)
+
+extra = {}
+if hasattr(pe, "FileInfo"):
+    for v1 in pe.FileInfo:
+        for v2 in v1:
+            if v2.name != "StringFileInfo":
+                continue
+
+            for v3 in v2.StringTable:
+                for k, v in v3.entries.items():
+                    v = v.decode("utf-8", "replace").strip()
+                    if not v:
+                        continue
+
+                    if k in [b"FileVersion", b"ProductVersion"]:
+                        extra["ver"] = v
+
+                    if k in [b"OriginalFilename", b"InternalName"]:
+                        extra["orig"] = v
+
+r = {
+    "arch": arch,
+    "built": buildtime,
+    "ui": ui,
+    "cksum": "{:08x}".format(pe.OPTIONAL_HEADER.CheckSum),
+}
+r.update(extra)
+
+print(json.dumps(r, indent=4))

bin/mtag/file-ext.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+
+"""
+example that just prints the file extension
+"""
+
+print(sys.argv[1].split(".")[-1])

bin/mtag/install-deps.sh

@@ -0,0 +1,265 @@
+#!/bin/bash
+set -e
+
+
+# install dependencies for audio-*.py
+#
+# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
+# win64: requires msys2-mingw64 environment
+# macos: requires macports
+#
+# has the following manual dependencies, especially on mac:
+#   https://www.vamp-plugins.org/pack.html
+#
+# installs stuff to the following locations:
+#   ~/pe/
+#   whatever your python uses for --user packages
+#
+# does the following terrible things:
+#   modifies the keyfinder python lib to load the .so in ~/pe
+
+
+linux=1
+
+win=
+[ ! -z "$MSYSTEM" ] || [ -e /msys2.exe ] && {
+	[ "$MSYSTEM" = MINGW64 ] || {
+		echo windows detected, msys2-mingw64 required
+		exit 1
+	}
+	pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	win=1
+	linux=
+}
+
+mac=
+[ $(uname -s) = Darwin ] && {
+	#pybin="$(printf '%s\n' /opt/local/bin/python* | (sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) | (sort -nr || cat) | (sed -E 's/([^ ]*) (.*)/\2\1/' || cat) | grep -E '/(python|pypy)[0-9\.-]*$' | head -n 1)"
+	pybin=/opt/local/bin/python3.9
+	[ -e "$pybin" ] || {
+		echo mac detected, python3 from macports required
+		exit 1
+	}
+	pkgs='ffmpeg python39 py39-wheel'
+	ninst=$(port installed | awk '/^  /{print$1}' | sort | uniq | grep -E '^('"$(echo "$pkgs" | tr ' ' '|')"')$' | wc -l)
+	[ $ninst -eq 3 ] || {
+		sudo port install $pkgs
+	}
+	mac=1
+	linux=
+}
+
+hash -r
+
+[ $mac ] || {
+	command -v python3 && pybin=python3 || pybin=python
+}
+
+$pybin -m pip install --user numpy
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+
+
+need() {
+	command -v $1 >/dev/null || {
+		echo need $1
+		exit 1
+	}
+}
+need cmake
+need ffmpeg
+need $pybin
+#need patchelf
+
+
+td="$(mktemp -d)"
+cln() {
+	rm -rf "$td"
+}
+trap cln EXIT
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	local yolo= ex=
+	[ $1 = "yolo" ] && yolo=1 && ex=k && shift
+	command -v curl >/dev/null && exec curl -${ex}JOL "$@"
+	
+	[ $yolo ] && ex=--no-check-certificate
+	exec wget --trust-server-names $ex "$@"
+}
+export -f dl_files
+
+
+github_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.tarball_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"tarball_url": "/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+gitlab_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.[0].assets.sources[]|select(.format|test("tar.gz")).url' ||
+
+		# fallback to abomination
+		tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	tee links |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+install_keyfinder() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
+	
+	github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
+
+	tar -xf mixxxdj-libkeyfinder-*
+	rm -- *.tar.gz
+	cd mixxxdj-libkeyfinder*
+	
+	h="$HOME"
+	so="lib/libkeyfinder.so"
+	memes=()
+
+	[ $win ] &&
+		so="bin/libkeyfinder.dll" &&
+		h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
+		memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
+	
+	[ $mac ] &&
+		so="lib/libkeyfinder.dylib"
+
+	cmake -DCMAKE_INSTALL_PREFIX="$h/pe/keyfinder" "${memes[@]}" -S . -B build
+	cmake --build build --parallel $(nproc || echo 4)
+	cmake --install build
+
+	libpath="$h/pe/keyfinder/$so"
+	[ $linux ] && [ ! -e "$libpath" ] &&
+		so=lib64/libkeyfinder.so
+	
+	libpath="$h/pe/keyfinder/$so"
+	[ -e "$libpath" ] || {
+		echo "so not found at $sop"
+		exit 1
+	}
+	
+	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
+	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
+	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
+	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
+	$pybin -m pip install --user keyfinder
+
+	pypath="$($pybin -c 'import keyfinder; print(keyfinder.__file__)')"
+	for pyso in "${pypath%/*}"/*.so; do
+		[ -e "$pyso" ] || break
+		patchelf --set-rpath "${libpath%/*}" "$pyso" ||
+			echo "WARNING: patchelf failed (only fatal on musl-based distros)"
+	done
+	
+	mv "$pypath"{,.bak}
+	(
+		printf 'import ctypes\nctypes.cdll.LoadLibrary("%s")\n' "$libpath"
+		cat "$pypath.bak"
+	) >"$pypath"
+
+	echo
+	echo libkeyfinder successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+have_beatroot() {
+	$pybin -c 'import vampyhost, sys; plugs = vampyhost.list_plugins(); sys.exit(0 if "beatroot-vamp:beatroot" in plugs else 1)'
+}
+
+
+install_vamp() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	
+	$pybin -m pip install --user vamp
+
+	have_beatroot || {
+		printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
+		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
+		sha512sum -c <(
+			echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874  -"
+		) <beatroot-vamp-v1.0.tar.gz
+		tar -xf beatroot-vamp-v1.0.tar.gz 
+		cd beatroot-vamp-v1.0
+		make -f Makefile.linux -j4
+		# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
+		mkdir ~/vamp
+		cp -pv beatroot-vamp.* ~/vamp/
+	}
+	
+	have_beatroot &&
+		printf '\033[32mfound the vamp beatroot plugin, nice\033[0m\n' ||
+		printf '\033[31mWARNING: could not find the vamp beatroot plugin, please install it for optimal results\033[0m\n'
+}
+
+
+# not in use because it kinda segfaults, also no windows support
+install_soundtouch() {
+	gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
+	
+	tar -xvf soundtouch-*
+	rm -- *.tar.gz
+	cd soundtouch-*
+	
+	# https://github.com/jrising/pysoundtouch
+	./bootstrap
+	./configure --enable-integer-samples CXXFLAGS="-fPIC" --prefix="$HOME/pe/soundtouch"
+	make -j$(nproc || echo 4)
+	make install
+	
+	CFLAGS=-I$HOME/pe/soundtouch/include/ \
+	LDFLAGS=-L$HOME/pe/soundtouch/lib \
+	$pybin -m pip install --user git+https://github.com/snowxmas/pysoundtouch.git
+	
+	pypath="$($pybin -c 'import importlib; print(importlib.util.find_spec("soundtouch").origin)')"
+	libpath="$(echo "$HOME/pe/soundtouch/lib/")"
+	patchelf --set-rpath "$libpath" "$pypath"
+
+	echo
+	echo soundtouch successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+[ "$1" = keyfinder ] && { install_keyfinder; exit $?; }
+[ "$1" = soundtouch ] && { install_soundtouch; exit $?; }
+[ "$1" = vamp ] && { install_vamp; exit $?; }
+
+echo no args provided, installing keyfinder and vamp
+install_keyfinder
+install_vamp

bin/mtag/media-hash.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import re
+import sys
+import json
+import time
+import base64
+import hashlib
+import subprocess as sp
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p
+
+
+"""
+dep: ffmpeg
+"""
+
+
+def det():
+    # fmt: off
+    cmd = [
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-i", fsenc(sys.argv[1]),
+        "-f", "framemd5",
+        "-"
+    ]
+    # fmt: on
+
+    p = sp.Popen(cmd, stdout=sp.PIPE)
+    # ps = io.TextIOWrapper(p.stdout, encoding="utf-8")
+    ps = p.stdout
+
+    chans = {}
+    for ln in ps:
+        if ln.startswith(b"#stream#"):
+            break
+
+        m = re.match(r"^#media_type ([0-9]): ([a-zA-Z])", ln.decode("utf-8"))
+        if m:
+            chans[m.group(1)] = m.group(2)
+
+    hashers = [hashlib.sha512(), hashlib.sha512()]
+    for ln in ps:
+        n = int(ln[:1])
+        v = ln.rsplit(b",", 1)[-1].strip()
+        hashers[n].update(v)
+
+    r = {}
+    for k, v in chans.items():
+        dg = hashers[int(k)].digest()[:12]
+        dg = base64.urlsafe_b64encode(dg).decode("ascii")
+        r[v[0].lower() + "hash"] = dg
+
+    print(json.dumps(r, indent=4))
+
+
+def main():
+    try:
+        det()
+    except:
+        pass  # mute
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/res/yt-ipr.conf

@@ -0,0 +1,39 @@
+# example config file to use copyparty as a youtube manifest collector,
+# use with copyparty like:  python copyparty.py -c yt-ipr.conf
+#
+# see docs/example.conf for a better explanation of the syntax, but
+# newlines are block separators, so adding blank lines inside a volume definition is bad
+# (use comments as separators instead)
+
+
+# create user ed, password wark
+u ed:wark
+
+
+# create a volume at /ytm which stores files at ./srv/ytm
+./srv/ytm
+/ytm
+# write-only, but read-write for user ed
+w
+rw ed
+# rescan the volume on startup
+c e2dsa
+# collect tags from all new files since last scan
+c e2ts
+# optionally enable compression to make the files 50% smaller
+c pk
+# only allow uploads which are between 16k and 1m large
+c sz=16k-1m
+# allow up to 10 uploads over 5 minutes from each ip
+c maxn=10,300
+# move uploads into subfolders: YEAR-MONTH / DAY-HOUR / <upload>
+c rotf=%Y-%m/%d-%H
+# delete uploads when they are 24 hours old
+c lifetime=86400
+# add the parser and tell copyparty what tags it can expect from it
+c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
+# decide which tags we want to index and in what order
+c mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
+
+
+# create any other volumes you'd like down here, or merge this with an existing config file

bin/mtag/res/yt-ipr.user.js

@@ -0,0 +1,47 @@
+// ==UserScript==
+// @name    youtube-playerdata-hub
+// @match   https://youtube.com/*
+// @match   https://*.youtube.com/*
+// @version 1.0
+// @grant   GM_addStyle
+// ==/UserScript==
+
+function main() {
+    var server = 'https://127.0.0.1:3923/ytm?pw=wark',
+        interval = 60; // sec
+
+    var sent = {};
+    function send(txt, mf_url, desc) {
+        if (sent[mf_url])
+            return;
+
+        fetch(server + '&_=' + Date.now(), { method: "PUT", body: txt });
+        console.log('[yt-pdh] yeet %d bytes, %s', txt.length, desc);
+        sent[mf_url] = 1;
+    }
+
+    function collect() {
+        try {
+            var pd = document.querySelector('ytd-watch-flexy');
+            if (!pd)
+                return console.log('[yt-pdh] no video found');
+
+            pd = pd.playerData;
+            var mu = pd.streamingData.dashManifestUrl || pd.streamingData.hlsManifestUrl;
+            if (!mu || !mu.length)
+                return console.log('[yt-pdh] no manifest found');
+
+            var desc = pd.videoDetails.videoId + ', ' + pd.videoDetails.title;
+            send(JSON.stringify(pd), mu, desc);
+        }
+        catch (ex) {
+            console.log("[yt-pdh]", ex);
+        }
+    }
+    setInterval(collect, interval * 1000);
+}
+
+var scr = document.createElement('script');
+scr.textContent = '(' + main.toString() + ')();';
+(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
+console.log('[yt-pdh] a');

bin/mtag/sleep.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+
+import time
+import random
+
+v = random.random() * 6
+time.sleep(v)
+print(f"{v:.2f}")

bin/mtag/yt-ipr.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+
+import re
+import os
+import sys
+import gzip
+import json
+import base64
+import string
+import urllib.request
+from datetime import datetime
+
+"""
+youtube initial player response
+
+it's probably best to use this through a config file; see res/yt-ipr.conf
+
+but if you want to use plain arguments instead then:
+  -v srv/ytm:ytm:w:rw,ed
+       :c,e2ts:c,e2dsa
+       :c,sz=16k-1m:c,maxn=10,300:c,rotf=%Y-%m/%d-%H
+       :c,mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
+       :c,mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
+
+see res/yt-ipr.user.js for the example userscript to go with this
+"""
+
+
+def main():
+    try:
+        with gzip.open(sys.argv[1], "rt", encoding="utf-8", errors="replace") as f:
+            txt = f.read()
+    except:
+        with open(sys.argv[1], "r", encoding="utf-8", errors="replace") as f:
+            txt = f.read()
+
+    txt = "{" + txt.split("{", 1)[1]
+
+    try:
+        pd = json.loads(txt)
+    except json.decoder.JSONDecodeError as ex:
+        pd = json.loads(txt[: ex.pos])
+
+    # print(json.dumps(pd, indent=2))
+
+    if "videoDetails" in pd:
+        parse_youtube(pd)
+    else:
+        parse_freg(pd)
+
+
+def get_expiration(url):
+    et = re.search(r"[?&]expire=([0-9]+)", url).group(1)
+    et = datetime.utcfromtimestamp(int(et))
+    return et.strftime("%Y-%m-%d, %H:%M")
+
+
+def parse_youtube(pd):
+    vd = pd["videoDetails"]
+    sd = pd["streamingData"]
+
+    et = sd["adaptiveFormats"][0]["url"]
+    et = get_expiration(et)
+
+    mf = []
+    if "dashManifestUrl" in sd:
+        mf.append("dash")
+    if "hlsManifestUrl" in sd:
+        mf.append("hls")
+
+    r = {
+        "yt-id": vd["videoId"],
+        "yt-title": vd["title"],
+        "yt-author": vd["author"],
+        "yt-channel": vd["channelId"],
+        "yt-views": vd["viewCount"],
+        "yt-private": vd["isPrivate"],
+        # "yt-expires": sd["expiresInSeconds"],
+        "yt-manifest": ",".join(mf),
+        "yt-expires": et,
+    }
+    print(json.dumps(r))
+
+    freg_conv(pd)
+
+
+def parse_freg(pd):
+    md = pd["metadata"]
+    r = {
+        "yt-id": md["id"],
+        "yt-title": md["title"],
+        "yt-author": md["channelName"],
+        "yt-channel": md["channelURL"].strip("/").split("/")[-1],
+        "yt-expires": get_expiration(list(pd["video"].values())[0]),
+    }
+    print(json.dumps(r))
+
+
+def freg_conv(pd):
+    # based on getURLs.js v1.5 (2021-08-07)
+    # fmt: off
+    priority = {
+        "video": [
+            337, 315, 266, 138,  # 2160p60
+            313, 336,  # 2160p
+            308,  # 1440p60
+            271, 264,  # 1440p
+            335, 303, 299,  # 1080p60
+            248, 169, 137,  # 1080p
+            334, 302, 298,  # 720p60
+            247, 136  # 720p
+        ],
+        "audio": [
+            251, 141, 171, 140, 250, 249, 139
+        ]
+    }
+
+    vid_id = pd["videoDetails"]["videoId"]
+    chan_id = pd["videoDetails"]["channelId"]
+
+    try:
+        thumb_url = pd["microformat"]["playerMicroformatRenderer"]["thumbnail"]["thumbnails"][0]["url"]
+        start_ts = pd["microformat"]["playerMicroformatRenderer"]["liveBroadcastDetails"]["startTimestamp"]
+    except:
+        thumb_url = f"https://img.youtube.com/vi/{vid_id}/maxresdefault.jpg"
+        start_ts = ""
+
+    # fmt: on
+
+    metadata = {
+        "title": pd["videoDetails"]["title"],
+        "id": vid_id,
+        "channelName": pd["videoDetails"]["author"],
+        "channelURL": "https://www.youtube.com/channel/" + chan_id,
+        "description": pd["videoDetails"]["shortDescription"],
+        "thumbnailUrl": thumb_url,
+        "startTimestamp": start_ts,
+    }
+
+    if [x for x in vid_id if x not in string.ascii_letters + string.digits + "_-"]:
+        print(f"malicious json", file=sys.stderr)
+        return
+
+    basepath = os.path.dirname(sys.argv[1])
+
+    thumb_fn = f"{basepath}/{vid_id}.jpg"
+    tmp_fn = f"{thumb_fn}.{os.getpid()}"
+    if not os.path.exists(thumb_fn) and (
+        thumb_url.startswith("https://img.youtube.com/vi/")
+        or thumb_url.startswith("https://i.ytimg.com/vi/")
+    ):
+        try:
+            with urllib.request.urlopen(thumb_url) as fi:
+                with open(tmp_fn, "wb") as fo:
+                    fo.write(fi.read())
+
+            os.rename(tmp_fn, thumb_fn)
+        except:
+            if os.path.exists(tmp_fn):
+                os.unlink(tmp_fn)
+
+    try:
+        with open(thumb_fn, "rb") as f:
+            thumb = base64.b64encode(f.read()).decode("ascii")
+    except:
+        thumb = "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/yQALCAABAAEBAREA/8wABgAQEAX/2gAIAQEAAD8A0s8g/9k="
+
+    metadata["thumbnail"] = "data:image/jpeg;base64," + thumb
+
+    ret = {
+        "metadata": metadata,
+        "version": "1.5",
+        "createTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
+    }
+
+    for stream, itags in priority.items():
+        for itag in itags:
+            url = None
+            for afmt in pd["streamingData"]["adaptiveFormats"]:
+                if itag == afmt["itag"]:
+                    url = afmt["url"]
+                    break
+
+            if url:
+                ret[stream] = {itag: url}
+                break
+
+    fn = f"{basepath}/{vid_id}.urls.json"
+    with open(fn, "w", encoding="utf-8", errors="replace") as f:
+        f.write(json.dumps(ret, indent=4))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except:
+        # raise
+        pass

bin/prisonparty.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+set -e
+
+# runs copyparty (or any other program really) in a chroot
+#
+# assumption: these directories, and everything within, are owned by root
+sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
+
+
+# error-handler
+help() { cat <<'EOF'
+
+usage:
+  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- copyparty-sfx.py [...]"
+
+example:
+  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- copyparty-sfx.py -v /mnt/nas/music::rwmd"
+
+EOF
+exit 1
+}
+
+
+# read arguments
+trap help EXIT
+jail="$(realpath "$1")"; shift
+uid="$1"; shift
+gid="$1"; shift
+
+vols=()
+while true; do
+	v="$1"; shift
+	[ "$v" = -- ] && break  # end of volumes
+	[ "$#" -eq 0 ] && break  # invalid usage
+	vols+=( "$(realpath "$v")" )
+done
+pybin="$1"; shift
+pybin="$(realpath "$pybin")"
+cpp="$1"; shift
+cpp="$(realpath "$cpp")"
+cppdir="$(dirname "$cpp")"
+trap - EXIT
+
+
+# debug/vis
+echo
+echo "chroot-dir = $jail"
+echo "user:group = $uid:$gid"
+echo " copyparty = $cpp"
+echo
+printf '\033[33m%s\033[0m\n' "copyparty can access these folders and all their subdirectories:"
+for v in "${vols[@]}"; do
+	printf '\033[36m ├─\033[0m %s \033[36m ── added by (You)\033[0m\n' "$v"
+done
+printf '\033[36m ├─\033[0m %s \033[36m ── where the copyparty binary is\033[0m\n' "$cppdir"
+printf '\033[36m ╰─\033[0m %s \033[36m ── the folder you are currently in\033[0m\n' "$PWD"
+vols+=("$cppdir" "$PWD")
+echo
+
+
+# remove any trailing slashes
+jail="${jail%/}"
+cppdir="${cppdir%/}"
+
+
+# bind-mount system directories and volumes
+printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | LC_ALL=C sort |
+while IFS= read -r v; do
+	[ -e "$v" ] || {
+		# printf '\033[1;31mfolder does not exist:\033[0m %s\n' "/$v"
+		continue
+	}
+	i1=$(stat -c%D.%i "$v"      2>/dev/null || echo a)
+	i2=$(stat -c%D.%i "$jail$v" 2>/dev/null || echo b)
+	[ $i1 = $i2 ] && continue
+	
+	mkdir -p "$jail$v"
+	mount --bind "$v" "$jail$v"
+done
+
+
+# create a tmp
+mkdir -p "$jail/tmp"
+chmod 777 "$jail/tmp"
+
+
+# run copyparty
+/sbin/chroot --userspec=$uid:$gid "$jail" "$pybin" "$cpp" "$@" && rv=0 || rv=$?
+
+
+# cleanup if not in use
+lsof "$jail" | grep -qF "$jail" &&
+	echo "chroot is in use, will not cleanup" ||
+{
+	mount | grep -qF " on $jail" |
+	awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
+	LC_ALL=C sort -r  | tee /dev/stderr | tr '\n' '\0' | xargs -r0 umount
+}
+exit $rv

contrib/README.md

@@ -9,6 +9,16 @@
 * assumes the webserver and copyparty is running on the same server/IP
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
+### [`sharex.sxcu`](sharex.sxcu)
+* sharex config file to upload screenshots and grab the URL
+* `RequestURL`: full URL to the target folder
+* `pw`: password (remove the `pw` line if anon-write)
+
+however if your copyparty is behind a reverse-proxy, you may want to use [`sharex-html.sxcu`](sharex-html.sxcu) instead:
+* `RequestURL`: full URL to the target folder
+* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
+* `pw`: password (remove `Parameters` if anon-write)
+
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
 * disables thumbnails and folder-type detection in windows explorer
 * makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
@@ -19,7 +29,8 @@
 
 # OS integration
 init-scripts to start copyparty as a service
-* [`systemd/copyparty.service`](systemd/copyparty.service)
+* [`systemd/copyparty.service`](systemd/copyparty.service) runs the sfx normally
+* [`systemd/prisonparty.service`](systemd/prisonparty.service) runs the sfx in a chroot
 * [`openrc/copyparty`](openrc/copyparty)
 
 # Reverse-proxy

contrib/nginx/copyparty.conf

@@ -1,3 +1,16 @@
+# when running copyparty behind a reverse proxy,
+# the following arguments are recommended:
+#
+#   -nc 512         important, see next paragraph
+#   --http-only     lower latency on initial connection
+#   -i 127.0.0.1    only accept connections from nginx
+#
+# -nc must match or exceed the webserver's max number of concurrent clients;
+# nginx default is 512  (worker_processes 1, worker_connections 512)
+#
+# you may also consider adding -j0 for CPU-intensive configurations
+# (not that i can really think of any good examples)
+
 upstream cpp {
 	server 127.0.0.1:3923;
 	keepalive 120;

contrib/openrc/copyparty

@@ -8,11 +8,11 @@
 #
 # you may want to:
 #   change '/usr/bin/python' to another interpreter
-#   change '/mnt::a' to another location or permission-set
+#   change '/mnt::rw' to another location or permission-set
 
 name="$SVCNAME"
 command_background=true
 pidfile="/var/run/$SVCNAME.pid"
 
 command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
-command_args="-q -v /mnt::a"
+command_args="-q -v /mnt::rw"

contrib/sharex-html.sxcu

@@ -0,0 +1,19 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty-html",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark"
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "RegexList": [
+    "bytes // <a href=\"/([^\"]+)\""
+  ],
+  "URL": "http://127.0.0.1:3923/$regex:1|1$"
+}

contrib/sharex.sxcu

@@ -0,0 +1,17 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark",
+    "j": null
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "URL": "$json:files[0].url$"
+}

contrib/systemd/copyparty.service

@@ -6,13 +6,28 @@
 #
 # you may want to:
 #   change '/usr/bin/python' to another interpreter
-#   change '/mnt::a' to another location or permission-set
+#   change '/mnt::rw' to another location or permission-set
+#
+# with `Type=notify`, copyparty will signal systemd when it is ready to
+#   accept connections; correctly delaying units depending on copyparty.
+#   But note that journalctl will get the timestamps wrong due to
+#   python disabling line-buffering, so messages are out-of-order:
+#   https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
+#
+# enable line-buffering for realtime logging (slight performance cost):
+#   modify ExecStart and prefix it with `/usr/bin/stdbuf -oL` like so:
+#   ExecStart=/usr/bin/stdbuf -oL /usr/bin/python3 [...]
+# but some systemd versions require this instead (higher performance cost):
+#   inside the [Service] block, add the following line:
+#   Environment=PYTHONUNBUFFERED=x
 
 [Unit]
 Description=copyparty file server
 
 [Service]
-ExecStart=/usr/bin/python /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+Type=notify
+SyslogIdentifier=copyparty
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 [Install]

contrib/systemd/prisonparty.service

@@ -0,0 +1,27 @@
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# in a chroot, preventing accidental access elsewhere
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   1) put copyparty-sfx.py and prisonparty.sh in /usr/local/bin
+#   2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
+#
+# you may want to:
+#   change '/mnt::rw' to another location or permission-set
+#    (remember to change the '/mnt' chroot arg too)
+#
+# enable line-buffering for realtime logging (slight performance cost):
+#   inside the [Service] block, add the following line:
+#   Environment=PYTHONUNBUFFERED=x
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+SyslogIdentifier=prisonparty
+WorkingDirectory=/usr/local/bin
+ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
+  /usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
+
+[Install]
+WantedBy=multi-user.target

copyparty/__init__.py

@@ -2,12 +2,16 @@
 from __future__ import print_function, unicode_literals
 
 import platform
+import time
 import sys
 import os
 
 PY2 = sys.version_info[0] == 2
 if PY2:
     sys.dont_write_bytecode = True
+    unicode = unicode
+else:
+    unicode = str
 
 WINDOWS = False
 if platform.system() == "Windows":
@@ -16,21 +20,46 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 
+def get_unix_home():
+    try:
+        v = os.environ["XDG_CONFIG_HOME"]
+        if not v:
+            raise Exception()
+        ret = os.path.normpath(v)
+        os.listdir(ret)
+        return ret
+    except:
+        pass
+
+    try:
+        v = os.path.expanduser("~/.config")
+        if v.startswith("~"):
+            raise Exception()
+        ret = os.path.normpath(v)
+        os.listdir(ret)
+        return ret
+    except:
+        return "/tmp"
+
+
 class EnvParams(object):
     def __init__(self):
+        self.t0 = time.time()
         self.mod = os.path.dirname(os.path.realpath(__file__))
+        if self.mod.endswith("__init__"):
+            self.mod = os.path.dirname(self.mod)
+
         if sys.platform == "win32":
             self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
         elif sys.platform == "darwin":
             self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
         else:
-            self.cfg = os.path.normpath(
-                os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
-                + "/copyparty"
-            )
+            self.cfg = get_unix_home() + "/copyparty"
 
         self.cfg = self.cfg.replace("\\", "/")
         try:

copyparty/__main__.py

@@ -16,12 +16,15 @@ import shutil
 import filecmp
 import locale
 import argparse
+import threading
+import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, VT100
+from .__init__ import E, WINDOWS, VT100, PY2, unicode
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab
+from .util import py_desc, align_tab, IMPLICATIONS, ansi_re
+from .authsrv import re_vol
 
 HAVE_SSL = True
 try:
@@ -29,6 +32,8 @@ try:
 except:
     HAVE_SSL = False
 
+printed = ""
+
 
 class RiceFormatter(argparse.HelpFormatter):
     def _get_help_string(self, action):
@@ -53,6 +58,27 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+class Dodge11874(RiceFormatter):
+    def __init__(self, *args, **kwargs):
+        kwargs["width"] = 9003
+        super(Dodge11874, self).__init__(*args, **kwargs)
+
+
+def lprint(*a, **ka):
+    global printed
+
+    txt = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    printed += txt
+    if not VT100:
+        txt = ansi_re.sub("", txt)
+
+    print(txt, **ka)
+
+
+def warn(msg):
+    lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+
+
 def ensure_locale():
     for x in [
         "en_US.UTF-8",
@@ -61,7 +87,7 @@ def ensure_locale():
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            print("Locale:", x)
+            lprint("Locale:", x)
             break
         except:
             continue
@@ -82,7 +108,7 @@ def ensure_cert():
 
     try:
         if filecmp.cmp(cert_cfg, cert_insec):
-            print(
+            lprint(
                 "\033[33m  using default TLS certificate; https will be insecure."
                 + "\033[36m\n  certificate location: {}\033[0m\n".format(cert_cfg)
             )
@@ -111,7 +137,7 @@ def configure_ssl_ver(al):
     if "help" in sslver:
         avail = [terse_sslver(x[6:]) for x in flags]
         avail = " ".join(sorted(avail) + ["all"])
-        print("\navailable ssl/tls versions:\n  " + avail)
+        lprint("\navailable ssl/tls versions:\n  " + avail)
         sys.exit(0)
 
     al.ssl_flags_en = 0
@@ -131,7 +157,7 @@ def configure_ssl_ver(al):
 
     for k in ["ssl_flags_en", "ssl_flags_de"]:
         num = getattr(al, k)
-        print("{}: {:8x} ({})".format(k, num, num))
+        lprint("{}: {:8x} ({})".format(k, num, num))
 
     # think i need that beer now
 
@@ -148,27 +174,287 @@ def configure_ssl_ciphers(al):
         try:
             ctx.set_ciphers(al.ciphers)
         except:
-            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+            lprint("\n\033[1;31mfailed to set ciphers\033[0m\n")
 
     if not hasattr(ctx, "get_ciphers"):
-        print("cannot read cipher list: openssl or python too old")
+        lprint("cannot read cipher list: openssl or python too old")
     else:
         ciphers = [x["description"] for x in ctx.get_ciphers()]
-        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+        lprint("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
 
     if is_help:
         sys.exit(0)
 
 
-def main():
+def sighandler(sig=None, frame=None):
+    msg = [""] * 5
+    for th in threading.enumerate():
+        msg.append(str(th))
+        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
+
+    msg.append("\n")
+    print("\n".join(msg))
+
+
+def run_argparse(argv, formatter):
+    ap = argparse.ArgumentParser(
+        formatter_class=formatter,
+        prog="copyparty",
+        description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
+    )
+
+    sects = [
+        [
+            "accounts",
+            "accounts and volumes",
+            dedent(
+                """
+            -a takes username:password,
+            -v takes src:dst:perm1:perm2:permN:volflag1:volflag2:volflagN:...
+               where "perm" is "accesslevels,username1,username2,..."
+               and "volflag" is config flags to set on this volume
+            
+            list of accesslevels:
+              "r" (read):   list folder contents, download files
+              "w" (write):  upload files; need "r" to see the uploads
+              "m" (move):   move files and folders; need "w" at destination
+              "d" (delete): permanently delete files and folders
+
+            too many volflags to list here, see the other sections
+
+            example:\033[35m
+              -a ed:hunter2 -v .::r:rw,ed -v ../inc:dump:w:rw,ed:c,nodupe  \033[36m
+              mount current directory at "/" with
+               * r (read-only) for everyone
+               * rw (read+write) for ed
+              mount ../inc at "/dump" with
+               * w (write-only) for everyone
+               * rw (read+write) for ed
+               * reject duplicate files  \033[0m
+            
+            if no accounts or volumes are configured,
+            current folder will be read/write for everyone
+
+            consider the config file for more flexible account/volume management,
+            including dynamic reload at runtime (and being more readable w)
+            """
+            ),
+        ],
+        [
+            "flags",
+            "list of volflags",
+            dedent(
+                """
+            volflags are appended to volume definitions, for example,
+            to create a write-only volume with the \033[33mnodupe\033[0m and \033[32mnosub\033[0m flags:
+              \033[35m-v /mnt/inc:/inc:w\033[33m:c,nodupe\033[32m:c,nosub
+
+            \033[0muploads, general:
+              \033[36mnodupe\033[35m rejects existing files (instead of symlinking them)
+              \033[36mnosub\033[35m forces all uploads into the top folder of the vfs
+              \033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz)
+              \033[36mpk\033[35m forces server-side compression, optional arg: xz,9
+            
+            \033[0mupload rules:
+              \033[36mmaxn=250,600\033[35m max 250 uploads over 15min
+              \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
+              \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
+            
+            \033[0mupload rotation:
+            (moves all uploads into the specified folder structure)
+              \033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each
+              \033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing
+              \033[36mlifetime=3600\033[35m uploads are deleted after 1 hour
+            
+            \033[0mdatabase, general:
+              \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags)
+              \033[36md2t\033[35m disables metadata collection, overrides -e2t*
+              \033[36md2d\033[35m disables all database stuff, overrides -e2*
+              \033[36mdhash\033[35m disables file hashing on initial scans, also ehash
+              \033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location
+              \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
+            
+            \033[0mdatabase, audio tags:
+            "mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...
+              \033[36mmtp=.bpm=f,audio-bpm.py\033[35m uses the "audio-bpm.py" program to
+                generate ".bpm" tags from uploads (f = overwrite tags)
+              \033[36mmtp=ahash,vhash=media-hash.py\033[35m collects two tags at once
+            \033[0m"""
+            ),
+        ],
+        [
+            "urlform",
+            "",
+            dedent(
+                """
+            values for --urlform:
+              \033[36mstash\033[35m dumps the data to file and returns length + checksum
+              \033[36msave,get\033[35m dumps to file and returns the page like a GET
+              \033[36mprint,get\033[35m prints the data in the log and returns GET
+              (leave out the ",get" to return an error instead)
+            """
+            ),
+        ],
+        [
+            "ls",
+            "volume inspection",
+            dedent(
+                """
+            \033[35m--ls USR,VOL,FLAGS
+              \033[36mUSR\033[0m is a user to browse as; * is anonymous, ** is all users
+              \033[36mVOL\033[0m is a single volume to scan, default is * (all vols)
+              \033[36mFLAG\033[0m is flags;
+                \033[36mv\033[0m in addition to realpaths, print usernames and vpaths
+                \033[36mln\033[0m only prints symlinks leaving the volume mountpoint
+                \033[36mp\033[0m exits 1 if any such symlinks are found
+                \033[36mr\033[0m resumes startup after the listing
+            examples:
+              --ls '**'          # list all files which are possible to read
+              --ls '**,*,ln'     # check for dangerous symlinks
+              --ls '**,*,ln,p,r' # check, then start normally if safe
+            """
+            ),
+        ],
+    ]
+
+    # fmt: off
+    u = unicode
+    ap2 = ap.add_argument_group('general options')
+    ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
+    ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
+    ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, USER:PASS; example [ed:wark")
+    ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
+    ap2.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
+    ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
+
+    ap2 = ap.add_argument_group('upload options')
+    ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
+
+    ap2 = ap.add_argument_group('network options')
+    ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
+    
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [help] shows available ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets")
+
+    ap2 = ap.add_argument_group('opt-outs')
+    ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
+    ap2.add_argument("--no-del", action="store_true", help="disable delete operations")
+    ap2.add_argument("--no-mv", action="store_true", help="disable move/rename operations")
+    ap2.add_argument("-nih", action="store_true", help="no info hostname")
+    ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+    ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)")
+
+    ap2 = ap.add_argument_group('safety options')
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
+    ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt")
+    ap2.add_argument("--no-dot-mv", action="store_true", help="disallow moving dotfiles; makes it impossible to move folders containing dotfiles")
+    ap2.add_argument("--no-dot-ren", action="store_true", help="disallow renaming dotfiles; makes it impossible to make something a dotfile")
+    ap2.add_argument("--no-logues", action="store_true", help="disable rendering .prologue/.epilogue.html into directory listings")
+    ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
+
+    ap2 = ap.add_argument_group('logging options')
+    ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
+    ap2.add_argument("--no-voldump", action="store_true", help="do not list volumes and permissions on startup")
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--log-htp", action="store_true", help="print http-server threadpool scaling")
+    ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
+
+    ap2 = ap.add_argument_group('admin panel options')
+    ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
+    ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
+
+    ap2 = ap.add_argument_group('thumbnail options')
+    ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
+    ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
+    ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
+    ap2.add_argument("--th-mt", metavar="CORES", type=int, default=0, help="max num cpu cores to use, 0=all")
+    ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
+    ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
+    ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
+    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
+    ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
+    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval; 0=disabled")
+    ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
+    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
+
+    ap2 = ap.add_argument_group('general db options')
+    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
+    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
+    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
+    ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
+    ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
+    ap2.add_argument("--re-int", metavar="SEC", type=int, default=30, help="disk rescan check interval")
+    ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
+    
+    ap2 = ap.add_argument_group('metadata db options')
+    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
+    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
+    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
+    ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
+    ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
+    ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
+    ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,res,.fps,ahash,vhash")
+    ap2.add_argument("-mth", metavar="M,M,M", type=u, help="tags to hide by default (comma-sep.)",
+        default=".vq,.aq,vc,ac,res,.fps")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", type=u, action="append", help="read tag M using bin")
+
+    ap2 = ap.add_argument_group('appearance options')
+    ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
+
+    ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
+    ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
+    ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
+    ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
+    ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
+    # fmt: on
+
+    ap2 = ap.add_argument_group("help sections")
+    for k, h, _ in sects:
+        ap2.add_argument("--help-" + k, action="store_true", help=h)
+
+    ret = ap.parse_args(args=argv[1:])
+    for k, h, t in sects:
+        k2 = "help_" + k.replace("-", "_")
+        if vars(ret)[k2]:
+            lprint("# {} help page".format(k))
+            lprint(t + "\033[0m")
+            sys.exit(0)
+
+    return ret
+
+
+def main(argv=None):
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
 
+    if argv is None:
+        argv = sys.argv
+
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
     if HAVE_SSL:
@@ -177,109 +463,58 @@ def main():
     deprecated = [["-e2s", "-e2ds"]]
     for dk, nk in deprecated:
         try:
-            idx = sys.argv.index(dk)
+            idx = argv.index(dk)
         except:
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
-        sys.argv[idx] = nk
+        lprint(msg.format(dk, nk))
+        argv[idx] = nk
         time.sleep(2)
 
-    ap = argparse.ArgumentParser(
-        formatter_class=RiceFormatter,
-        prog="copyparty",
-        description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
-        epilog=dedent(
-            """
-            -a takes username:password,
-            -v takes src:dst:permset:permset:cflag:cflag:...
-               where "permset" is accesslevel followed by username (no separator)
-               and "cflag" is config flags to set on this volume
-            
-            list of cflags:
-              "cnodupe" rejects existing files (instead of symlinking them)
+    try:
+        al = run_argparse(argv, RiceFormatter)
+    except AssertionError:
+        al = run_argparse(argv, Dodge11874)
 
-            example:\033[35m
-              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
-              mount current directory at "/" with
-               * r (read-only) for everyone
-               * a (read+write) for ed
-              mount ../inc at "/dump" with
-               * w (write-only) for everyone
-               * a (read+write) for ed
-               * reject duplicate files  \033[0m
-            
-            if no accounts or volumes are configured,
-            current folder will be read/write for everyone
+    nstrs = []
+    anymod = False
+    for ostr in al.v or []:
+        m = re_vol.match(ostr)
+        if not m:
+            # not our problem
+            nstrs.append(ostr)
+            continue
 
-            consider the config file for more flexible account/volume management,
-            including dynamic reload at runtime (and being more readable w)
+        src, dst, perms = m.groups()
+        na = [src, dst]
+        mod = False
+        for opt in perms.split(":"):
+            if re.match("c[^,]", opt):
+                mod = True
+                na.append("c," + opt[1:])
+            elif re.sub("^[rwmd]*", "", opt) and "," not in opt:
+                mod = True
+                perm = opt[0]
+                if perm == "a":
+                    perm = "rw"
+                na.append(perm + "," + opt[1:])
+            else:
+                na.append(opt)
 
-            values for --urlform:
-              "stash" dumps the data to file and returns length + checksum
-              "save,get" dumps to file and returns the page like a GET
-              "print,get" prints the data in the log and returns GET
-              (leave out the ",get" to return an error instead)
+        nstr = ":".join(na)
+        nstrs.append(nstr if mod else ostr)
+        if mod:
+            msg = "\033[1;31mWARNING:\033[0;1m\n  -v {} \033[0;33mwas replaced with\033[0;1m\n  -v {} \n\033[0m"
+            lprint(msg.format(ostr, nstr))
+            anymod = True
 
-            --ciphers help = available ssl/tls ciphers,
-            --ssl-ver help = available ssl/tls versions,
-              default is what python considers safe, usually >= TLS1
-            """
-        ),
-    )
-    # fmt: off
-    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
-    ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
-    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
-    ap.add_argument("-q", action="store_true", help="quiet")
-    ap.add_argument("-ed", action="store_true", help="enable ?dots")
-    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
-    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
-    ap.add_argument("-nih", action="store_true", help="no info hostname")
-    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
-    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
-    ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
-    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
-
-    ap2 = ap.add_argument_group('database options')
-    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
-    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
-    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
-    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
-    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
-    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
-    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
-    ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
-    ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
-    ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
-        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
-
-    ap2 = ap.add_argument_group('SSL/TLS options')
-    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
-    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", type=str, help="ssl/tls versions to allow")
-    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
-    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
-    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
-    
-    al = ap.parse_args()
-    # fmt: on
+    if anymod:
+        al.v = nstrs
+        time.sleep(2)
 
     # propagate implications
-    for k1, k2 in [
-        ["e2dsa", "e2ds"],
-        ["e2ds", "e2d"],
-        ["e2tsr", "e2ts"],
-        ["e2ts", "e2t"],
-        ["e2t", "e2d"],
-    ]:
+    for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):
             setattr(al, k2, True)
 
@@ -300,9 +535,20 @@ def main():
         if al.ciphers:
             configure_ssl_ciphers(al)
     else:
-        print("\033[33m  ssl module does not exist; cannot enable https\033[0m\n")
+        warn("ssl module does not exist; cannot enable https")
 
-    SvcHub(al).run()
+    if PY2 and WINDOWS and al.e2d:
+        warn(
+            "windows py2 cannot do unicode filenames with -e2d\n"
+            + "  (if you crash with codec errors then that is why)"
+        )
+
+    if sys.version_info < (3, 6):
+        al.no_scandir = True
+
+    # signal.signal(signal.SIGINT, sighandler)
+
+    SvcHub(al, argv, printed).run()
 
 
 if __name__ == "__main__":

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 9, 3)
-CODENAME = "the strongest music server"
-BUILD_DT = (2021, 3, 4)
+VERSION = (0, 13, 14)
+CODENAME = "future-proof"
+BUILD_DT = (2021, 9, 6)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/bos/bos.py

@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+from ..util import fsenc, fsdec
+from . import path
+
+
+# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
+# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
+
+
+def chmod(p, mode):
+    return os.chmod(fsenc(p), mode)
+
+
+def listdir(p="."):
+    return [fsdec(x) for x in os.listdir(fsenc(p))]
+
+
+def lstat(p):
+    return os.lstat(fsenc(p))
+
+
+def makedirs(name, mode=0o755, exist_ok=True):
+    bname = fsenc(name)
+    try:
+        os.makedirs(bname, mode=mode)
+    except:
+        if not exist_ok or not os.path.isdir(bname):
+            raise
+
+
+def mkdir(p, mode=0o755):
+    return os.mkdir(fsenc(p), mode=mode)
+
+
+def rename(src, dst):
+    return os.rename(fsenc(src), fsenc(dst))
+
+
+def replace(src, dst):
+    return os.replace(fsenc(src), fsenc(dst))
+
+
+def rmdir(p):
+    return os.rmdir(fsenc(p))
+
+
+def stat(p):
+    return os.stat(fsenc(p))
+
+
+def unlink(p):
+    return os.unlink(fsenc(p))
+
+
+def utime(p, times=None):
+    return os.utime(fsenc(p), times)

copyparty/bos/path.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+from ..util import fsenc, fsdec
+
+
+def abspath(p):
+    return fsdec(os.path.abspath(fsenc(p)))
+
+
+def exists(p):
+    return os.path.exists(fsenc(p))
+
+
+def getmtime(p):
+    return os.path.getmtime(fsenc(p))
+
+
+def getsize(p):
+    return os.path.getsize(fsenc(p))
+
+
+def isdir(p):
+    return os.path.isdir(fsenc(p))
+
+
+def islink(p):
+    return os.path.islink(fsenc(p))
+
+
+def realpath(p):
+    return fsdec(os.path.realpath(fsenc(p)))

copyparty/broker_mp.py

@@ -4,17 +4,11 @@ from __future__ import print_function, unicode_literals
 import time
 import threading
 
-from .__init__ import PY2, WINDOWS, VT100
 from .broker_util import try_exec
 from .broker_mpw import MpWorker
 from .util import mp
 
 
-if PY2 and not WINDOWS:
-    from multiprocessing.reduction import ForkingPickler
-    from StringIO import StringIO as MemesIO  # pylint: disable=import-error
-
-
 class BrokerMp(object):
     """external api; manages MpWorkers"""
 
@@ -28,38 +22,33 @@ class BrokerMp(object):
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
 
-        cores = self.args.j
-        if not cores:
-            cores = mp.cpu_count()
-
-        self.log("broker", "booting {} subprocesses".format(cores))
-        for n in range(cores):
+        self.num_workers = self.args.j or mp.cpu_count()
+        self.log("broker", "booting {} subprocesses".format(self.num_workers))
+        for n in range(1, self.num_workers + 1):
             q_pend = mp.Queue(1)
             q_yield = mp.Queue(64)
 
             proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
             proc.q_pend = q_pend
             proc.q_yield = q_yield
-            proc.nid = n
             proc.clients = {}
-            proc.workload = 0
 
-            thr = threading.Thread(target=self.collector, args=(proc,))
+            thr = threading.Thread(
+                target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
+            )
             thr.daemon = True
             thr.start()
 
             self.procs.append(proc)
             proc.start()
 
-        if True:
-            thr = threading.Thread(target=self.debug_load_balancer)
-            thr.daemon = True
-            thr.start()
-
     def shutdown(self):
         self.log("broker", "shutting down")
-        for proc in self.procs:
-            thr = threading.Thread(target=proc.q_pend.put([0, "shutdown", []]))
+        for n, proc in enumerate(self.procs):
+            thr = threading.Thread(
+                target=proc.q_pend.put([0, "shutdown", []]),
+                name="mp-shutdown-{}-{}".format(n, len(self.procs)),
+            )
             thr.start()
 
         with self.mutex:
@@ -82,20 +71,6 @@ class BrokerMp(object):
             if dest == "log":
                 self.log(*args)
 
-            elif dest == "workload":
-                with self.mutex:
-                    proc.workload = args[0]
-
-            elif dest == "httpdrop":
-                addr = args[0]
-
-                with self.mutex:
-                    del proc.clients[addr]
-                    if not proc.clients:
-                        proc.workload = 0
-
-                self.hub.tcpsrv.num_clients.add(-1)
-
             elif dest == "retq":
                 # response from previous ipc call
                 with self.retpend_mutex:
@@ -121,38 +96,12 @@ class BrokerMp(object):
         returns a Queue object which eventually contains the response if want_retval
         (not-impl here since nothing uses it yet)
         """
-        if dest == "httpconn":
-            sck, addr = args
-            sck2 = sck
-            if PY2:
-                buf = MemesIO()
-                ForkingPickler(buf).dump(sck)
-                sck2 = buf.getvalue()
+        if dest == "listen":
+            for p in self.procs:
+                p.q_pend.put([0, dest, [args[0], len(self.procs)]])
 
-            proc = sorted(self.procs, key=lambda x: x.workload)[0]
-            proc.q_pend.put([0, dest, [sck2, addr]])
-
-            with self.mutex:
-                proc.clients[addr] = 50
-                proc.workload += 50
+        elif dest == "cb_httpsrv_up":
+            self.hub.cb_httpsrv_up()
 
         else:
             raise Exception("what is " + str(dest))
-
-    def debug_load_balancer(self):
-        fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
-        if not VT100:
-            fmt = "({}{:4})"
-
-        last = ""
-        while self.procs:
-            msg = ""
-            for proc in self.procs:
-                msg += fmt.format(len(proc.clients), proc.workload)
-
-            if msg != last:
-                last = msg
-                with self.hub.log_mutex:
-                    print(msg)
-
-            time.sleep(0.1)

copyparty/broker_mpw.py

@@ -2,17 +2,13 @@
 from __future__ import print_function, unicode_literals
 
 import sys
-import time
 import signal
 import threading
 
-from .__init__ import PY2, WINDOWS
 from .broker_util import ExceptionalQueue
 from .httpsrv import HttpSrv
 from .util import FAKE_MP
-
-if PY2 and not WINDOWS:
-    import pickle  # nosec
+from copyparty.authsrv import AuthSrv
 
 
 class MpWorker(object):
@@ -24,39 +20,43 @@ class MpWorker(object):
         self.args = args
         self.n = n
 
+        self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
+
         self.retpend = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
-        self.workload_thr_active = False
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
         if not FAKE_MP:
-            signal.signal(signal.SIGINT, self.signal_handler)
+            for sig in [signal.SIGINT, signal.SIGTERM]:
+                signal.signal(sig, self.signal_handler)
+
+        # starting to look like a good idea
+        self.asrv = AuthSrv(args, None, False)
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
-        self.httpsrv.disconnect_func = self.httpdrop
+        self.httpsrv = HttpSrv(self, n)
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
-        thr = threading.Thread(target=self.main)
+        thr = threading.Thread(target=self.main, name="mpw-main")
         thr.daemon = True
         thr.start()
         thr.join()
 
-    def signal_handler(self, signal, frame):
+    def signal_handler(self, sig, frame):
         # print('k')
         pass
 
-    def log(self, src, msg):
-        self.q_yield.put([0, "log", [src, msg]])
+    def _log_enabled(self, src, msg, c=0):
+        self.q_yield.put([0, "log", [src, msg, c]])
 
-    def logw(self, msg):
-        self.log("mp{}".format(self.n), msg)
+    def _log_disabled(self, src, msg, c=0):
+        pass
 
-    def httpdrop(self, addr):
-        self.q_yield.put([0, "httpdrop", [addr]])
+    def logw(self, msg, c=0):
+        self.log("mp{}".format(self.n), msg, c)
 
     def main(self):
         while True:
@@ -64,24 +64,13 @@ class MpWorker(object):
 
             # self.logw("work: [{}]".format(d[0]))
             if dest == "shutdown":
+                self.httpsrv.shutdown()
                 self.logw("ok bye")
                 sys.exit(0)
                 return
 
-            elif dest == "httpconn":
-                sck, addr = args
-                if PY2:
-                    sck = pickle.loads(sck)  # nosec
-
-                self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
-                self.httpsrv.accept(sck, addr)
-
-                with self.mutex:
-                    if not self.workload_thr_active:
-                        self.workload_thr_alive = True
-                        thr = threading.Thread(target=self.thr_workload)
-                        thr.daemon = True
-                        thr.start()
+            elif dest == "listen":
+                self.httpsrv.listen(args[0], args[1])
 
             elif dest == "retq":
                 # response from previous ipc call
@@ -105,16 +94,3 @@ class MpWorker(object):
 
         self.q_yield.put([retq_id, dest, args])
         return retq
-
-    def thr_workload(self):
-        """announce workloads to MpSrv (the mp controller / loadbalancer)"""
-        # avoid locking in extract_filedata by tracking difference here
-        while True:
-            time.sleep(0.2)
-            with self.mutex:
-                if self.httpsrv.num_clients() == 0:
-                    # no clients rn, termiante thread
-                    self.workload_thr_alive = False
-                    return
-
-            self.q_yield.put([0, "workload", [self.httpsrv.workload]])

copyparty/broker_thr.py

@@ -14,22 +14,22 @@ class BrokerThr(object):
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
+        self.asrv = hub.asrv
 
         self.mutex = threading.Lock()
+        self.num_workers = 1
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
-        self.httpsrv.disconnect_func = self.httpdrop
+        self.httpsrv = HttpSrv(self, None)
 
     def shutdown(self):
         # self.log("broker", "shutting down")
+        self.httpsrv.shutdown()
         pass
 
     def put(self, want_retval, dest, *args):
-        if dest == "httpconn":
-            sck, addr = args
-            self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
-            self.httpsrv.accept(sck, addr)
+        if dest == "listen":
+            self.httpsrv.listen(args[0], 1)
 
         else:
             # new ipc invoking managed service in hub
@@ -46,6 +46,3 @@ class BrokerThr(object):
             retq = ExceptionalQueue(1)
             retq.put(rv)
             return retq
-
-    def httpdrop(self, addr):
-        self.hub.tcpsrv.num_clients.add(-1)

copyparty/httpconn.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
-import sys
 import time
 import socket
 
@@ -12,27 +12,13 @@ try:
 except:
     HAVE_SSL = False
 
-try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * {} -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-""".format(
-            os.path.basename(sys.executable)
-        )
-    )
-    sys.exit(1)
-
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
 from .u2idx import U2idx
+from .th_cli import ThumbCli
+from .th_srv import HAVE_PIL
+from .ico import Ico
 
 
 class HttpConn(object):
@@ -47,23 +33,29 @@ class HttpConn(object):
         self.hsrv = hsrv
 
         self.args = hsrv.args
-        self.auth = hsrv.auth
+        self.asrv = hsrv.asrv
         self.cert_path = hsrv.cert_path
 
+        enth = HAVE_PIL and not self.args.no_thumb
+        self.thumbcli = ThumbCli(hsrv.broker) if enth else None
+        self.ico = Ico(self.args)
+
         self.t0 = time.time()
+        self.stopping = False
+        self.nreq = 0
         self.nbyte = 0
-        self.workload = 0
         self.u2idx = None
         self.log_func = hsrv.log
+        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
         self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
+    def shutdown(self):
+        self.stopping = True
+        try:
+            self.s.shutdown(socket.SHUT_RDWR)
+            self.s.close()
+        except:
+            pass
 
     def set_rproxy(self, ip=None):
         if ip is None:
@@ -81,12 +73,12 @@ class HttpConn(object):
     def respath(self, res_name):
         return os.path.join(E.mod, "web", res_name)
 
-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)
 
     def get_u2idx(self):
         if not self.u2idx:
-            self.u2idx = U2idx(self.args, self.log_func)
+            self.u2idx = U2idx(self)
 
         return self.u2idx
 
@@ -112,7 +104,9 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
-                self.log(err)
+                if method:
+                    self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 
@@ -129,7 +123,7 @@ class HttpConn(object):
 
         if is_https:
             if self.sr:
-                self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
+                self.log("TODO: cannot do https in jython", c="1;31")
                 return
 
             self.log_src = self.log_src.replace("[36m", "[35m")
@@ -176,18 +170,19 @@ class HttpConn(object):
                     self.log("client rejected our certificate (nice)")
 
                 elif "ALERT_CERTIFICATE_UNKNOWN" in em:
-                    # chrome-android keeps doing this
+                    # android-chrome keeps doing this
                     pass
 
                 else:
-                    self.log("\033[35mhandshake\033[0m " + em)
+                    self.log("handshake\033[0m " + em, c=5)
 
                 return
 
         if not self.sr:
             self.sr = Unrecv(self.s)
 
-        while True:
+        while not self.stopping:
+            self.nreq += 1
             cli = HttpCli(self)
             if not cli.run():
                 return

copyparty/httpsrv.py

@@ -2,13 +2,39 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
+import math
+import base64
 import socket
 import threading
 
-from .__init__ import E, MACOS
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
+from .__init__ import E, PY2, MACOS
+from .util import spack, min_ex, start_stackmon, start_log_thrs
+from .bos import bos
 from .httpconn import HttpConn
-from .authsrv import AuthSrv
+
+if PY2:
+    import Queue as queue
+else:
+    import queue
 
 
 class HttpSrv(object):
@@ -17,38 +43,208 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker):
+    def __init__(self, broker, nid):
         self.broker = broker
+        self.nid = nid
         self.args = broker.args
         self.log = broker.log
+        self.asrv = broker.asrv
 
-        self.disconnect_func = None
+        self.name = "httpsrv" + ("-n{}-i{:x}".format(nid, os.getpid()) if nid else "")
         self.mutex = threading.Lock()
+        self.stopping = False
 
-        self.clients = {}
-        self.workload = 0
-        self.workload_thr_alive = False
-        self.auth = AuthSrv(self.args, self.log)
+        self.tp_nthr = 0  # actual
+        self.tp_ncli = 0  # fading
+        self.tp_time = None  # latest worker collect
+        self.tp_q = None if self.args.no_htp else queue.LifoQueue()
+
+        self.srvs = []
+        self.ncli = 0  # exact
+        self.clients = {}  # laggy
+        self.nclimax = 0
+        self.cb_ts = 0
+        self.cb_v = 0
+
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+        }
 
         cert_path = os.path.join(E.cfg, "cert.pem")
-        if os.path.exists(cert_path):
+        if bos.path.exists(cert_path):
             self.cert_path = cert_path
         else:
             self.cert_path = None
 
+        if self.tp_q:
+            self.start_threads(4)
+
+            name = "httpsrv-scaler" + ("-{}".format(nid) if nid else "")
+            t = threading.Thread(target=self.thr_scaler, name=name)
+            t.daemon = True
+            t.start()
+
+        if nid:
+            if self.args.stackmon:
+                start_stackmon(self.args.stackmon, nid)
+
+            if self.args.log_thrs:
+                start_log_thrs(self.log, self.args.log_thrs, nid)
+
+    def start_threads(self, n):
+        self.tp_nthr += n
+        if self.args.log_htp:
+            self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
+
+        for _ in range(n):
+            thr = threading.Thread(
+                target=self.thr_poolw,
+                name=self.name + "-poolw",
+            )
+            thr.daemon = True
+            thr.start()
+
+    def stop_threads(self, n):
+        self.tp_nthr -= n
+        if self.args.log_htp:
+            self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
+
+        for _ in range(n):
+            self.tp_q.put(None)
+
+    def thr_scaler(self):
+        while True:
+            time.sleep(2 if self.tp_ncli else 30)
+            with self.mutex:
+                self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
+                if self.tp_nthr > self.tp_ncli + 8:
+                    self.stop_threads(4)
+
+    def listen(self, sck, nlisteners):
+        ip, port = sck.getsockname()
+        self.srvs.append(sck)
+        self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
+        t = threading.Thread(
+            target=self.thr_listen,
+            args=(sck,),
+            name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
+        )
+        t.daemon = True
+        t.start()
+
+    def thr_listen(self, srv_sck):
+        """listens on a shared tcp server"""
+        ip, port = srv_sck.getsockname()
+        fno = srv_sck.fileno()
+        msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
+        self.log(self.name, msg)
+        self.broker.put(False, "cb_httpsrv_up")
+        while not self.stopping:
+            if self.args.log_conn:
+                self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
+
+            if self.ncli >= self.nclimax:
+                self.log(self.name, "at connection limit; waiting", 3)
+                while self.ncli >= self.nclimax:
+                    time.sleep(0.1)
+
+            if self.args.log_conn:
+                self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
+
+            try:
+                sck, addr = srv_sck.accept()
+            except (OSError, socket.error) as ex:
+                self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
+                time.sleep(0.02)
+                continue
+
+            if self.args.log_conn:
+                m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                    "-" * 3, ip, port % 8, port
+                )
+                self.log("%s %s" % addr, m, c="1;30")
+
+            self.accept(sck, addr)
+
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log("%s %s" % addr, "\033[1;30m|%sC-cthr\033[0m" % ("-" * 5,))
-        thr = threading.Thread(target=self.thr_client, args=(sck, addr))
+        now = time.time()
+
+        if now - (self.tp_time or now) > 300:
+            m = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
+            self.log(self.name, m.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
+            self.tp_time = None
+            self.tp_q = None
+
+        with self.mutex:
+            self.ncli += 1
+            if self.tp_q:
+                self.tp_time = self.tp_time or now
+                self.tp_ncli = max(self.tp_ncli, self.ncli)
+                if self.tp_nthr < self.ncli + 4:
+                    self.start_threads(8)
+
+                self.tp_q.put((sck, addr))
+                return
+
+        if not self.args.no_htp:
+            m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
+            self.log(self.name, m, 1)
+
+        thr = threading.Thread(
+            target=self.thr_client,
+            args=(sck, addr),
+            name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+        )
         thr.daemon = True
         thr.start()
 
-    def num_clients(self):
-        with self.mutex:
-            return len(self.clients)
+    def thr_poolw(self):
+        while True:
+            task = self.tp_q.get()
+            if not task:
+                break
+
+            with self.mutex:
+                self.tp_time = None
+
+            try:
+                sck, addr = task
+                me = threading.current_thread()
+                me.name = "httpconn-{}-{}".format(
+                    addr[0].split(".", 2)[-1][-6:], addr[1]
+                )
+                self.thr_client(sck, addr)
+                me.name = self.name + "-poolw"
+            except:
+                self.log(self.name, "thr_client: " + min_ex(), 3)
 
     def shutdown(self):
-        self.log("ok bye")
+        self.stopping = True
+        for srv in self.srvs:
+            try:
+                srv.close()
+            except:
+                pass
+
+        clients = list(self.clients.keys())
+        for cli in clients:
+            try:
+                cli.shutdown()
+            except:
+                pass
+
+        if self.tp_q:
+            self.stop_threads(self.tp_nthr)
+            for _ in range(10):
+                time.sleep(0.05)
+                if self.tp_q.empty():
+                    break
+
+        self.log(self.name, "ok bye")
 
     def thr_client(self, sck, addr):
         """thread managing one tcp client"""
@@ -57,65 +253,69 @@ class HttpSrv(object):
         cli = HttpConn(sck, addr, self)
         with self.mutex:
             self.clients[cli] = 0
-            self.workload += 50
-
-            if not self.workload_thr_alive:
-                self.workload_thr_alive = True
-                thr = threading.Thread(target=self.thr_workload)
-                thr.daemon = True
-                thr.start()
 
+        fno = sck.fileno()
         try:
-            self.log("%s %s" % addr, "\033[1;30m|%sC-crun\033[0m" % ("-" * 6,))
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
+
             cli.run()
 
+        except (OSError, socket.error) as ex:
+            if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
+                self.log(
+                    "%s %s" % addr,
+                    "run({}): {}".format(fno, ex),
+                    c=6,
+                )
+
         finally:
-            self.log("%s %s" % addr, "\033[1;30m|%sC-cdone\033[0m" % ("-" * 7,))
+            sck = cli.s
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
+
             try:
+                fno = sck.fileno()
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
             except (OSError, socket.error) as ex:
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "\033[1;30mshut({}): {}\033[0m".format(sck.fileno(), ex),
+                        "shut({}): {}".format(fno, ex),
+                        c="1;30",
                     )
-                if ex.errno not in [10038, 10054, 107, 57, 9]:
+                if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
                     # 10038 No longer considered a socket
                     # 10054 Foribly closed by remote
                     #   107 Transport endpoint not connected
                     #    57 Socket is not connected
+                    #    49 Can't assign requested address (wifi down)
                     #     9 Bad file descriptor
                     raise
             finally:
                 with self.mutex:
                     del self.clients[cli]
+                    self.ncli -= 1
 
-                if self.disconnect_func:
-                    self.disconnect_func(addr)  # pylint: disable=not-callable
+    def cachebuster(self):
+        if time.time() - self.cb_ts < 1:
+            return self.cb_v
 
-    def thr_workload(self):
-        """indicates the python interpreter workload caused by this HttpSrv"""
-        # avoid locking in extract_filedata by tracking difference here
-        while True:
-            time.sleep(0.2)
-            with self.mutex:
-                if not self.clients:
-                    # no clients rn, termiante thread
-                    self.workload_thr_alive = False
-                    self.workload = 0
-                    return
+        with self.mutex:
+            if time.time() - self.cb_ts < 1:
+                return self.cb_v
 
-            total = 0
-            with self.mutex:
-                for cli in self.clients.keys():
-                    now = cli.workload
-                    delta = now - self.clients[cli]
-                    if delta < 0:
-                        # was reset in HttpCli to prevent overflow
-                        delta = now
+            v = E.t0
+            try:
+                with os.scandir(os.path.join(E.mod, "web")) as dh:
+                    for fh in dh:
+                        inf = fh.stat()
+                        v = max(v, inf.st_mtime)
+            except:
+                pass
 
-                    total += delta
-                    self.clients[cli] = now
-
-            self.workload = total
+            v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
+            self.cb_v = v.decode("ascii")[-4:]
+            self.cb_ts = time.time()
+            return self.cb_v

copyparty/ico.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import hashlib
+import colorsys
+
+from .__init__ import PY2
+
+
+class Ico(object):
+    def __init__(self, args):
+        self.args = args
+
+    def get(self, ext, as_thumb):
+        """placeholder to make thumbnails not break"""
+
+        h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        if PY2:
+            h = [ord(x) for x in h]
+
+        c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
+        c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
+        c = list(c1) + list(c2)
+        c = [int(x * 255) for x in c]
+        c = "".join(["{:02x}".format(x) for x in c])
+
+        h = 30
+        if not self.args.th_no_crop and as_thumb:
+            w, h = self.args.th_size.split("x")
+            h = int(100 / (float(w) / float(h)))
+
+        svg = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<svg version="1.1" viewBox="0 0 100 {}" xmlns="http://www.w3.org/2000/svg"><g>
+<rect width="100%" height="100%" fill="#{}" />
+<text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" xml:space="preserve"
+  fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
+</g></svg>
+"""
+        svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
+
+        return ["image/svg+xml", svg]

copyparty/mtag.py

@@ -1,49 +1,274 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
 import os
 import sys
+import json
 import shutil
 import subprocess as sp
 
-from .__init__ import PY2, WINDOWS
-from .util import fsenc, fsdec
+from .__init__ import PY2, WINDOWS, unicode
+from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
+from .bos import bos
+
+
+def have_ff(cmd):
+    if PY2:
+        print("# checking {}".format(cmd))
+        cmd = (cmd + " -version").encode("ascii").split(b" ")
+        try:
+            sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
+            return True
+        except:
+            return False
+    else:
+        return bool(shutil.which(cmd))
+
+
+HAVE_FFMPEG = have_ff("ffmpeg")
+HAVE_FFPROBE = have_ff("ffprobe")
+
+
+class MParser(object):
+    def __init__(self, cmdline):
+        self.tag, args = cmdline.split("=", 1)
+        self.tags = self.tag.split(",")
+
+        self.timeout = 30
+        self.force = False
+        self.audio = "y"
+        self.ext = []
+
+        while True:
+            try:
+                bp = os.path.expanduser(args)
+                if WINDOWS:
+                    bp = uncyg(bp)
+
+                if bos.path.exists(bp):
+                    self.bin = bp
+                    return
+            except:
+                pass
+
+            arg, args = args.split(",", 1)
+            arg = arg.lower()
+
+            if arg.startswith("a"):
+                self.audio = arg[1:]  # [r]equire [n]ot [d]ontcare
+                continue
+
+            if arg == "f":
+                self.force = True
+                continue
+
+            if arg.startswith("t"):
+                self.timeout = int(arg[1:])
+                continue
+
+            if arg.startswith("e"):
+                self.ext.append(arg[1:])
+                continue
+
+            raise Exception()
+
+
+def ffprobe(abspath):
+    cmd = [
+        b"ffprobe",
+        b"-hide_banner",
+        b"-show_streams",
+        b"-show_format",
+        b"--",
+        fsenc(abspath),
+    ]
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    r = p.communicate()
+    txt = r[0].decode("utf-8", "replace")
+    return parse_ffprobe(txt)
+
+
+def parse_ffprobe(txt):
+    """ffprobe -show_format -show_streams"""
+    streams = []
+    fmt = {}
+    g = None
+    for ln in [x.rstrip("\r") for x in txt.split("\n")]:
+        try:
+            k, v = ln.split("=", 1)
+            g[k] = v
+            continue
+        except:
+            pass
+
+        if ln == "[STREAM]":
+            g = {}
+            streams.append(g)
+
+        if ln == "[FORMAT]":
+            g = {"codec_type": "format"}  # heh
+            fmt = g
+
+    streams = [fmt] + streams
+    ret = {}  # processed
+    md = {}  # raw tags
+
+    is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
+    if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
+        is_audio = True
+
+    # if audio file, ensure audio stream appears first
+    if (
+        is_audio
+        and len(streams) > 2
+        and streams[1].get("codec_type") != "audio"
+        and streams[2].get("codec_type") == "audio"
+    ):
+        streams = [fmt, streams[2], streams[1]] + streams[3:]
+
+    have = {}
+    for strm in streams:
+        typ = strm.get("codec_type")
+        if typ in have:
+            continue
+
+        have[typ] = True
+        kvm = []
+
+        if typ == "audio":
+            kvm = [
+                ["codec_name", "ac"],
+                ["channel_layout", "chs"],
+                ["sample_rate", ".hz"],
+                ["bit_rate", ".aq"],
+                ["duration", ".dur"],
+            ]
+
+        if typ == "video":
+            if strm.get("DISPOSITION:attached_pic") == "1" or is_audio:
+                continue
+
+            kvm = [
+                ["codec_name", "vc"],
+                ["pix_fmt", "pixfmt"],
+                ["r_frame_rate", ".fps"],
+                ["bit_rate", ".vq"],
+                ["width", ".resw"],
+                ["height", ".resh"],
+                ["duration", ".dur"],
+            ]
+
+        if typ == "format":
+            kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
+
+        for sk, rk in kvm:
+            v = strm.get(sk)
+            if v is None:
+                continue
+
+            if rk.startswith("."):
+                try:
+                    v = float(v)
+                    v2 = ret.get(rk)
+                    if v2 is None or v > v2:
+                        ret[rk] = v
+                except:
+                    # sqlite doesnt care but the code below does
+                    if v not in ["N/A"]:
+                        ret[rk] = v
+            else:
+                ret[rk] = v
+
+    if ret.get("vc") == "ansi":  # shellscript
+        return {}, {}
+
+    for strm in streams:
+        for k, v in strm.items():
+            if not k.startswith("TAG:"):
+                continue
+
+            k = k[4:].strip()
+            v = v.strip()
+            if k and v and k not in md:
+                md[k] = [v]
+
+    for k in [".q", ".vq", ".aq"]:
+        if k in ret:
+            ret[k] /= 1000  # bit_rate=320000
+
+    for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
+        if k in ret:
+            ret[k] = int(ret[k])
+
+    if ".fps" in ret:
+        fps = ret[".fps"]
+        if "/" in fps:
+            fa, fb = fps.split("/")
+            fps = int(fa) * 1.0 / int(fb)
+
+        if fps < 1000 and fmt.get("format_name") not in ["image2", "png_pipe"]:
+            ret[".fps"] = round(fps, 3)
+        else:
+            del ret[".fps"]
+
+    if ".dur" in ret:
+        if ret[".dur"] < 0.1:
+            del ret[".dur"]
+            if ".q" in ret:
+                del ret[".q"]
+
+    if ".resw" in ret and ".resh" in ret:
+        ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
+
+    ret = {k: [0, v] for k, v in ret.items()}
+
+    return ret, md
 
 
 class MTag(object):
     def __init__(self, log_func, args):
         self.log_func = log_func
+        self.args = args
         self.usable = True
-        self.prefer_mt = False
-        mappings = args.mtm
+        self.prefer_mt = not args.no_mtag_ff
         self.backend = "ffprobe" if args.no_mutagen else "mutagen"
+        self.can_ffprobe = (
+            HAVE_FFPROBE
+            and not args.no_mtag_ff
+            and (not WINDOWS or sys.version_info >= (3, 8))
+        )
+        mappings = args.mtm
+        or_ffprobe = " or FFprobe"
 
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
                 import mutagen
             except:
-                self.log("\033[33mcould not load mutagen, trying ffprobe instead")
+                self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
         if self.backend == "ffprobe":
+            self.usable = self.can_ffprobe
             self.get = self.get_ffprobe
             self.prefer_mt = True
-            # about 20x slower
-            if PY2:
-                cmd = ["ffprobe", "-version"]
-                try:
-                    sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
-                except:
-                    self.usable = False
-            else:
-                if not shutil.which("ffprobe"):
-                    self.usable = False
+
+            if not HAVE_FFPROBE:
+                pass
+
+            elif args.no_mtag_ff:
+                msg = "found FFprobe but it was disabled by --no-mtag-ff"
+                self.log(msg, c=3)
+
+            elif WINDOWS and sys.version_info < (3, 8):
+                or_ffprobe = " or python >= 3.8"
+                msg = "found FFprobe but your python is too old; need 3.8 or newer"
+                self.log(msg, c=1)
 
         if not self.usable:
-            msg = "\033[31mneed mutagen or ffprobe to read media tags so please run this:\n  {} -m pip install --user mutagen \033[0m"
-            self.log(msg.format(os.path.basename(sys.executable)))
+            msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
+            pybin = os.path.basename(sys.executable)
+            self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
             return
 
         # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -115,8 +340,8 @@ class MTag(object):
         }
         # self.get = self.compare
 
-    def log(self, msg):
-        self.log_func("mtag", msg)
+    def log(self, msg, c=0):
+        self.log_func("mtag", msg, c)
 
     def normalize_tags(self, ret, md):
         for k, v in dict(md).items():
@@ -133,7 +358,7 @@ class MTag(object):
                 ret[mk] = [pref, v[0]]
 
         # take first value
-        ret = {k: str(v[1]).strip() for k, v in ret.items()}
+        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
 
         # track 3/7 => track 3
         for k, v in ret.items():
@@ -141,6 +366,12 @@ class MTag(object):
                 v = v.split("/")[0].strip().lstrip("0")
                 ret[k] = v or 0
 
+        # normalize key notation to rkeobo
+        okey = ret.get("key")
+        if okey:
+            key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
+            ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
+
         return ret
 
     def compare(self, abspath):
@@ -167,7 +398,7 @@ class MTag(object):
             v2 = r2.get(k)
             if v1 == v2:
                 print("  ", k, v1)
-            elif v1 != "0000":  # ffprobe date=0
+            elif v1 != "0000":  # FFprobe date=0
                 diffs.append(k)
                 print(" 1", k, v1)
                 print(" 2", k, v2)
@@ -185,122 +416,82 @@ class MTag(object):
         import mutagen
 
         try:
-            md = mutagen.File(abspath, easy=True)
+            md = mutagen.File(fsenc(abspath), easy=True)
             x = md.info.length
         except Exception as ex:
-            return {}
+            return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
-        ret = {}
-        try:
-            dur = int(md.info.length)
+        sz = bos.path.getsize(abspath)
+        ret = {".q": [0, int((sz / md.info.length) / 128)]}
+
+        for attr, k, norm in [
+            ["codec", "ac", unicode],
+            ["channels", "chs", int],
+            ["sample_rate", ".hz", int],
+            ["bitrate", ".aq", int],
+            ["length", ".dur", int],
+        ]:
             try:
-                q = int(md.info.bitrate / 1024)
+                v = getattr(md.info, attr)
             except:
-                q = int((os.path.getsize(abspath) / dur) / 128)
+                if k != "ac":
+                    continue
 
-            ret[".dur"] = [0, dur]
-            ret[".q"] = [0, q]
-        except:
-            pass
+                try:
+                    v = str(md.info).split(".")[1]
+                    if v.startswith("ogg"):
+                        v = v[3:]
+                except:
+                    continue
+
+            if not v:
+                continue
+
+            if k == ".aq":
+                v /= 1000
+
+            if k == "ac" and v.startswith("mp4a.40."):
+                v = "aac"
+
+            ret[k] = [0, norm(v)]
 
         return self.normalize_tags(ret, md)
 
     def get_ffprobe(self, abspath):
-        cmd = ["ffprobe", "-hide_banner", "--", fsenc(abspath)]
-        p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
-        r = p.communicate()
-        txt = r[1].decode("utf-8", "replace")
-        txt = [x.rstrip("\r") for x in txt.split("\n")]
+        ret, md = ffprobe(abspath)
+        return self.normalize_tags(ret, md)
 
-        """
-        note:
-          tags which contain newline will be truncated on first \n,
-          ffmpeg emits \n and spacepads the : to align visually
-        note:
-          the Stream ln always mentions Audio: if audio
-          the Stream ln usually has kb/s, is more accurate
-          the Duration ln always has kb/s
-          the Metadata: after Chapter may contain BPM info,
-            title : Tempo: 126.0
-
-        Input #0, wav,
-          Metadata:
-            date : <OK>
-          Duration:
-            Chapter #
-            Metadata:
-              title : <NG>
-
-        Input #0, mp3,
-          Metadata:
-            album : <OK>
-          Duration:
-            Stream #0:0: Audio:
-            Stream #0:1: Video:
-            Metadata:
-              comment : <NG>
-        """
-
-        ptn_md_beg = re.compile("^( +)Metadata:$")
-        ptn_md_kv = re.compile("^( +)([^:]+) *: (.*)")
-        ptn_dur = re.compile("^ *Duration: ([^ ]+)(, |$)")
-        ptn_br1 = re.compile("^ *Duration: .*, bitrate: ([0-9]+) kb/s(, |$)")
-        ptn_br2 = re.compile("^ *Stream.*: Audio:.* ([0-9]+) kb/s(, |$)")
-        ptn_audio = re.compile("^ *Stream .*: Audio: ")
-        ptn_au_parent = re.compile("^ *(Input #|Stream .*: Audio: )")
+    def get_bin(self, parsers, abspath):
+        pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
+        pypath = str(os.pathsep.join(pypath))
+        env = os.environ.copy()
+        env["PYTHONPATH"] = pypath
 
         ret = {}
-        md = {}
-        in_md = False
-        is_audio = False
-        au_parent = False
-        for ln in txt:
-            m = ptn_md_kv.match(ln)
-            if m and in_md and len(m.group(1)) == in_md:
-                _, k, v = [x.strip() for x in m.groups()]
-                if k != "" and v != "":
-                    md[k] = [v]
-                continue
-            else:
-                in_md = False
+        for tagname, mp in parsers.items():
+            try:
+                cmd = [sys.executable, mp.bin, abspath]
+                args = {"env": env, "timeout": mp.timeout}
 
-            m = ptn_md_beg.match(ln)
-            if m and au_parent:
-                in_md = len(m.group(1)) + 2
-                continue
+                if WINDOWS:
+                    args["creationflags"] = 0x4000
+                else:
+                    cmd = ["nice"] + cmd
 
-            au_parent = bool(ptn_au_parent.search(ln))
+                cmd = [fsenc(x) for x in cmd]
+                v = sp.check_output(cmd, **args).strip()
+                if not v:
+                    continue
 
-            if ptn_audio.search(ln):
-                is_audio = True
+                if "," not in tagname:
+                    ret[tagname] = v.decode("utf-8")
+                else:
+                    v = json.loads(v)
+                    for tag in tagname.split(","):
+                        if tag and tag in v:
+                            ret[tag] = v[tag]
+            except:
+                pass
 
-            m = ptn_dur.search(ln)
-            if m:
-                sec = 0
-                tstr = m.group(1)
-                if tstr.lower() != "n/a":
-                    try:
-                        tf = tstr.split(",")[0].split(".")[0].split(":")
-                        for f in tf:
-                            sec *= 60
-                            sec += int(f)
-                    except:
-                        self.log(
-                            "\033[33minvalid timestr from ffmpeg: [{}]".format(tstr)
-                        )
-
-                ret[".dur"] = sec
-                m = ptn_br1.search(ln)
-                if m:
-                    ret[".q"] = m.group(1)
-
-            m = ptn_br2.search(ln)
-            if m:
-                ret[".q"] = m.group(1)
-
-        if not is_audio:
-            return {}
-
-        ret = {k: [0, v] for k, v in ret.items()}
-
-        return self.normalize_tags(ret, md)
+        return ret

copyparty/star.py

@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+from .bos import bos
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, log, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.log = log
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen, name="star-gen")
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            bos.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf, txt = errdesc(errors)
+            self.log("\n".join(([repr(self.errf)] + txt[1:])))
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/stolen/surrogateescape.py

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 """
 This is Victor Stinner's pure-Python implementation of PEP 383: the "surrogateescape" error
 handler of Python 3.
@@ -171,7 +173,7 @@ FS_ENCODING = sys.getfilesystemencoding()
 
 if WINDOWS and not PY3:
     # py2 thinks win* is mbcs, probably a bug? anyways this works
-    FS_ENCODING = 'utf-8'
+    FS_ENCODING = "utf-8"
 
 
 # normalize the filesystem encoding name.

copyparty/sutil.py

@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import time
+import tempfile
+from datetime import datetime
+
+from .bos import bos
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcnow().strftime("%Y-%m%d-%H%M%S")
+
+    bos.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": bos.stat(tf_path),
+    }, report

copyparty/svchub.py

@@ -2,17 +2,23 @@
 from __future__ import print_function, unicode_literals
 
 import re
+import os
 import sys
 import time
+import shlex
+import string
+import signal
+import socket
 import threading
 from datetime import datetime, timedelta
 import calendar
 
-from .__init__ import PY2, WINDOWS, MACOS, VT100
+from .__init__ import E, PY2, WINDOWS, ANYWIN, MACOS, VT100, unicode
+from .util import mp, start_log_thrs, start_stackmon, min_ex, ansi_re
 from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
-from .util import mp
+from .th_srv import ThumbSrv, HAVE_PIL, HAVE_WEBP
 
 
 class SvcHub(object):
@@ -26,19 +32,51 @@ class SvcHub(object):
     put() can return a queue (if want_reply=True) which has a blocking get() with the response.
     """
 
-    def __init__(self, args):
+    def __init__(self, args, argv, printed):
         self.args = args
+        self.argv = argv
+        self.logf = None
+        self.stop_req = False
+        self.stopping = False
+        self.stop_cond = threading.Condition()
+        self.httpsrv_up = 0
 
-        self.ansi_re = re.compile("\033\\[[^m]*m")
         self.log_mutex = threading.Lock()
         self.next_day = 0
 
         self.log = self._log_disabled if args.q else self._log_enabled
+        if args.lo:
+            self._setup_logfile(printed)
+
+        if args.stackmon:
+            start_stackmon(args.stackmon, 0)
+
+        if args.log_thrs:
+            start_log_thrs(self.log, args.log_thrs, 0)
 
         # initiate all services to manage
+        self.asrv = AuthSrv(self.args, self.log)
+        if args.ls:
+            self.asrv.dbg_ls()
+
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
+        self.thumbsrv = None
+        if not args.no_thumb:
+            if HAVE_PIL:
+                if not HAVE_WEBP:
+                    args.th_no_webp = True
+                    msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
+                    self.log("thumb", msg, c=3)
+
+                self.thumbsrv = ThumbSrv(self)
+            else:
+                msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
+                self.log(
+                    "thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3
+                )
+
         # decide which worker impl to use
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker
@@ -48,50 +86,208 @@ class SvcHub(object):
 
         self.broker = Broker(self)
 
-    def run(self):
-        thr = threading.Thread(target=self.tcpsrv.run)
+    def thr_httpsrv_up(self):
+        time.sleep(5)
+        failed = self.broker.num_workers - self.httpsrv_up
+        if not failed:
+            return
+
+        m = "{}/{} workers failed to start"
+        m = m.format(failed, self.broker.num_workers)
+        self.log("root", m, 1)
+        os._exit(1)
+
+    def cb_httpsrv_up(self):
+        self.httpsrv_up += 1
+        if self.httpsrv_up != self.broker.num_workers:
+            return
+
+        self.log("root", "workers OK\n")
+        self.up2k.init_vols()
+
+        thr = threading.Thread(target=self.sd_notify, name="sd-notify")
         thr.daemon = True
         thr.start()
 
-        # winxp/py2.7 support: thr.join() kills signals
-        try:
-            while True:
-                time.sleep(9001)
+    def _logname(self):
+        dt = datetime.utcnow()
+        fn = self.args.lo
+        for fs in "YmdHMS":
+            fs = "%" + fs
+            if fs in fn:
+                fn = fn.replace(fs, dt.strftime(fs))
 
-        except KeyboardInterrupt:
+        return fn
+
+    def _setup_logfile(self, printed):
+        base_fn = fn = sel_fn = self._logname()
+        if fn != self.args.lo:
+            ctr = 0
+            # yup this is a race; if started sufficiently concurrently, two
+            # copyparties can grab the same logfile (considered and ignored)
+            while os.path.exists(sel_fn):
+                ctr += 1
+                sel_fn = "{}.{}".format(fn, ctr)
+
+        fn = sel_fn
+
+        try:
+            import lzma
+
+            lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+
+        except:
+            import codecs
+
+            lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
+
+        lh.base_fn = base_fn
+
+        argv = [sys.executable] + self.argv
+        if hasattr(shlex, "quote"):
+            argv = [shlex.quote(x) for x in argv]
+        else:
+            argv = ['"{}"'.format(x) for x in argv]
+
+        msg = "[+] opened logfile [{}]\n".format(fn)
+        printed += msg
+        lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
+        self.logf = lh
+        print(msg, end="")
+
+    def run(self):
+        self.tcpsrv.run()
+
+        thr = threading.Thread(target=self.thr_httpsrv_up)
+        thr.daemon = True
+        thr.start()
+
+        for sig in [signal.SIGINT, signal.SIGTERM]:
+            signal.signal(sig, self.signal_handler)
+
+        # macos hangs after shutdown on sigterm with while-sleep,
+        # windows cannot ^c stop_cond (and win10 does the macos thing but winxp is fine??)
+        # linux is fine with both,
+        # never lucky
+        if ANYWIN:
+            # msys-python probably fine but >msys-python
+            thr = threading.Thread(target=self.stop_thr, name="svchub-sig")
+            thr.daemon = True
+            thr.start()
+
+            try:
+                while not self.stop_req:
+                    time.sleep(1)
+            except:
+                pass
+
+            self.shutdown()
+            thr.join()
+        else:
+            self.stop_thr()
+
+    def stop_thr(self):
+        while not self.stop_req:
+            with self.stop_cond:
+                self.stop_cond.wait(9001)
+
+        self.shutdown()
+
+    def signal_handler(self, sig, frame):
+        if self.stopping:
+            return
+
+        self.stop_req = True
+        with self.stop_cond:
+            self.stop_cond.notify_all()
+
+    def shutdown(self):
+        if self.stopping:
+            return
+
+        self.stopping = True
+        self.stop_req = True
+        with self.stop_cond:
+            self.stop_cond.notify_all()
+
+        ret = 1
+        try:
             with self.log_mutex:
                 print("OPYTHAT")
 
             self.tcpsrv.shutdown()
             self.broker.shutdown()
-            print("nailed it")
+            self.up2k.shutdown()
+            if self.thumbsrv:
+                self.thumbsrv.shutdown()
 
-    def _log_disabled(self, src, msg):
-        pass
+                for n in range(200):  # 10s
+                    time.sleep(0.05)
+                    if self.thumbsrv.stopped():
+                        break
 
-    def _log_enabled(self, src, msg):
+                    if n == 3:
+                        print("waiting for thumbsrv (10sec)...")
+
+            print("nailed it", end="")
+            ret = 0
+        finally:
+            print("\033[0m")
+            if self.logf:
+                self.logf.close()
+
+            sys.exit(ret)
+
+    def _log_disabled(self, src, msg, c=0):
+        if not self.logf:
+            return
+
+        with self.log_mutex:
+            ts = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")[:-3]
+            self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
+
+            now = time.time()
+            if now >= self.next_day:
+                self._set_next_day()
+
+    def _set_next_day(self):
+        if self.next_day and self.logf and self.logf.base_fn != self._logname():
+            self.logf.close()
+            self._setup_logfile("")
+
+        dt = datetime.utcnow()
+
+        # unix timestamp of next 00:00:00 (leap-seconds safe)
+        day_now = dt.day
+        while dt.day == day_now:
+            dt += timedelta(hours=12)
+
+        dt = dt.replace(hour=0, minute=0, second=0)
+        self.next_day = calendar.timegm(dt.utctimetuple())
+
+    def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
         with self.log_mutex:
             now = time.time()
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
                 print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
-
-                # unix timestamp of next 00:00:00 (leap-seconds safe)
-                day_now = dt.day
-                while dt.day == day_now:
-                    dt += timedelta(hours=12)
-
-                dt = dt.replace(hour=0, minute=0, second=0)
-                self.next_day = calendar.timegm(dt.utctimetuple())
+                self._set_next_day()
 
             fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
                 fmt = "{} {:21} {}\n"
                 if "\033" in msg:
-                    msg = self.ansi_re.sub("", msg)
+                    msg = ansi_re.sub("", msg)
                 if "\033" in src:
-                    src = self.ansi_re.sub("", src)
+                    src = ansi_re.sub("", src)
+            elif c:
+                if isinstance(c, int):
+                    msg = "\033[3{}m{}".format(c, msg)
+                elif "\033" not in c:
+                    msg = "\033[{}m{}\033[0m".format(c, msg)
+                else:
+                    msg = "{}{}\033[0m".format(c, msg)
 
             ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             msg = fmt.format(ts, src, msg)
@@ -103,20 +299,20 @@ class SvcHub(object):
                 except:
                     print(msg.encode("ascii", "replace").decode(), end="")
 
+            if self.logf:
+                self.logf.write(msg)
+
     def check_mp_support(self):
         vmin = sys.version_info[1]
         if WINDOWS:
             msg = "need python 3.3 or newer for multiprocessing;"
-            if PY2:
-                # py2 pickler doesn't support winsock
-                return msg
-            elif vmin < 3:
+            if PY2 or vmin < 3:
                 return msg
         elif MACOS:
             return "multiprocessing is wonky on mac osx;"
         else:
-            msg = "need python 2.7 or 3.3+ for multiprocessing;"
-            if not PY2 and vmin < 3:
+            msg = "need python 3.3+ for multiprocessing;"
+            if PY2 or vmin < 3:
                 return msg
 
         try:
@@ -148,5 +344,24 @@ class SvcHub(object):
         if not err:
             return True
         else:
-            self.log("root", err)
+            self.log("svchub", err)
             return False
+
+    def sd_notify(self):
+        try:
+            addr = os.getenv("NOTIFY_SOCKET")
+            if not addr:
+                return
+
+            addr = unicode(addr)
+            if addr.startswith("@"):
+                addr = "\0" + addr[1:]
+
+            m = "".join(x for x in addr if x in string.printable)
+            self.log("sd_notify", m)
+
+            sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+            sck.connect(addr)
+            sck.sendall(b"READY=1")
+        except:
+            self.log("sd_notify", min_ex())

copyparty/szip.py

@@ -0,0 +1,275 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import time
+import zlib
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn, spack, sunpack
+from .bos import bos
+
+
+def dostime2unix(buf):
+    t, d = sunpack(b"<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return spack(b"<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = b"<LQQ" if z64 else b"<LLL"
+    ret += spack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = spack(b"<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += spack(b"<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, "/", [])
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += spack(b"<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += spack(b"<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += spack(b"<HH" + b"Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += spack(b"<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += spack(b"<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += spack(b"<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, log, fgen, utf8=False, pre_crc=False):
+        self.log = log
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf, txt = errdesc(errors)
+            self.log("\n".join(([repr(errf)] + txt[1:])))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -2,11 +2,10 @@
 from __future__ import print_function, unicode_literals
 
 import re
-import time
 import socket
-import select
 
-from .util import chkcmd, Counter
+from .__init__ import MACOS, ANYWIN
+from .util import chkcmd
 
 
 class TcpSrv(object):
@@ -20,7 +19,7 @@ class TcpSrv(object):
         self.args = hub.args
         self.log = hub.log
 
-        self.num_clients = Counter()
+        self.stopping = False
 
         ip = "127.0.0.1"
         eps = {ip: "local only"}
@@ -31,14 +30,16 @@ class TcpSrv(object):
                 for x in nonlocals:
                     eps[x] = "external"
 
+        msgs = []
+        m = "available @ http://{}:{}/  (\033[33m{}\033[0m)"
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
             for port in sorted(self.args.p):
-                self.log(
-                    "tcpsrv",
-                    "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
-                        ip, port, desc
-                    ),
-                )
+                msgs.append(m.format(ip, port, desc))
+
+        if msgs:
+            msgs[-1] += "\n"
+            for m in msgs:
+                self.log("tcpsrv", m)
 
         self.srv = []
         for ip in self.args.i:
@@ -65,50 +66,118 @@ class TcpSrv(object):
         for srv in self.srv:
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
-            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
+            fno = srv.fileno()
+            msg = "listening @ {}:{}  f{}".format(ip, port, fno)
+            self.log("tcpsrv", msg)
+            if self.args.q:
+                print(msg)
 
-        while True:
-            self.log("tcpsrv", "\033[1;30m|%sC-ncli\033[0m" % ("-" * 1,))
-            if self.num_clients.v >= self.args.nc:
-                time.sleep(0.1)
-                continue
-
-            self.log("tcpsrv", "\033[1;30m|%sC-acc1\033[0m" % ("-" * 2,))
-            ready, _, _ = select.select(self.srv, [], [])
-            for srv in ready:
-                sck, addr = srv.accept()
-                sip, sport = srv.getsockname()
-                self.log(
-                    "%s %s" % addr,
-                    "\033[1;30m|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
-                        "-" * 3, sip, sport % 8, sport
-                    ),
-                )
-                self.num_clients.add()
-                self.hub.broker.put(False, "httpconn", sck, addr)
+            self.hub.broker.put(False, "listen", srv)
 
     def shutdown(self):
+        self.stopping = True
+        try:
+            for srv in self.srv:
+                srv.close()
+        except:
+            pass
+
         self.log("tcpsrv", "ok bye")
 
-    def detect_interfaces(self, listen_ips):
+    def ips_linux(self):
         eps = {}
-
-        # get all ips and their interfaces
         try:
-            ip_addr, _ = chkcmd("ip", "addr")
+            txt, _ = chkcmd(["ip", "addr"])
         except:
-            ip_addr = None
+            return eps
 
-        if ip_addr:
-            r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
-            for ln in ip_addr.split("\n"):
-                try:
-                    ip, dev = r.match(ln.rstrip()).groups()
-                    for lip in listen_ips:
-                        if lip in ["0.0.0.0", ip]:
-                            eps[ip] = dev
-                except:
-                    pass
+        r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
+        for ln in txt.split("\n"):
+            try:
+                ip, dev = r.match(ln.rstrip()).groups()
+                eps[ip] = dev
+            except:
+                pass
+
+        return eps
+
+    def ips_macos(self):
+        eps = {}
+        try:
+            txt, _ = chkcmd(["ifconfig"])
+        except:
+            return eps
+
+        rdev = re.compile(r"^([^ ]+):")
+        rip = re.compile(r"^\tinet ([0-9\.]+) ")
+        dev = None
+        for ln in txt.split("\n"):
+            m = rdev.match(ln)
+            if m:
+                dev = m.group(1)
+
+            m = rip.match(ln)
+            if m:
+                eps[m.group(1)] = dev
+                dev = None
+
+        return eps
+
+    def ips_windows_ipconfig(self):
+        eps = {}
+        try:
+            txt, _ = chkcmd(["ipconfig"])
+        except:
+            return eps
+
+        rdev = re.compile(r"(^[^ ].*):$")
+        rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
+        dev = None
+        for ln in txt.replace("\r", "").split("\n"):
+            m = rdev.match(ln)
+            if m:
+                dev = m.group(1).split(" adapter ", 1)[-1]
+
+            m = rip.match(ln)
+            if m and dev:
+                eps[m.group(1)] = dev
+                dev = None
+
+        return eps
+
+    def ips_windows_netsh(self):
+        eps = {}
+        try:
+            txt, _ = chkcmd("netsh interface ip show address".split())
+        except:
+            return eps
+
+        rdev = re.compile(r'.* "([^"]+)"$')
+        rip = re.compile(r".* IP\b.*: +([0-9\.]{7,15})$")
+        dev = None
+        for ln in txt.replace("\r", "").split("\n"):
+            m = rdev.match(ln)
+            if m:
+                dev = m.group(1)
+
+            m = rip.match(ln)
+            if m and dev:
+                eps[m.group(1)] = dev
+                dev = None
+
+        return eps
+
+    def detect_interfaces(self, listen_ips):
+        if MACOS:
+            eps = self.ips_macos()
+        elif ANYWIN:
+            eps = self.ips_windows_ipconfig()  # sees more interfaces
+            eps.update(self.ips_windows_netsh())  # has better names
+        else:
+            eps = self.ips_linux()
+
+        if "0.0.0.0" not in listen_ips:
+            eps = {k: v for k, v in eps.items() if k in listen_ips}
 
         default_route = None
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

copyparty/th_cli.py

@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+
+from .util import Cooldown
+from .th_srv import thumb_path, THUMBABLE, FMT_FF
+from .bos import bos
+
+
+class ThumbCli(object):
+    def __init__(self, broker):
+        self.broker = broker
+        self.args = broker.args
+        self.asrv = broker.asrv
+
+        # cache on both sides for less broker spam
+        self.cooldown = Cooldown(self.args.th_poke)
+
+    def get(self, ptop, rem, mtime, fmt):
+        ext = rem.rsplit(".")[-1].lower()
+        if ext not in THUMBABLE:
+            return None
+
+        is_vid = ext in FMT_FF
+        if is_vid and self.args.no_vthumb:
+            return None
+
+        if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg"]:
+            return os.path.join(ptop, rem)
+
+        if fmt == "j" and self.args.th_no_jpg:
+            fmt = "w"
+
+        if fmt == "w":
+            if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
+                fmt = "j"
+
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
+        ret = None
+        try:
+            st = bos.stat(tpath)
+            if st.st_size:
+                ret = tpath
+            else:
+                return None
+        except:
+            pass
+
+        if ret:
+            tdir = os.path.dirname(tpath)
+            if self.cooldown.poke(tdir):
+                self.broker.put(False, "thumbsrv.poke", tdir)
+
+            return ret
+
+        x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
+        return x.get()

copyparty/th_srv.py

@@ -0,0 +1,424 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import time
+import shutil
+import base64
+import hashlib
+import threading
+import subprocess as sp
+
+from .__init__ import PY2, unicode
+from .util import fsenc, vsplit, runcmd, Queue, Cooldown, BytesIO, min_ex
+from .bos import bos
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+
+
+HAVE_PIL = False
+HAVE_HEIF = False
+HAVE_AVIF = False
+HAVE_WEBP = False
+
+try:
+    from PIL import Image, ImageOps, ExifTags
+
+    HAVE_PIL = True
+    try:
+        Image.new("RGB", (2, 2)).save(BytesIO(), format="webp")
+        HAVE_WEBP = True
+    except:
+        pass
+
+    try:
+        from pyheif_pillow_opener import register_heif_opener
+
+        register_heif_opener()
+        HAVE_HEIF = True
+    except:
+        pass
+
+    try:
+        import pillow_avif
+
+        HAVE_AVIF = True
+    except:
+        pass
+except:
+    pass
+
+# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
+# ffmpeg -formats
+FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
+FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc mts h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
+
+if HAVE_HEIF:
+    FMT_PIL += " heif heifs heic heics"
+
+if HAVE_AVIF:
+    FMT_PIL += " avif avifs"
+
+FMT_PIL, FMT_FF = [{x: True for x in y.split(" ") if x} for y in [FMT_PIL, FMT_FF]]
+
+
+THUMBABLE = {}
+
+if HAVE_PIL:
+    THUMBABLE.update(FMT_PIL)
+
+if HAVE_FFMPEG and HAVE_FFPROBE:
+    THUMBABLE.update(FMT_FF)
+
+
+def thumb_path(histpath, rem, mtime, fmt):
+    # base16 = 16 = 256
+    # b64-lc = 38 = 1444
+    # base64 = 64 = 4096
+    rd, fn = vsplit(rem)
+    if rd:
+        h = hashlib.sha512(fsenc(rd)).digest()
+        b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+        rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
+    else:
+        rd = "top"
+
+    # could keep original filenames but this is safer re pathlen
+    h = hashlib.sha512(fsenc(fn)).digest()
+    fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+
+    return "{}/th/{}/{}.{:x}.{}".format(
+        histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
+    )
+
+
+class ThumbSrv(object):
+    def __init__(self, hub):
+        self.hub = hub
+        self.asrv = hub.asrv
+        self.args = hub.args
+        self.log_func = hub.log
+
+        res = hub.args.th_size.split("x")
+        self.res = tuple([int(x) for x in res])
+        self.poke_cd = Cooldown(self.args.th_poke)
+
+        self.mutex = threading.Lock()
+        self.busy = {}
+        self.stopping = False
+        self.nthr = self.args.th_mt
+        if not self.nthr:
+            self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4
+
+        self.q = Queue(self.nthr * 4)
+        for n in range(self.nthr):
+            t = threading.Thread(
+                target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
+            )
+            t.daemon = True
+            t.start()
+
+        if not self.args.no_vthumb and (not HAVE_FFMPEG or not HAVE_FFPROBE):
+            missing = []
+            if not HAVE_FFMPEG:
+                missing.append("FFmpeg")
+
+            if not HAVE_FFPROBE:
+                missing.append("FFprobe")
+
+            msg = "cannot create video thumbnails because some of the required programs are not available: "
+            msg += ", ".join(missing)
+            self.log(msg, c=3)
+
+        if self.args.th_clean:
+            t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
+            t.daemon = True
+            t.start()
+
+    def log(self, msg, c=0):
+        self.log_func("thumb", msg, c)
+
+    def shutdown(self):
+        self.stopping = True
+        for _ in range(self.nthr):
+            self.q.put(None)
+
+    def stopped(self):
+        with self.mutex:
+            return not self.nthr
+
+    def get(self, ptop, rem, mtime, fmt):
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
+        abspath = os.path.join(ptop, rem)
+        cond = threading.Condition(self.mutex)
+        do_conv = False
+        with self.mutex:
+            try:
+                self.busy[tpath].append(cond)
+                self.log("wait {}".format(tpath))
+            except:
+                thdir = os.path.dirname(tpath)
+                bos.makedirs(thdir)
+
+                inf_path = os.path.join(thdir, "dir.txt")
+                if not bos.path.exists(inf_path):
+                    with open(inf_path, "wb") as f:
+                        f.write(fsenc(os.path.dirname(abspath)))
+
+                self.busy[tpath] = [cond]
+                do_conv = True
+
+        if do_conv:
+            self.q.put([abspath, tpath])
+            self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
+
+        while not self.stopping:
+            with self.mutex:
+                if tpath not in self.busy:
+                    break
+
+            with cond:
+                cond.wait(3)
+
+        try:
+            st = bos.stat(tpath)
+            if st.st_size:
+                return tpath
+        except:
+            pass
+
+        return None
+
+    def worker(self):
+        while not self.stopping:
+            task = self.q.get()
+            if not task:
+                break
+
+            abspath, tpath = task
+            ext = abspath.split(".")[-1].lower()
+            fun = None
+            if not bos.path.exists(tpath):
+                if ext in FMT_PIL:
+                    fun = self.conv_pil
+                elif ext in FMT_FF:
+                    fun = self.conv_ffmpeg
+
+            if fun:
+                try:
+                    fun(abspath, tpath)
+                except:
+                    msg = "{} could not create thumbnail of {}\n{}"
+                    self.log(msg.format(fun.__name__, abspath, min_ex()), "1;30")
+                    with open(tpath, "wb") as _:
+                        pass
+
+            with self.mutex:
+                subs = self.busy[tpath]
+                del self.busy[tpath]
+
+            for x in subs:
+                with x:
+                    x.notify_all()
+
+        with self.mutex:
+            self.nthr -= 1
+
+    def fancy_pillow(self, im):
+        # exif_transpose is expensive (loads full image + unconditional copy)
+        r = max(*self.res) * 2
+        im.thumbnail((r, r), resample=Image.LANCZOS)
+        try:
+            k = next(k for k, v in ExifTags.TAGS.items() if v == "Orientation")
+            exif = im.getexif()
+            rot = int(exif[k])
+            del exif[k]
+        except:
+            rot = 1
+
+        rots = {8: Image.ROTATE_90, 3: Image.ROTATE_180, 6: Image.ROTATE_270}
+        if rot in rots:
+            im = im.transpose(rots[rot])
+
+        if self.args.th_no_crop:
+            im.thumbnail(self.res, resample=Image.LANCZOS)
+        else:
+            iw, ih = im.size
+            dw, dh = self.res
+            res = (min(iw, dw), min(ih, dh))
+            im = ImageOps.fit(im, res, method=Image.LANCZOS)
+
+        return im
+
+    def conv_pil(self, abspath, tpath):
+        with Image.open(fsenc(abspath)) as im:
+            try:
+                im = self.fancy_pillow(im)
+            except Exception as ex:
+                self.log("fancy_pillow {}".format(ex), "1;30")
+                im.thumbnail(self.res)
+
+            fmts = ["RGB", "L"]
+            args = {"quality": 40}
+
+            if tpath.endswith(".webp"):
+                # quality 80 = pillow-default
+                # quality 75 = ffmpeg-default
+                # method 0 = pillow-default, fast
+                # method 4 = ffmpeg-default
+                # method 6 = max, slow
+                fmts += ["RGBA", "LA"]
+                args["method"] = 6
+            else:
+                # default q = 75
+                args["progressive"] = True
+
+            if im.mode not in fmts:
+                # print("conv {}".format(im.mode))
+                im = im.convert("RGB")
+
+            im.save(tpath, **args)
+
+    def conv_ffmpeg(self, abspath, tpath):
+        ret, _ = ffprobe(abspath)
+
+        ext = abspath.rsplit(".")[-1]
+        if ext in ["h264", "h265"]:
+            seek = []
+        else:
+            dur = ret[".dur"][1] if ".dur" in ret else 4
+            seek = "{:.0f}".format(dur / 3)
+            seek = [b"-ss", seek.encode("utf-8")]
+
+        scale = "scale={0}:{1}:force_original_aspect_ratio="
+        if self.args.th_no_crop:
+            scale += "decrease,setsar=1:1"
+        else:
+            scale += "increase,crop={0}:{1},setsar=1:1"
+
+        scale = scale.format(*list(self.res)).encode("utf-8")
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner"
+        ]
+        cmd += seek
+        cmd += [
+            b"-i", fsenc(abspath),
+            b"-map", b"0:v:0",
+            b"-vf", scale,
+            b"-frames:v", b"1",
+            b"-metadata:s:v:0", b"rotate=0",
+        ]
+        # fmt: on
+
+        if tpath.endswith(".jpg"):
+            cmd += [
+                b"-q:v",
+                b"6",  # default=??
+            ]
+        else:
+            cmd += [
+                b"-q:v",
+                b"50",  # default=75
+                b"-compression_level:v",
+                b"6",  # default=4, 0=fast, 6=max
+            ]
+
+        cmd += [fsenc(tpath)]
+        # self.log((b" ".join(cmd)).decode("utf-8"))
+
+        ret, sout, serr = runcmd(cmd)
+        if ret != 0:
+            m = "FFmpeg failed (probably a corrupt video file):\n"
+            m += "\n".join(["ff: {}".format(x) for x in serr.split("\n")])
+            self.log(m, c="1;30")
+            raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
+
+    def poke(self, tdir):
+        if not self.poke_cd.poke(tdir):
+            return
+
+        ts = int(time.time())
+        try:
+            p1 = os.path.dirname(tdir)
+            p2 = os.path.dirname(p1)
+            for dp in [tdir, p1, p2]:
+                bos.utime(dp, (ts, ts))
+        except:
+            pass
+
+    def cleaner(self):
+        interval = self.args.th_clean
+        while True:
+            time.sleep(interval)
+            ndirs = 0
+            for vol, histpath in self.asrv.vfs.histtab.items():
+                if histpath.startswith(vol):
+                    self.log("\033[Jcln {}/\033[A".format(histpath))
+                else:
+                    self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
+
+                ndirs += self.clean(histpath)
+
+            self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
+
+    def clean(self, histpath):
+        thumbpath = os.path.join(histpath, "th")
+        # self.log("cln {}".format(thumbpath))
+        maxage = self.args.th_maxage
+        now = time.time()
+        prev_b64 = None
+        prev_fp = None
+        try:
+            ents = bos.listdir(thumbpath)
+        except:
+            return 0
+
+        ndirs = 0
+        for f in sorted(ents):
+            fp = os.path.join(thumbpath, f)
+            cmp = fp.lower().replace("\\", "/")
+
+            # "top" or b64 prefix/full (a folder)
+            if len(f) <= 3 or len(f) == 24:
+                age = now - bos.path.getmtime(fp)
+                if age > maxage:
+                    with self.mutex:
+                        safe = True
+                        for k in self.busy.keys():
+                            if k.lower().replace("\\", "/").startswith(cmp):
+                                safe = False
+                                break
+
+                        if safe:
+                            ndirs += 1
+                            self.log("rm -rf [{}]".format(fp))
+                            shutil.rmtree(fp, ignore_errors=True)
+                else:
+                    ndirs += self.clean(fp)
+                continue
+
+            # thumb file
+            try:
+                b64, ts, ext = f.split(".")
+                if len(b64) != 24 or len(ts) != 8 or ext not in ["jpg", "webp"]:
+                    raise Exception()
+
+                ts = int(ts, 16)
+            except:
+                if f != "dir.txt":
+                    self.log("foreign file in thumbs dir: [{}]".format(fp), 1)
+
+                continue
+
+            if b64 == prev_b64:
+                self.log("rm replaced [{}]".format(fp))
+                bos.unlink(prev_fp)
+
+            prev_b64 = b64
+            prev_fp = fp
+
+        return ndirs

copyparty/u2idx.py

@@ -1,10 +1,15 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
+import time
+import threading
 from datetime import datetime
 
-from .util import u8safe
+from .__init__ import unicode
+from .util import s3dec, Pebkac, min_ex
+from .bos import bos
 from .up2k import up2k_wark_from_hashlist
 
 
@@ -16,18 +21,25 @@ except:
 
 
 class U2idx(object):
-    def __init__(self, args, log_func):
-        self.args = args
-        self.log_func = log_func
+    def __init__(self, conn):
+        self.log_func = conn.log_func
+        self.asrv = conn.asrv
+        self.args = conn.args
+        self.timeout = self.args.srch_time
 
         if not HAVE_SQLITE3:
-            self.log("could not load sqlite3; searchign wqill be disabled")
+            self.log("your python does not have sqlite3; searching will be disabled")
             return
 
         self.cur = {}
+        self.mem_cur = sqlite3.connect(":memory:")
+        self.mem_cur.execute(r"create table a (b text)")
 
-    def log(self, msg):
-        self.log_func("u2idx", msg)
+        self.p_end = None
+        self.p_dur = 0
+
+    def log(self, msg, c=0):
+        self.log_func("u2idx", msg, c)
 
     def fsearch(self, vols, body):
         """search by up2k hashlist"""
@@ -37,47 +49,187 @@ class U2idx(object):
         fsize = body["size"]
         fhash = body["hash"]
         wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
-        return self.run_query(vols, "w = ?", [wark], "", [])[0]
+
+        uq = "where substr(w,1,16) = ? and w = ?"
+        uv = [wark[:16], wark]
+
+        try:
+            return self.run_query(vols, uq, uv)[0]
+        except:
+            raise Pebkac(500, min_ex())
 
     def get_cur(self, ptop):
+        if not HAVE_SQLITE3:
+            return None
+
         cur = self.cur.get(ptop)
         if cur:
             return cur
 
-        cur = _open(ptop)
-        if not cur:
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
+        if not bos.path.exists(db_path):
             return None
 
+        cur = sqlite3.connect(db_path, 2).cursor()
         self.cur[ptop] = cur
         return cur
 
-    def search(self, vols, body):
+    def search(self, vols, uq):
         """search by query params"""
         if not HAVE_SQLITE3:
             return []
 
-        qobj = {}
-        _conv_sz(qobj, body, "sz_min", "up.sz >= ?")
-        _conv_sz(qobj, body, "sz_max", "up.sz <= ?")
-        _conv_dt(qobj, body, "dt_min", "up.mt >= ?")
-        _conv_dt(qobj, body, "dt_max", "up.mt <= ?")
-        for seg, dk in [["path", "up.rd"], ["name", "up.fn"]]:
-            if seg in body:
-                _conv_txt(qobj, body, seg, dk)
+        q = ""
+        va = []
+        joins = ""
+        is_key = True
+        is_size = False
+        is_date = False
+        kw_key = ["(", ")", "and ", "or ", "not "]
+        kw_val = ["==", "=", "!=", ">", ">=", "<", "<=", "like "]
+        ptn_mt = re.compile(r"^\.?[a-z_-]+$")
+        mt_ctr = 0
+        mt_keycmp = "substr(up.w,1,16)"
+        mt_keycmp2 = None
+        ptn_lc = re.compile(r" (mt[0-9]+\.v) ([=<!>]+) \? $")
+        ptn_lcv = re.compile(r"[a-zA-Z]")
 
-        uq, uv = _sqlize(qobj)
+        while True:
+            uq = uq.strip()
+            if not uq:
+                break
 
-        tq = ""
-        tv = []
-        qobj = {}
-        if "tags" in body:
-            _conv_txt(qobj, body, "tags", "mt.v")
-            tq, tv = _sqlize(qobj)
+            ok = False
+            for kw in kw_key + kw_val:
+                if uq.startswith(kw):
+                    is_key = kw in kw_key
+                    uq = uq[len(kw) :]
+                    ok = True
+                    q += kw
+                    break
 
-        return self.run_query(vols, uq, uv, tq, tv)
+            if ok:
+                continue
 
-    def run_query(self, vols, uq, uv, tq, tv):
-        self.log("qs: {} {} ,  {} {}".format(uq, repr(uv), tq, repr(tv)))
+            v, uq = (uq + " ").split(" ", 1)
+            if is_key:
+                is_key = False
+
+                if v == "size":
+                    v = "up.sz"
+                    is_size = True
+
+                elif v == "date":
+                    v = "up.mt"
+                    is_date = True
+
+                elif v == "path":
+                    v = "up.rd"
+
+                elif v == "name":
+                    v = "up.fn"
+
+                elif v == "tags" or ptn_mt.match(v):
+                    mt_ctr += 1
+                    mt_keycmp2 = "mt{}.w".format(mt_ctr)
+                    joins += "inner join mt mt{} on {} = {} ".format(
+                        mt_ctr, mt_keycmp, mt_keycmp2
+                    )
+                    mt_keycmp = mt_keycmp2
+                    if v == "tags":
+                        v = "mt{0}.v".format(mt_ctr)
+                    else:
+                        v = "+mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
+
+                else:
+                    raise Pebkac(400, "invalid key [" + v + "]")
+
+                q += v + " "
+                continue
+
+            head = ""
+            tail = ""
+
+            if is_date:
+                is_date = False
+                v = v.upper().rstrip("Z").replace(",", " ").replace("T", " ")
+                while "  " in v:
+                    v = v.replace("  ", " ")
+
+                for fmt in [
+                    "%Y-%m-%d %H:%M:%S",
+                    "%Y-%m-%d %H:%M",
+                    "%Y-%m-%d %H",
+                    "%Y-%m-%d",
+                ]:
+                    try:
+                        v = datetime.strptime(v, fmt).timestamp()
+                        break
+                    except:
+                        pass
+
+            elif is_size:
+                is_size = False
+                v = int(float(v) * 1024 * 1024)
+
+            else:
+                if v.startswith("*"):
+                    head = "'%'||"
+                    v = v[1:]
+
+                if v.endswith("*"):
+                    tail = "||'%'"
+                    v = v[:-1]
+
+            q += " {}?{} ".format(head, tail)
+            va.append(v)
+            is_key = True
+
+            # lowercase tag searches
+            m = ptn_lc.search(q)
+            if not m or not ptn_lcv.search(unicode(v)):
+                continue
+
+            va.pop()
+            va.append(v.lower())
+            q = q[: m.start()]
+
+            field, oper = m.groups()
+            if oper in ["=", "=="]:
+                q += " {} like ? ".format(field)
+            else:
+                q += " lower({}) {} ? ".format(field, oper)
+
+        try:
+            return self.run_query(vols, joins + "where " + q, va)
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
+
+    def run_query(self, vols, uq, uv):
+        done_flag = []
+        self.active_id = "{:.6f}_{}".format(
+            time.time(), threading.current_thread().ident
+        )
+        thr = threading.Thread(
+            target=self.terminator,
+            args=(
+                self.active_id,
+                done_flag,
+            ),
+            name="u2idx-terminator",
+        )
+        thr.daemon = True
+        thr.start()
+
+        if not uq or not uv:
+            q = "select * from up"
+            v = ()
+        else:
+            q = "select up.* from up " + uq
+            v = tuple(uv)
+
+        self.log("qs: {!r} {!r}".format(q, v))
 
         ret = []
         lim = 1000
@@ -87,105 +239,52 @@ class U2idx(object):
             if not cur:
                 continue
 
-            if not tq:
-                if not uq:
-                    q = "select * from up"
-                    v = ()
-                else:
-                    q = "select * from up where " + uq
-                    v = tuple(uv)
-            else:
-                # naive assumption: tags first
-                q = "select up.* from up inner join mt on substr(up.w,1,16) = mt.w where {}"
-                q = q.format(" and ".join([tq, uq]) if uq else tq)
-                v = tuple(tv + uv)
+            self.active_cur = cur
 
             sret = []
             c = cur.execute(q, v)
             for hit in c:
-                w, ts, sz, rd, fn = hit
+                w, ts, sz, rd, fn, ip, at = hit
                 lim -= 1
                 if lim <= 0:
                     break
 
-                rp = os.path.join(vtop, rd, fn).replace("\\", "/")
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
+                rp = "/".join([x for x in [vtop, rd, fn] if x])
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
 
             for hit in sret:
                 w = hit["w"]
                 del hit["w"]
                 tags = {}
-                q = "select k, v from mt where w = ? and k != 'x'"
-                for k, v in cur.execute(q, (w,)):
+                q2 = "select k, v from mt where w = ? and k != 'x'"
+                for k, v2 in cur.execute(q2, (w,)):
                     taglist[k] = True
-                    tags[k] = v
+                    tags[k] = v2
 
                 hit["tags"] = tags
 
             ret.extend(sret)
+            # print("[{}] {}".format(ptop, sret))
+
+        done_flag.append(True)
+        self.active_id = None
+
+        # undupe hits from multiple metadata keys
+        if len(ret) > 1:
+            ret = [ret[0]] + [
+                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+            ]
 
         return ret, list(taglist.keys())
 
+    def terminator(self, identifier, done_flag):
+        for _ in range(self.timeout):
+            time.sleep(1)
+            if done_flag:
+                return
 
-def _open(ptop):
-    db_path = os.path.join(ptop, ".hist", "up2k.db")
-    if os.path.exists(db_path):
-        return sqlite3.connect(db_path).cursor()
-
-
-def _conv_sz(q, body, k, sql):
-    if k in body:
-        q[sql] = int(float(body[k]) * 1024 * 1024)
-
-
-def _conv_dt(q, body, k, sql):
-    if k not in body:
-        return
-
-    v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ")
-    while "  " in v:
-        v = v.replace("  ", " ")
-
-    for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]:
-        try:
-            ts = datetime.strptime(v, fmt).timestamp()
-            break
-        except:
-            ts = None
-
-    if ts:
-        q[sql] = ts
-
-
-def _conv_txt(q, body, k, sql):
-    for v in body[k].split(" "):
-        inv = ""
-        if v.startswith("-"):
-            inv = "not"
-            v = v[1:]
-
-        if not v:
-            continue
-
-        head = "'%'||"
-        if v.startswith("^"):
-            head = ""
-            v = v[1:]
-
-        tail = "||'%'"
-        if v.endswith("$"):
-            tail = ""
-            v = v[:-1]
-
-        qk = "{} {} like {}?{}".format(sql, inv, head, tail)
-        q[qk + "\n" + v] = u8safe(v)
-
-
-def _sqlize(qobj):
-    keys = []
-    values = []
-    for k, v in sorted(qobj.items()):
-        keys.append(k.split("\n")[0])
-        values.append(v)
-
-    return " and ".join(keys), values
+        if identifier == self.active_id:
+            self.active_cur.connection.interrupt()

copyparty/util.py

@@ -4,18 +4,22 @@ from __future__ import print_function, unicode_literals
 import re
 import os
 import sys
+import stat
 import time
 import base64
 import select
 import struct
 import hashlib
 import platform
+import traceback
 import threading
 import mimetypes
 import contextlib
 import subprocess as sp  # nosec
+from datetime import datetime
+from collections import Counter
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN, VT100
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -33,10 +37,29 @@ if not PY2:
     from urllib.parse import unquote_to_bytes as unquote
     from urllib.parse import quote_from_bytes as quote
     from queue import Queue
+    from io import BytesIO
 else:
     from urllib import unquote  # pylint: disable=no-name-in-module
     from urllib import quote  # pylint: disable=no-name-in-module
     from Queue import Queue  # pylint: disable=import-error,no-name-in-module
+    from StringIO import StringIO as BytesIO
+
+
+try:
+    struct.unpack(b">i", b"idgi")
+    spack = struct.pack
+    sunpack = struct.unpack
+except:
+
+    def spack(f, *a, **ka):
+        return struct.pack(f.decode("ascii"), *a, **ka)
+
+    def sunpack(f, *a, **ka):
+        return struct.unpack(f.decode("ascii"), *a, **ka)
+
+
+ansi_re = re.compile("\033\\[[^mK]*[mK]")
+
 
 surrogateescape.register_surrogateescape()
 FS_ENCODING = sys.getfilesystemencoding()
@@ -44,35 +67,106 @@ if WINDOWS and PY2:
     FS_ENCODING = "utf-8"
 
 
+HTTP_TS_FMT = "%a, %d %b %Y %H:%M:%S GMT"
+
+
 HTTPCODE = {
     200: "OK",
     204: "No Content",
     206: "Partial Content",
+    302: "Found",
     304: "Not Modified",
     400: "Bad Request",
     403: "Forbidden",
     404: "Not Found",
     405: "Method Not Allowed",
+    411: "Length Required",
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
 }
 
 
-class Counter(object):
-    def __init__(self, v=0):
-        self.v = v
+IMPLICATIONS = [
+    ["e2dsa", "e2ds"],
+    ["e2ds", "e2d"],
+    ["e2tsr", "e2ts"],
+    ["e2ts", "e2t"],
+    ["e2t", "e2d"],
+]
+
+
+MIMES = {
+    "md": "text/plain; charset=UTF-8",
+    "opus": "audio/ogg; codecs=opus",
+    "webp": "image/webp",
+}
+
+
+REKOBO_KEY = {
+    v: ln.split(" ", 1)[0]
+    for ln in """
+1B 6d B
+2B 7d Gb F#
+3B 8d Db C#
+4B 9d Ab G#
+5B 10d Eb D#
+6B 11d Bb A#
+7B 12d F
+8B 1d C
+9B 2d G
+10B 3d D
+11B 4d A
+12B 5d E
+1A 6m Abm G#m
+2A 7m Ebm D#m
+3A 8m Bbm A#m
+4A 9m Fm
+5A 10m Cm
+6A 11m Gm
+7A 12m Dm
+8A 1m Am
+9A 2m Em
+10A 3m Bm
+11A 4m Gbm F#m
+12A 5m Dbm C#m
+""".strip().split(
+        "\n"
+    )
+    for v in ln.strip().split(" ")[1:]
+    if v
+}
+
+REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
+
+
+class Cooldown(object):
+    def __init__(self, maxage):
+        self.maxage = maxage
         self.mutex = threading.Lock()
+        self.hist = {}
+        self.oldest = 0
 
-    def add(self, delta=1):
+    def poke(self, key):
         with self.mutex:
-            self.v += delta
+            now = time.time()
 
-    def set(self, absval):
-        with self.mutex:
-            self.v = absval
+            ret = False
+            v = self.hist.get(key, 0)
+            if now - v > self.maxage:
+                self.hist[key] = now
+                ret = True
+
+            if self.oldest - now > self.maxage * 2:
+                self.hist = {
+                    k: v for k, v in self.hist.items() if now - v < self.maxage
+                }
+                self.oldest = sorted(self.hist.values())[0]
+
+            return ret
 
 
 class Unrecv(object):
@@ -105,7 +199,7 @@ class ProgressPrinter(threading.Thread):
     """
 
     def __init__(self):
-        threading.Thread.__init__(self)
+        threading.Thread.__init__(self, name="pp")
         self.daemon = True
         self.msg = None
         self.end = False
@@ -113,25 +207,164 @@ class ProgressPrinter(threading.Thread):
 
     def run(self):
         msg = None
+        fmt = " {}\033[K\r" if VT100 else " {} $\r"
         while not self.end:
             time.sleep(0.1)
             if msg == self.msg or self.end:
                 continue
 
             msg = self.msg
-            m = " {}\033[K\r".format(msg)
-            try:
-                print(m, end="")
-            except UnicodeEncodeError:
-                try:
-                    print(m.encode("utf-8", "replace").decode(), end="")
-                except:
-                    print(m.encode("ascii", "replace").decode(), end="")
+            uprint(fmt.format(msg))
+            if PY2:
+                sys.stdout.flush()
+
+        if VT100:
+            print("\033[K", end="")
+        elif msg:
+            print("------------------------")
 
-        print("\033[K", end="")
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
 
 
+def uprint(msg):
+    try:
+        print(msg, end="")
+    except UnicodeEncodeError:
+        try:
+            print(msg.encode("utf-8", "replace").decode(), end="")
+        except:
+            print(msg.encode("ascii", "replace").decode(), end="")
+
+
+def nuprint(msg):
+    uprint("{}\n".format(msg))
+
+
+def rice_tid():
+    tid = threading.current_thread().ident
+    c = sunpack(b"B" * 5, spack(b">Q", tid)[-5:])
+    return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
+
+
+def trace(*args, **kwargs):
+    t = time.time()
+    stack = "".join(
+        "\033[36m{}\033[33m{}".format(x[0].split(os.sep)[-1][:-3], x[1])
+        for x in traceback.extract_stack()[3:-1]
+    )
+    parts = ["{:.6f}".format(t), rice_tid(), stack]
+
+    if args:
+        parts.append(repr(args))
+
+    if kwargs:
+        parts.append(repr(kwargs))
+
+    msg = "\033[0m ".join(parts)
+    # _tracebuf.append(msg)
+    nuprint(msg)
+
+
+def alltrace():
+    threads = {}
+    names = dict([(t.ident, t.name) for t in threading.enumerate()])
+    for tid, stack in sys._current_frames().items():
+        name = "{} ({:x})".format(names.get(tid), tid)
+        threads[name] = stack
+
+    rret = []
+    bret = []
+    for name, stack in sorted(threads.items()):
+        ret = ["\n\n# {}".format(name)]
+        pad = None
+        for fn, lno, name, line in traceback.extract_stack(stack):
+            fn = os.sep.join(fn.split(os.sep)[-3:])
+            ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
+            if line:
+                ret.append("  " + str(line.strip()))
+                if "self.not_empty.wait()" in line:
+                    pad = " " * 4
+
+        if pad:
+            bret += [ret[0]] + [pad + x for x in ret[1:]]
+        else:
+            rret += ret
+
+    return "\n".join(rret + bret)
+
+
+def start_stackmon(arg_str, nid):
+    suffix = "-{}".format(nid) if nid else ""
+    fp, f = arg_str.rsplit(",", 1)
+    f = int(f)
+    t = threading.Thread(
+        target=stackmon,
+        args=(fp, f, suffix),
+        name="stackmon" + suffix,
+    )
+    t.daemon = True
+    t.start()
+
+
+def stackmon(fp, ival, suffix):
+    ctr = 0
+    while True:
+        ctr += 1
+        time.sleep(ival)
+        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
+        with open(fp + suffix, "wb") as f:
+            f.write(st.encode("utf-8", "replace"))
+
+
+def start_log_thrs(logger, ival, nid):
+    ival = int(ival)
+    tname = lname = "log-thrs"
+    if nid:
+        tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
+        lname = tname[3:]
+
+    t = threading.Thread(
+        target=log_thrs,
+        args=(logger, ival, lname),
+        name=tname,
+    )
+    t.daemon = True
+    t.start()
+
+
+def log_thrs(log, ival, name):
+    while True:
+        time.sleep(ival)
+        tv = [x.name for x in threading.enumerate()]
+        tv = [
+            x.split("-")[0]
+            if x.startswith("httpconn-") or x.startswith("thumb-")
+            else "listen"
+            if "-listen-" in x
+            else x
+            for x in tv
+            if not x.startswith("pydevd.")
+        ]
+        tv = ["{}\033[36m{}".format(v, k) for k, v in sorted(Counter(tv).items())]
+        log(name, "\033[0m \033[33m".join(tv), 3)
+
+
+def vol_san(vols, txt):
+    for vol in vols:
+        txt = txt.replace(vol.realpath.encode("utf-8"), vol.vpath.encode("utf-8"))
+
+    return txt
+
+
+def min_ex():
+    et, ev, tb = sys.exc_info()
+    tb = traceback.extract_tb(tb)
+    fmt = "{} @ {} <{}>: {}"
+    ex = [fmt.format(fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in tb]
+    ex.append("[{}] {}".format(et.__name__, ev))
+    return "\n".join(ex[-8:])
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -142,6 +375,11 @@ def ren_open(fname, *args, **kwargs):
             yield {"orz": [f, fname]}
             return
 
+    if suffix:
+        ext = fname.split(".")[-1]
+        if len(ext) < 7:
+            suffix += "." + ext
+
     orig_name = fname
     bname = fname
     ext = ""
@@ -162,7 +400,7 @@ def ren_open(fname, *args, **kwargs):
             else:
                 fpath = fname
 
-            if suffix and os.path.exists(fpath):
+            if suffix and os.path.exists(fsenc(fpath)):
                 fpath += suffix
                 fname += suffix
                 ext += suffix
@@ -185,7 +423,7 @@ def ren_open(fname, *args, **kwargs):
         if not b64:
             b64 = (bname + ext).encode("utf-8", "replace")
             b64 = hashlib.sha512(b64).digest()[:12]
-            b64 = base64.urlsafe_b64encode(b64).decode("utf-8").rstrip("=")
+            b64 = base64.urlsafe_b64encode(b64).decode("utf-8")
 
         badlen = len(fname)
         while len(fname) >= badlen:
@@ -441,8 +679,10 @@ def read_header(sr):
             else:
                 continue
 
-        sr.unrecv(ret[ofs + 4 :])
-        return ret[:ofs].decode("utf-8", "surrogateescape").split("\r\n")
+        if len(ret) > ofs + 4:
+            sr.unrecv(ret[ofs + 4 :])
+
+        return ret[:ofs].decode("utf-8", "surrogateescape").lstrip("\r\n").split("\r\n")
 
 
 def humansize(sz, terse=False):
@@ -460,6 +700,17 @@ def humansize(sz, terse=False):
     return ret.replace("iB", "").replace(" ", "")
 
 
+def unhumanize(sz):
+    try:
+        return float(sz)
+    except:
+        pass
+
+    mul = sz[-1:].lower()
+    mul = {"k": 1024, "m": 1024 * 1024, "g": 1024 * 1024 * 1024}.get(mul, 1)
+    return float(sz[:-1]) * mul
+
+
 def get_spd(nbyte, t0, t=None):
     if t is None:
         t = time.time()
@@ -470,6 +721,26 @@ def get_spd(nbyte, t0, t=None):
     return "{} \033[0m{}/s\033[0m".format(s1, s2)
 
 
+def s2hms(s, optional_h=False):
+    s = int(s)
+    h, s = divmod(s, 3600)
+    m, s = divmod(s, 60)
+    if not h and optional_h:
+        return "{}:{:02}".format(m, s)
+
+    return "{}:{:02}:{:02}".format(h, m, s)
+
+
+def uncyg(path):
+    if len(path) < 2 or not path.startswith("/"):
+        return path
+
+    if len(path) > 2 and path[2] != "/":
+        return path
+
+    return "{}:\\{}".format(path[1], path[3:])
+
+
 def undot(path):
     ret = []
     for node in path.split("/"):
@@ -486,11 +757,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
-    fn = fn.replace("\\", "/").split("/")[-1]
+def sanitize_fn(fn, ok, bad):
+    if "/" not in ok:
+        fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -500,19 +772,33 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
-            fn = fn.replace(bad, good)
+        ]
+        for a, b in [x for x in remap if x[0] not in ok]:
+            fn = fn.replace(a, b)
 
-        bad = ["con", "prn", "aux", "nul"]
+        bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):
             bad += "com{0} lpt{0}".format(n).split(" ")
 
-        if fn.lower() in bad:
-            fn = "_" + fn
+    if fn.lower() in bad:
+        fn = "_" + fn
 
     return fn.strip()
 
 
+def absreal(fpath):
+    try:
+        return fsdec(os.path.abspath(os.path.realpath(fsenc(fpath))))
+    except:
+        if not WINDOWS:
+            raise
+
+        # cpython bug introduced in 3.8, still exists in 3.9.1,
+        # some win7sp1 and win10:20H2 boxes cannot realpath a
+        # networked drive letter such as b"n:" or b"n:\\"
+        return os.path.abspath(os.path.realpath(fpath))
+
+
 def u8safe(txt):
     try:
         return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
@@ -524,17 +810,29 @@ def exclude_dotfiles(filepaths):
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def http_ts(ts):
+    file_dt = datetime.utcfromtimestamp(ts)
+    return file_dt.strftime(HTTP_TS_FMT)
+
+
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 
@@ -558,6 +856,13 @@ def unquotep(txt):
     return w8dec(unq2)
 
 
+def vsplit(vpath):
+    if "/" not in vpath:
+        return "", vpath
+
+    return vpath.rsplit("/", 1)
+
+
 def w8dec(txt):
     """decodes filesystem-bytes to wtf8"""
     if PY2:
@@ -597,7 +902,34 @@ else:
     fsdec = w8dec
 
 
+def s3enc(mem_cur, rd, fn):
+    ret = []
+    for v in [rd, fn]:
+        try:
+            mem_cur.execute("select * from a where b = ?", (v,))
+            ret.append(v)
+        except:
+            ret.append("//" + w8b64enc(v))
+            # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+    return tuple(ret)
+
+
+def s3dec(rd, fn):
+    ret = []
+    for k, v in [["d", rd], ["f", fn]]:
+        if v.startswith("//"):
+            ret.append(w8b64dec(v[2:]))
+            # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+        else:
+            ret.append(v)
+
+    return tuple(ret)
+
+
 def atomic_move(src, dst):
+    src = fsenc(src)
+    dst = fsenc(dst)
     if not PY2:
         os.replace(src, dst)
     else:
@@ -665,21 +997,26 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
-def hashcopy(actor, fin, fout):
-    u32_lim = int((2 ** 31) * 0.9)
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
+def hashcopy(fin, fout):
     hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
-        actor.workload += 1
-        if actor.workload > u32_lim:
-            actor.workload = 100  # prevent overflow
-
         tlen += len(buf)
         hashobj.update(buf)
         fout.write(buf)
 
-    digest32 = hashobj.digest()[:32]
-    digest_b64 = base64.urlsafe_b64encode(digest32).decode("utf-8").rstrip("=")
+    digest = hashobj.digest()[:33]
+    digest_b64 = base64.urlsafe_b64encode(digest).decode("utf-8")
 
     return tlen, hashobj.hexdigest(), digest_b64
 
@@ -689,7 +1026,7 @@ def sendfile_py(lower, upper, f, s):
     f.seek(lower)
     while remains > 0:
         # time.sleep(0.01)
-        buf = f.read(min(4096, remains))
+        buf = f.read(min(1024 * 32, remains))
         if not buf:
             return remains
 
@@ -725,6 +1062,9 @@ def sendfile_kern(lower, upper, f, s):
 
 
 def statdir(logger, scandir, lstat, top):
+    if lstat and not os.supports_follow_symlinks:
+        scandir = False
+
     try:
         btop = fsenc(top)
         if scandir and hasattr(os, "scandir"):
@@ -734,7 +1074,7 @@ def statdir(logger, scandir, lstat, top):
                     try:
                         yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
                     except Exception as ex:
-                        logger("scan-stat: {} @ {}".format(repr(ex), fsdec(fh.path)))
+                        logger(src, "[s] {} @ {}".format(repr(ex), fsdec(fh.path)), 6)
         else:
             src = "listdir"
             fun = os.lstat if lstat else os.stat
@@ -743,9 +1083,33 @@ def statdir(logger, scandir, lstat, top):
                 try:
                     yield [fsdec(name), fun(abspath)]
                 except Exception as ex:
-                    logger("list-stat: {} @ {}".format(repr(ex), fsdec(abspath)))
+                    logger(src, "[s] {} @ {}".format(repr(ex), fsdec(abspath)), 6)
+
     except Exception as ex:
-        logger("{}: {} @ {}".format(src, repr(ex), top))
+        logger(src, "{} @ {}".format(repr(ex), top), 1)
+
+
+def rmdirs(logger, scandir, lstat, top):
+    if not os.path.exists(fsenc(top)) or not os.path.isdir(fsenc(top)):
+        top = os.path.dirname(top)
+
+    dirs = statdir(logger, scandir, lstat, top)
+    dirs = [x[0] for x in dirs if stat.S_ISDIR(x[1].st_mode)]
+    dirs = [os.path.join(top, x) for x in dirs]
+    ok = []
+    ng = []
+    for d in dirs[::-1]:
+        a, b = rmdirs(logger, scandir, lstat, d)
+        ok += a
+        ng += b
+
+    try:
+        os.rmdir(fsenc(top))
+        ok.append(top)
+    except:
+        ng.append(top)
+
+    return ok, ng
 
 
 def unescape_cookie(orig):
@@ -776,33 +1140,53 @@ def unescape_cookie(orig):
     return ret
 
 
-def guess_mime(url):
-    if url.endswith(".md"):
-        return ["text/plain; charset=UTF-8"]
+def guess_mime(url, fallback="application/octet-stream"):
+    try:
+        _, ext = url.rsplit(".", 1)
+    except:
+        return fallback
 
-    return mimetypes.guess_type(url)
+    ret = MIMES.get(ext) or mimetypes.guess_type(url)[0] or fallback
+
+    if ";" not in ret:
+        if ret.startswith("text/") or ret.endswith("/javascript"):
+            ret += "; charset=UTF-8"
+
+    return ret
 
 
-def runcmd(*argv):
+def runcmd(argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
-    stdout = stdout.decode("utf-8")
-    stderr = stderr.decode("utf-8")
+    stdout = stdout.decode("utf-8", "replace")
+    stderr = stderr.decode("utf-8", "replace")
     return [p.returncode, stdout, stderr]
 
 
-def chkcmd(*argv):
-    ok, sout, serr = runcmd(*argv)
+def chkcmd(argv):
+    ok, sout, serr = runcmd(argv)
     if ok != 0:
         raise Exception(serr)
 
     return sout, serr
 
 
+def mchkcmd(argv, timeout=10):
+    if PY2:
+        with open(os.devnull, "wb") as f:
+            rv = sp.call(argv, stdout=f, stderr=f)
+    else:
+        rv = sp.call(argv, stdout=sp.DEVNULL, stderr=sp.DEVNULL, timeout=timeout)
+
+    if rv:
+        raise sp.CalledProcessError(rv, (argv[0], b"...", argv[-1]))
+
+
 def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
-        return struct.unpack(b"I", f.read(4))[0]
+        rv = f.read(4)
+        return sunpack(b"I", rv)[0]
 
 
 def py_desc():
@@ -812,7 +1196,11 @@ def py_desc():
     if ofs > 0:
         py_ver = py_ver[:ofs]
 
-    bitness = struct.calcsize(b"P") * 8
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
     host_os = platform.system()
     compiler = platform.python_compiler()
 

copyparty/web/baguettebox.js

@@ -0,0 +1,889 @@
+/*!
+ * baguetteBox.js
+ * @author  feimosi
+ * @version 1.11.1-mod
+ * @url https://github.com/feimosi/baguetteBox.js
+ */
+
+window.baguetteBox = (function () {
+    'use strict';
+
+    var options = {},
+        defaults = {
+            captions: true,
+            buttons: 'auto',
+            noScrollbars: false,
+            bodyClass: 'bbox-open',
+            titleTag: false,
+            async: false,
+            preload: 2,
+            animation: 'slideIn',
+            afterShow: null,
+            afterHide: null,
+            onChange: null,
+        },
+        overlay, slider, btnPrev, btnNext, btnHelp, btnRotL, btnRotR, btnSel, btnVmode, btnClose,
+        currentGallery = [],
+        currentIndex = 0,
+        isOverlayVisible = false,
+        touch = {},  // start-pos
+        touchFlag = false,  // busy
+        re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
+        re_v = /.+\.(webm|mp4)(\?|$)/i,
+        data = {},  // all galleries
+        imagesElements = [],
+        documentLastFocus = null,
+        isFullscreen = false,
+        vmute = false,
+        vloop = false,
+        vnext = false,
+        resume_mp = false;
+
+    var onFSC = function (e) {
+        isFullscreen = !!document.fullscreenElement;
+    };
+
+    var overlayClickHandler = function (e) {
+        if (e.target.id.indexOf('baguette-img') !== -1)
+            hideOverlay();
+    };
+
+    var touchstartHandler = function (e) {
+        touch.count = e.touches.length;
+        if (touch.count > 1)
+            touch.multitouch = true;
+
+        touch.startX = e.changedTouches[0].pageX;
+        touch.startY = e.changedTouches[0].pageY;
+    };
+    var touchmoveHandler = function (e) {
+        if (touchFlag || touch.multitouch)
+            return;
+
+        e.preventDefault ? e.preventDefault() : e.returnValue = false;
+        var touchEvent = e.touches[0] || e.changedTouches[0];
+        if (touchEvent.pageX - touch.startX > 40) {
+            touchFlag = true;
+            showPreviousImage();
+        } else if (touchEvent.pageX - touch.startX < -40) {
+            touchFlag = true;
+            showNextImage();
+        } else if (touch.startY - touchEvent.pageY > 100) {
+            hideOverlay();
+        }
+    };
+    var touchendHandler = function (e) {
+        touch.count--;
+        if (e && e.touches)
+            touch.count = e.touches.length;
+
+        if (touch.count <= 0)
+            touch.multitouch = false;
+
+        touchFlag = false;
+    };
+    var contextmenuHandler = function () {
+        touchendHandler();
+    };
+
+    var trapFocusInsideOverlay = function (e) {
+        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(e.target))) {
+            e.stopPropagation();
+            btnClose.focus();
+        }
+    };
+
+    function run(selector, userOptions) {
+        buildOverlay();
+        removeFromCache(selector);
+        return bindImageClickListeners(selector, userOptions);
+    }
+
+    function bindImageClickListeners(selector, userOptions) {
+        var galleryNodeList = QSA(selector);
+        var selectorData = {
+            galleries: [],
+            nodeList: galleryNodeList
+        };
+        data[selector] = selectorData;
+
+        [].forEach.call(galleryNodeList, function (galleryElement) {
+            var tagsNodeList = [];
+            if (galleryElement.tagName === 'A')
+                tagsNodeList = [galleryElement];
+            else
+                tagsNodeList = galleryElement.getElementsByTagName('a');
+
+            tagsNodeList = [].filter.call(tagsNodeList, function (element) {
+                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1)
+                    return re_i.test(element.href) || re_v.test(element.href);
+            });
+            if (!tagsNodeList.length)
+                return;
+
+            var gallery = [];
+            [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
+                var imageElementClickHandler = function (e) {
+                    if (ctrl(e))
+                        return true;
+
+                    e.preventDefault ? e.preventDefault() : e.returnValue = false;
+                    prepareOverlay(gallery, userOptions);
+                    showOverlay(imageIndex);
+                };
+                var imageItem = {
+                    eventHandler: imageElementClickHandler,
+                    imageElement: imageElement
+                };
+                bind(imageElement, 'click', imageElementClickHandler);
+                gallery.push(imageItem);
+            });
+            selectorData.galleries.push(gallery);
+        });
+
+        return selectorData.galleries;
+    }
+
+    function clearCachedData() {
+        for (var selector in data)
+            if (data.hasOwnProperty(selector))
+                removeFromCache(selector);
+    }
+
+    function removeFromCache(selector) {
+        if (!data.hasOwnProperty(selector))
+            return;
+
+        var galleries = data[selector].galleries;
+        [].forEach.call(galleries, function (gallery) {
+            [].forEach.call(gallery, function (imageItem) {
+                unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
+            });
+
+            if (currentGallery === gallery)
+                currentGallery = [];
+        });
+
+        delete data[selector];
+    }
+
+    function buildOverlay() {
+        overlay = ebi('bbox-overlay');
+        if (!overlay) {
+            var ctr = mknod('div');
+            ctr.innerHTML = (
+                '<div id="bbox-overlay" role="dialog">' +
+                '<div id="bbox-slider"></div>' +
+                '<button id="bbox-prev" class="bbox-btn" type="button" aria-label="Previous">&lt;</button>' +
+                '<button id="bbox-next" class="bbox-btn" type="button" aria-label="Next">&gt;</button>' +
+                '<div id="bbox-btns">' +
+                '<button id="bbox-help" type="button">?</button>' +
+                '<button id="bbox-rotl" type="button">↶</button>' +
+                '<button id="bbox-rotr" type="button">↷</button>' +
+                '<button id="bbox-tsel" type="button">sel</button>' +
+                '<button id="bbox-vmode" type="button" tt="a"></button>' +
+                '<button id="bbox-close" type="button" aria-label="Close">X</button>' +
+                '</div></div>'
+            );
+            overlay = ctr.firstChild;
+            QS('body').appendChild(overlay);
+            tt.att(overlay);
+        }
+        slider = ebi('bbox-slider');
+        btnPrev = ebi('bbox-prev');
+        btnNext = ebi('bbox-next');
+        btnHelp = ebi('bbox-help');
+        btnRotL = ebi('bbox-rotl');
+        btnRotR = ebi('bbox-rotr');
+        btnSel = ebi('bbox-tsel');
+        btnVmode = ebi('bbox-vmode');
+        btnClose = ebi('bbox-close');
+        bindEvents();
+    }
+
+    function halp() {
+        if (ebi('bbox-halp'))
+            return;
+
+        var list = [
+            ['<b># hotkey</b>', '<b># operation</b>'],
+            ['escape', 'close'],
+            ['left, J', 'previous file'],
+            ['right, L', 'next file'],
+            ['home', 'first file'],
+            ['end', 'last file'],
+            ['R', 'rotate (shift=ccw)'],
+            ['S', 'toggle file selection'],
+            ['space, P, K', 'video: play / pause'],
+            ['U', 'video: seek 10sec back'],
+            ['P', 'video: seek 10sec ahead'],
+            ['M', 'video: toggle mute'],
+            ['V', 'video: toggle loop'],
+            ['C', 'video: toggle auto-next'],
+            ['F', 'video: toggle fullscreen'],
+        ],
+            d = mknod('table'),
+            html = ['<tbody>'];
+
+        for (var a = 0; a < list.length; a++)
+            html.push('<tr><td>' + list[a][0] + '</td><td>' + list[a][1] + '</td></tr>');
+
+        d.innerHTML = html.join('\n') + '</tbody>';
+        d.setAttribute('id', 'bbox-halp');
+        d.onclick = function () {
+            overlay.removeChild(d);
+        };
+        overlay.appendChild(d);
+    }
+
+    function keyDownHandler(e) {
+        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
+            return;
+
+        var k = e.code + '', v = vid();
+
+        if (k == "ArrowLeft" || k == "KeyJ")
+            showPreviousImage();
+        else if (k == "ArrowRight" || k == "KeyL")
+            showNextImage();
+        else if (k == "Escape")
+            hideOverlay();
+        else if (k == "Home")
+            showFirstImage(e);
+        else if (k == "End")
+            showLastImage(e);
+        else if (k == "Space" || k == "KeyP" || k == "KeyK")
+            playpause();
+        else if (k == "KeyU" || k == "KeyO")
+            relseek(k == "KeyU" ? -10 : 10);
+        else if (k == "KeyM" && v) {
+            v.muted = vmute = !vmute;
+            mp_ctl();
+        }
+        else if (k == "KeyV" && v) {
+            vloop = !vloop;
+            vnext = vnext && !vloop;
+            setVmode();
+        }
+        else if (k == "KeyC" && v) {
+            vnext = !vnext;
+            vloop = vloop && !vnext;
+            setVmode();
+        }
+        else if (k == "KeyF")
+            try {
+                if (isFullscreen)
+                    document.exitFullscreen();
+                else
+                    v.requestFullscreen();
+            }
+            catch (ex) { }
+        else if (k == "KeyS")
+            tglsel();
+        else if (k == "KeyR")
+            rotn(e.shiftKey ? -1 : 1);
+    }
+
+    function setVmode() {
+        var v = vid();
+        ebi('bbox-vmode').style.display = v ? '' : 'none';
+        if (!v)
+            return;
+
+        var msg = 'When video ends, ', tts = '', lbl;
+        if (vloop) {
+            lbl = 'Loop';
+            msg += 'repeat it';
+            tts = '$NHotkey: V';
+        }
+        else if (vnext) {
+            lbl = 'Cont';
+            msg += 'continue to next';
+            tts = '$NHotkey: C';
+        }
+        else {
+            lbl = 'Stop';
+            msg += 'just stop'
+        }
+        btnVmode.setAttribute('aria-label', msg);
+        btnVmode.setAttribute('tt', msg + tts);
+        btnVmode.textContent = lbl;
+
+        v.loop = vloop
+        if (vloop && v.paused)
+            v.play();
+    }
+
+    function tglVmode() {
+        if (vloop) {
+            vnext = true;
+            vloop = false;
+        }
+        else if (vnext)
+            vnext = false;
+        else
+            vloop = true;
+
+        setVmode();
+        if (tt.en)
+            tt.show.bind(this)();
+    }
+
+    function tglsel() {
+        var thumb = currentGallery[currentIndex].imageElement,
+            name = vsplit(thumb.href)[1],
+            files = msel.getall();
+
+        for (var a = 0; a < files.length; a++)
+            if (vsplit(files[a].vp)[1] == name)
+                clmod(ebi(files[a].id).closest('tr'), 'sel', 't');
+
+        msel.selui();
+        selbg();
+    }
+
+    function selbg() {
+        var img = vidimg(),
+            thumb = currentGallery[currentIndex].imageElement,
+            name = vsplit(thumb.href)[1],
+            files = msel.getsel(),
+            sel = false;
+
+        for (var a = 0; a < files.length; a++)
+            if (vsplit(files[a].vp)[1] == name)
+                sel = true;
+
+        ebi('bbox-overlay').style.background = sel ?
+            'rgba(153,34,85,0.7)' : '';
+
+        img.style.borderRadius = sel ? '1em' : '';
+        btnSel.style.color = sel ? '#fff' : '';
+        btnSel.style.background = sel ? '#d48' : '';
+        btnSel.style.textShadow = sel ? '1px 1px 0 #b38' : '';
+        btnSel.style.boxShadow = sel ? '.15em .15em 0 #502' : '';
+    }
+
+    function keyUpHandler(e) {
+        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
+            return;
+
+        var k = e.code + '';
+
+        if (k == "Space")
+            ev(e);
+    }
+
+    var passiveSupp = false;
+    try {
+        var opts = {
+            get passive() {
+                passiveSupp = true;
+                return false;
+            }
+        };
+        window.addEventListener('test', null, opts);
+        window.removeEventListener('test', null, opts);
+    }
+    catch (ex) {
+        passiveSupp = false;
+    }
+    var passiveEvent = passiveSupp ? { passive: false } : null;
+    var nonPassiveEvent = passiveSupp ? { passive: true } : null;
+
+    function bindEvents() {
+        bind(overlay, 'click', overlayClickHandler);
+        bind(btnPrev, 'click', showPreviousImage);
+        bind(btnNext, 'click', showNextImage);
+        bind(btnClose, 'click', hideOverlay);
+        bind(btnVmode, 'click', tglVmode);
+        bind(btnHelp, 'click', halp);
+        bind(btnRotL, 'click', rotl);
+        bind(btnRotR, 'click', rotr);
+        bind(btnSel, 'click', tglsel);
+        bind(slider, 'contextmenu', contextmenuHandler);
+        bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        bind(overlay, 'touchend', touchendHandler);
+        bind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function unbindEvents() {
+        unbind(overlay, 'click', overlayClickHandler);
+        unbind(btnPrev, 'click', showPreviousImage);
+        unbind(btnNext, 'click', showNextImage);
+        unbind(btnClose, 'click', hideOverlay);
+        unbind(btnVmode, 'click', tglVmode);
+        unbind(btnHelp, 'click', halp);
+        unbind(btnRotL, 'click', rotl);
+        unbind(btnRotR, 'click', rotr);
+        unbind(btnSel, 'click', tglsel);
+        unbind(slider, 'contextmenu', contextmenuHandler);
+        unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        unbind(overlay, 'touchend', touchendHandler);
+        unbind(document, 'focus', trapFocusInsideOverlay, true);
+        timer.rm(rotn);
+    }
+
+    function prepareOverlay(gallery, userOptions) {
+        if (currentGallery === gallery)
+            return;
+
+        currentGallery = gallery;
+        setOptions(userOptions);
+        slider.innerHTML = '';
+        imagesElements.length = 0;
+
+        var imagesFiguresIds = [];
+        var imagesCaptionsIds = [];
+        for (var i = 0, fullImage; i < gallery.length; i++) {
+            fullImage = mknod('div');
+            fullImage.className = 'full-image';
+            fullImage.id = 'baguette-img-' + i;
+            imagesElements.push(fullImage);
+
+            imagesFiguresIds.push('bbox-figure-' + i);
+            imagesCaptionsIds.push('bbox-figcaption-' + i);
+            slider.appendChild(imagesElements[i]);
+        }
+        overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
+        overlay.setAttribute('aria-describedby', imagesCaptionsIds.join(' '));
+    }
+
+    function setOptions(newOptions) {
+        if (!newOptions)
+            newOptions = {};
+
+        for (var item in defaults) {
+            options[item] = defaults[item];
+            if (typeof newOptions[item] !== 'undefined')
+                options[item] = newOptions[item];
+        }
+        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
+            options.animation === 'slideIn' ? '' : 'none');
+
+        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1))
+            options.buttons = false;
+
+        btnPrev.style.display = btnNext.style.display = (options.buttons ? '' : 'none');
+    }
+
+    function showOverlay(chosenImageIndex) {
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'hidden';
+            document.body.style.overflowY = 'scroll';
+        }
+        if (overlay.style.display === 'block')
+            return;
+
+        bind(document, 'keydown', keyDownHandler);
+        bind(document, 'keyup', keyUpHandler);
+        bind(document, 'fullscreenchange', onFSC);
+        currentIndex = chosenImageIndex;
+        touch = {
+            count: 0,
+            startX: null,
+            startY: null
+        };
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+
+        updateOffset();
+        overlay.style.display = 'block';
+        // Fade in overlay
+        setTimeout(function () {
+            overlay.className = 'visible';
+            if (options.bodyClass && document.body.classList)
+                document.body.classList.add(options.bodyClass);
+
+            if (options.afterShow)
+                options.afterShow();
+        }, 50);
+
+        if (options.onChange)
+            options.onChange(currentIndex, imagesElements.length);
+
+        documentLastFocus = document.activeElement;
+        btnClose.focus();
+        isOverlayVisible = true;
+    }
+
+    function hideOverlay(e) {
+        ev(e);
+        playvid(false);
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'auto';
+            document.body.style.overflowY = 'auto';
+        }
+        if (overlay.style.display === 'none')
+            return;
+
+        unbind(document, 'keydown', keyDownHandler);
+        unbind(document, 'keyup', keyUpHandler);
+        unbind(document, 'fullscreenchange', onFSC);
+        // Fade out and hide the overlay
+        overlay.className = '';
+        setTimeout(function () {
+            overlay.style.display = 'none';
+            if (options.bodyClass && document.body.classList)
+                document.body.classList.remove(options.bodyClass);
+
+            var h = ebi('bbox-halp');
+            if (h)
+                h.parentNode.removeChild(h);
+
+            if (options.afterHide)
+                options.afterHide();
+
+            documentLastFocus && documentLastFocus.focus();
+            isOverlayVisible = false;
+        }, 500);
+    }
+
+    function loadImage(index, callback) {
+        var imageContainer = imagesElements[index];
+        var galleryItem = currentGallery[index];
+
+        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined')
+            return;  // out-of-bounds or gallery dirty
+
+        if (imageContainer.querySelector('img, video'))
+            // was loaded, cb and bail
+            return callback ? callback() : null;
+
+        // maybe unloaded video
+        while (imageContainer.firstChild)
+            imageContainer.removeChild(imageContainer.firstChild);
+
+        var imageElement = galleryItem.imageElement,
+            imageSrc = imageElement.href,
+            is_vid = re_v.test(imageSrc),
+            thumbnailElement = imageElement.querySelector('img, video'),
+            imageCaption = typeof options.captions === 'function' ?
+                options.captions.call(currentGallery, imageElement) :
+                imageElement.getAttribute('data-caption') || imageElement.title;
+
+        imageSrc += imageSrc.indexOf('?') < 0 ? '?cache' : '&cache';
+
+        if (is_vid && index != currentIndex)
+            return;  // no preload
+
+        var figure = mknod('figure');
+        figure.id = 'bbox-figure-' + index;
+        figure.innerHTML = '<div class="bbox-spinner">' +
+            '<div class="bbox-double-bounce1"></div>' +
+            '<div class="bbox-double-bounce2"></div>' +
+            '</div>';
+
+        if (options.captions && imageCaption) {
+            var figcaption = mknod('figcaption');
+            figcaption.id = 'bbox-figcaption-' + index;
+            figcaption.innerHTML = imageCaption;
+            figure.appendChild(figcaption);
+        }
+        imageContainer.appendChild(figure);
+
+        var image = mknod(is_vid ? 'video' : 'img');
+        clmod(imageContainer, 'vid', is_vid);
+
+        image.addEventListener(is_vid ? 'loadedmetadata' : 'load', function () {
+            // Remove loader element
+            var spinner = QS('#baguette-img-' + index + ' .bbox-spinner');
+            figure.removeChild(spinner);
+            if (!options.async && callback)
+                callback();
+        });
+        image.setAttribute('src', imageSrc);
+        if (is_vid) {
+            image.setAttribute('controls', 'controls');
+            image.onended = vidEnd;
+        }
+        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (options.titleTag && imageCaption)
+            image.title = imageCaption;
+
+        figure.appendChild(image);
+
+        if (options.async && callback)
+            callback();
+    }
+
+    function showNextImage(e) {
+        ev(e);
+        return show(currentIndex + 1);
+    }
+
+    function showPreviousImage(e) {
+        ev(e);
+        return show(currentIndex - 1);
+    }
+
+    function showFirstImage(e) {
+        if (e)
+            e.preventDefault();
+
+        return show(0);
+    }
+
+    function showLastImage(e) {
+        if (e)
+            e.preventDefault();
+
+        return show(currentGallery.length - 1);
+    }
+
+    function show(index, gallery) {
+        if (!isOverlayVisible && index >= 0 && index < gallery.length) {
+            prepareOverlay(gallery, options);
+            showOverlay(index);
+            return true;
+        }
+        if (index < 0) {
+            if (options.animation)
+                bounceAnimation('left');
+
+            return false;
+        }
+        if (index >= imagesElements.length) {
+            if (options.animation)
+                bounceAnimation('right');
+
+            return false;
+        }
+
+        var v = vid();
+        if (v) {
+            v.src = '';
+            v.load();
+            v.parentNode.removeChild(v);
+        }
+
+        currentIndex = index;
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+        updateOffset();
+
+        if (options.onChange)
+            options.onChange(currentIndex, imagesElements.length);
+
+        return true;
+    }
+
+    var prev_cw = 0, prev_ch = 0, unrot_timer = null;
+    function rotn(n) {
+        var el = vidimg(),
+            orot = parseInt(el.getAttribute('rot') || 0),
+            frot = orot + (n || 0) * 90;
+
+        if (!frot && !orot)
+            return;  // reflow noop
+
+        var co = ebi('bbox-overlay'),
+            cw = co.clientWidth,
+            ch = co.clientHeight;
+
+        if (!n && prev_cw === cw && prev_ch === ch)
+            return;  // reflow noop
+
+        prev_cw = cw;
+        prev_ch = ch;
+        var rot = frot,
+            iw = el.naturalWidth || el.videoWidth,
+            ih = el.naturalHeight || el.videoHeight,
+            magic = 4,  // idk, works in enough browsers
+            dl = el.closest('div').querySelector('figcaption a'),
+            vw = cw,
+            vh = ch - dl.offsetHeight + magic,
+            pmag = Math.min(1, Math.min(vw / ih, vh / iw)),
+            wmag = Math.min(1, Math.min(vw / iw, vh / ih));
+
+        while (rot < 0) rot += 360;
+        while (rot >= 360) rot -= 360;
+        var q = rot == 90 || rot == 270 ? 1 : 0,
+            mag = q ? pmag : wmag;
+
+        el.style.cssText = 'max-width:none; max-height:none; position:absolute; display:block; margin:0';
+        if (!orot) {
+            el.style.width = iw * wmag + 'px';
+            el.style.height = ih * wmag + 'px';
+            el.style.left = (vw - iw * wmag) / 2 + 'px';
+            el.style.top = (vh - ih * wmag) / 2 - magic + 'px';
+            q = el.offsetHeight;
+        }
+        el.style.width = iw * mag + 'px';
+        el.style.height = ih * mag + 'px';
+        el.style.left = (vw - iw * mag) / 2 + 'px';
+        el.style.top = (vh - ih * mag) / 2 - magic + 'px';
+        el.style.transform = 'rotate(' + frot + 'deg)';
+        el.setAttribute('rot', frot);
+        timer.add(rotn);
+        if (!rot) {
+            clearTimeout(unrot_timer);
+            unrot_timer = setTimeout(unrot, 300);
+        }
+    }
+    function rotl() {
+        rotn(-1);
+    }
+    function rotr() {
+        rotn(1);
+    }
+    function unrot() {
+        var el = vidimg(),
+            orot = el.getAttribute('rot'),
+            rot = parseInt(orot || 0);
+
+        while (rot < 0) rot += 360;
+        while (rot >= 360) rot -= 360;
+        if (rot || orot === null)
+            return;
+
+        clmod(el, 'nt', 1);
+        el.removeAttribute('rot');
+        el.removeAttribute("style");
+        rot = el.offsetHeight;
+        clmod(el, 'nt');
+        timer.rm(rotn);
+    }
+
+    function vid() {
+        return imagesElements[currentIndex].querySelector('video');
+    }
+
+    function vidimg() {
+        return imagesElements[currentIndex].querySelector('img, video');
+    }
+
+    function playvid(play) {
+        if (vid())
+            vid()[play ? 'play' : 'pause']();
+    }
+
+    function playpause() {
+        var v = vid();
+        if (v)
+            v[v.paused ? "play" : "pause"]();
+    }
+
+    function relseek(sec) {
+        if (vid())
+            vid().currentTime += sec;
+    }
+
+    function vidEnd() {
+        if (this == vid() && vnext)
+            showNextImage();
+    }
+
+    function mp_ctl() {
+        var v = vid();
+        if (!vmute && v && mp.au && !mp.au.paused) {
+            mp.fade_out();
+            resume_mp = true;
+        }
+        else if (resume_mp && (vmute || !v) && mp.au && mp.au.paused) {
+            mp.fade_in();
+            resume_mp = false;
+        }
+    }
+
+    function bounceAnimation(direction) {
+        slider.className = 'bounce-from-' + direction;
+        setTimeout(function () {
+            slider.className = '';
+        }, 400);
+    }
+
+    function updateOffset() {
+        var offset = -currentIndex * 100 + '%',
+            xform = slider.style.perspective !== undefined;
+
+        if (options.animation === 'fadeIn') {
+            slider.style.opacity = 0;
+            setTimeout(function () {
+                xform ?
+                    slider.style.transform = 'translate3d(' + offset + ',0,0)' :
+                    slider.style.left = offset;
+                slider.style.opacity = 1;
+            }, 400);
+        } else {
+            xform ?
+                slider.style.transform = 'translate3d(' + offset + ',0,0)' :
+                slider.style.left = offset;
+        }
+        playvid(false);
+        var v = vid();
+        if (v) {
+            playvid(true);
+            v.muted = vmute;
+            v.loop = vloop;
+        }
+        selbg();
+        mp_ctl();
+        setVmode();
+
+        var el = vidimg();
+        if (el.getAttribute('rot'))
+            timer.add(rotn);
+        else
+            timer.rm(rotn);
+
+        var prev = QS('.full-image.vis');
+        if (prev)
+            clmod(prev, 'vis');
+
+        clmod(el.closest('div'), 'vis', 1);
+    }
+
+    function preloadNext(index) {
+        if (index - currentIndex >= options.preload)
+            return;
+
+        loadImage(index + 1, function () {
+            preloadNext(index + 1);
+        });
+    }
+
+    function preloadPrev(index) {
+        if (currentIndex - index >= options.preload)
+            return;
+
+        loadImage(index - 1, function () {
+            preloadPrev(index - 1);
+        });
+    }
+
+    function bind(element, event, callback, options) {
+        element.addEventListener(event, callback, options);
+    }
+
+    function unbind(element, event, callback, options) {
+        element.removeEventListener(event, callback, options);
+    }
+
+    function destroyPlugin() {
+        unbindEvents();
+        clearCachedData();
+        unbind(document, 'keydown', keyDownHandler);
+        unbind(document, 'keyup', keyUpHandler);
+        document.getElementsByTagName('body')[0].removeChild(ebi('bbox-overlay'));
+        data = {};
+        currentGallery = [];
+        currentIndex = 0;
+    }
+
+    return {
+        run: run,
+        show: show,
+        showNext: showNextImage,
+        showPrevious: showPreviousImage,
+        relseek: relseek,
+        playpause: playpause,
+        hide: hideOverlay,
+        destroy: destroyPlugin
+    };
+})();

copyparty/web/browser.html

@@ -2,120 +2,143 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <title>⇆🎉 {{ title }}</title>
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
+	<meta charset="utf-8">
+	<title>⇆🎉 {{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
+	{%- if css %}
+	<link rel="stylesheet" media="screen" href="{{ css }}?_={{ ts }}">
+	{%- endif %}
 </head>
 
 <body>
-    <div id="ops">
-        <a href="#" data-dest="">---</a>
-        <a href="#" data-perm="read" data-dest="search">🔎</a>
-        {%- if have_up2k_idx %}
-        <a href="#" data-dest="up2k">🚀</a>
-        {%- else %}
-        <a href="#" data-perm="write" data-dest="up2k">🚀</a>
-        {%- endif %}
-        <a href="#" data-perm="write" data-dest="bup">🎈</a>
-        <a href="#" data-perm="write" data-dest="mkdir">📂</a>
-        <a href="#" data-perm="write" data-dest="new_md">📝</a>
-        <a href="#" data-perm="write" data-dest="msg">📟</a>
-    </div>
+	<div id="ops"></div>
 
-    <div id="op_search" class="opview">
-        {%- if have_tags_idx %}
-        <table id="srch_form" class="tags"></table>
-        {%- else %}
-        <table id="srch_form"></table>
-        {%- endif %}
-        <div id="srch_q"></div>
-    </div>
-    {%- include 'upload.html' %}
-    
-    <h1 id="path">
-        <a href="#" id="entree">🌲</a>
-        {%- for n in vpnodes %}
-        <a href="/{{ n[0] }}">{{ n[1] }}</a>
-        {%- endfor %}
-    </h1>
-    
-    <div id="pro" class="logue">{{ logues[0] }}</div>
+	<div id="op_search" class="opview">
+		{%- if have_tags_idx %}
+		<div id="srch_form" class="tags"></div>
+		{%- else %}
+		<div id="srch_form"></div>
+		{%- endif %}
+		<div id="srch_q"></div>
+	</div>
 
-    <table id="treetab">
-        <tr>
-            <td id="tree">
-                <a href="#" id="detree">🍞...</a>
-                <a href="#" step="2" id="twobytwo">+</a>
-                <a href="#" step="-2" id="twig">&ndash;</a>
-                <a href="#" id="dyntree">a</a>
-                <ul id="treeul"></ul>
-            </td>
-            <td id="treefiles"></td>
-        </tr>
-    </table>
+	<div id="op_player" class="opview opbox opwide"></div>
 
-    <table id="files">
-        <thead>
-            <tr>
-                <th></th>
-                <th><span>File Name</span></th>
-                <th sort="int"><span>Size</span></th>
-                {%- for k in taglist %}
-                    {%- if k.startswith('.') %}
-                        <th sort="int"><span>{{ k[1:] }}</span></th>
-                    {%- else %}
-                        <th><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
-                    {%- endif %}
-                {%- endfor %}
-                <th><span>T</span></th>
-                <th><span>Date</span></th>
-            </tr>
-        </thead>
-        <tbody>
+	<div id="op_bup" class="opview opbox act">
+		<div id="u2err"></div>
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="bput" />
+			<input type="file" name="f" multiple><br />
+			<input type="submit" value="start upload">
+		</form>
+	</div>
+
+	<div id="op_mkdir" class="opview opbox act">
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="mkdir" />
+			📂<input type="text" name="name" size="30">
+			<input type="submit" value="make directory">
+		</form>
+	</div>
+
+	<div id="op_new_md" class="opview opbox">
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="new_md" />
+			📝<input type="text" name="name" size="30">
+			<input type="submit" value="new markdown doc">
+		</form>
+	</div>
+
+	<div id="op_msg" class="opview opbox act">
+		<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
+			📟<input type="text" name="msg" size="30">
+			<input type="submit" value="send msg to server log">
+		</form>
+	</div>
+
+	<div id="op_unpost" class="opview opbox"></div>
+
+	<div id="op_up2k" class="opview"></div>
+
+	<div id="op_cfg" class="opview opbox opwide"></div>
+	
+	<h1 id="path">
+		<a href="#" id="entree" tt="show navpane (directory tree sidebar)$NHotkey: B">🌲</a>
+		{%- for n in vpnodes %}
+		<a href="/{{ n[0] }}">{{ n[1] }}</a>
+		{%- endfor %}
+	</h1>
+	
+	<div id="tree"></div>
+
+<div id="wrap">
+
+	<div id="pro" class="logue">{{ logues[0] }}</div>
+
+	<table id="files">
+		<thead>
+			<tr>
+				<th name="lead"><span>c</span></th>
+				<th name="href"><span>File Name</span></th>
+				<th name="sz" sort="int"><span>Size</span></th>
+				{%- for k in taglist %}
+					{%- if k.startswith('.') %}
+				<th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
+					{%- else %}
+				<th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
+					{%- endif %}
+				{%- endfor %}
+				<th name="ext"><span>T</span></th>
+				<th name="ts"><span>Date</span></th>
+			</tr>
+		</thead>
+<tbody>
 
 {%- for f in files %}
-    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td>
-    {%- if f.tags is defined %}
-        {%- for k in taglist %}
-            <td>{{ f.tags[k] }}</td>
-        {%- endfor %}
-    {%- endif %}
-    <td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td>
+	{%- if f.tags is defined %}
+		{%- for k in taglist %}
+<td>{{ f.tags[k] }}</td>
+		{%- endfor %}
+	{%- endif %}
+<td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
-        </tbody>
-    </table>
-    
-    <div id="epi" class="logue">{{ logues[1] }}</div>
+		</tbody>
+	</table>
+	
+	<div id="epi" class="logue">{{ logues[1] }}</div>
 
-    <h2><a href="?h">control-panel</a></h2>
+	<h2><a href="/?h">control-panel</a></h2>
+	
+	<a href="#" id="repl">π</a>
 
-    {%- if srv_info %}
-    <div id="srv_info"><span>{{ srv_info }}</span></div>
-    {%- endif %}
+</div>
 
-    <div id="widget">
-        <div id="wtoggle">♫</div>
-        <div id="widgeti">
-            <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
-            <canvas id="pvol" width="288" height="38"></canvas>
-            <canvas id="barpos"></canvas>
-            <canvas id="barbuf"></canvas>
-        </div>
-    </div>
+	{%- if srv_info %}
+	<div id="srv_info"><span>{{ srv_info }}</span></div>
+	{%- endif %}
 
-    <script>
-        var tag_order_cfg = {{ tag_order }};
-    </script>
-    <script src="/.cpr/util.js{{ ts }}"></script>
-    <script src="/.cpr/browser.js{{ ts }}"></script>
-    <script src="/.cpr/up2k.js{{ ts }}"></script>
-    <script>
-        apply_perms({{ perms }});
-    </script>
+	<div id="widget"></div>
+
+	<script>
+		var acct = "{{ acct }}",
+			perms = {{ perms }},
+			def_hcols = {{ def_hcols|tojson }},
+			have_up2k_idx = {{ have_up2k_idx|tojson }},
+			have_tags_idx = {{ have_tags_idx|tojson }},
+			have_mv = {{ have_mv|tojson }},
+			have_del = {{ have_del|tojson }},
+			have_unpost = {{ have_unpost|tojson }},
+			have_zip = {{ have_zip|tojson }},
+			readme = {{ readme|tojson }};
+	</script>
+	<script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/browser.js?_={{ ts }}"></script>
+	<script src="/.cpr/up2k.js?_={{ ts }}"></script>
 </body>
 
 </html>

copyparty/web/browser2.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>{{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<style>
+		html{font-family:sans-serif}
+		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
+		a{display:block}
+	</style>
+</head>
+
+<body>
+	{%- if srv_info %}
+	<p><span>{{ srv_info }}</span></p>
+	{%- endif %}
+
+	{%- if have_b_u %}
+	<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+		<input type="hidden" name="act" value="bput" />
+		<input type="file" name="f" multiple /><br />
+		<input type="submit" value="start upload" />
+	</form>
+	<br />
+	{%- endif %}
+
+	{%- if logues[0] %}
+	<div>{{ logues[0] }}</div><br />
+	{%- endif %}
+
+	<table id="files">
+		<thead>
+			<tr>
+				<th name="lead"><span>c</span></th>
+				<th name="href"><span>File Name</span></th>
+				<th name="sz" sort="int"><span>Size</span></th>
+				<th name="ts"><span>Date</span></th>
+			</tr>
+		</thead>
+		<tbody>
+<tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+
+{%- for f in files %}
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+{%- endfor %}
+
+		</tbody>
+	</table>
+	
+	{%- if logues[1] %}
+	<div>{{ logues[1] }}</div><br />
+	{%- endif %}
+	
+	<h2><a href="/{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
+
+</body>
+</html>

copyparty/web/dbg-audio.js

@@ -0,0 +1,61 @@
+var ofun = audio_eq.apply.bind(audio_eq);
+audio_eq.apply = function () {
+    var ac1 = mp.ac;
+    ofun();
+    var ac = mp.ac,
+        w = 2048,
+        h = 256;
+
+    if (!audio_eq.filters.length) {
+        audio_eq.ana = null;
+        return;
+    }
+
+    var can = ebi('fft_can');
+    if (!can) {
+        can = mknod('canvas');
+        can.setAttribute('id', 'fft_can');
+        can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
+        document.body.appendChild(can);
+        can.width = w;
+        can.height = h;
+    }
+    var cc = can.getContext('2d');
+    if (!ac)
+        return;
+
+    var ana = ac.createAnalyser();
+    ana.smoothingTimeConstant = 0;
+    ana.fftSize = 8192;
+
+    audio_eq.filters[0].connect(ana);
+    audio_eq.ana = ana;
+
+    var buf = new Uint8Array(ana.frequencyBinCount),
+        colw = can.width / buf.length;
+
+    cc.fillStyle = '#fc0';
+    function draw() {
+        if (ana == audio_eq.ana)
+            requestAnimationFrame(draw);
+
+        ana.getByteFrequencyData(buf);
+
+        cc.clearRect(0, 0, can.width, can.height);
+
+        /*var x = 0, w = 1;
+        for (var a = 0; a < buf.length; a++) {
+            cc.fillRect(x, h - buf[a], w, h);
+            x += w;
+        }*/
+        var mul = Math.pow(w, 4) / buf.length;
+        for (var x = 0; x < w; x++) {
+            var a = Math.floor(Math.pow(x, 4) / mul),
+                v = buf[a];
+
+            cc.fillRect(x, h - v, 1, v);
+        }
+    }
+    draw();
+};
+audio_eq.apply();

copyparty/web/md.css

@@ -1,13 +1,17 @@
-@font-face {
-	font-family: 'scp';
-	src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(/.cpr/deps/scp.woff2) format('woff2');
-}
 html, body {
 	color: #333;
 	background: #eee;
 	font-family: sans-serif;
 	line-height: 1.5em;
 }
+#repl {
+	position: absolute;
+	top: 0;
+	right: .5em;
+	border: none;
+	color: inherit;
+	background: none;
+}
 #mtw {
 	display: none;
 }
@@ -15,119 +19,12 @@ html, body {
 	margin: 0 auto;
 	padding: 0 1.5em;
 }
-pre, code, a {
-	color: #480;
-	background: #f7f7f7;
-	border: .07em solid #ddd;
-	border-radius: .2em;
-	padding: .1em .3em;
-	margin: 0 .1em;
+#toast {
+	bottom: auto;
+	top: 1.4em;
 }
-code {
-	font-size: .96em;
-}
-pre, code {
-	font-family: 'scp', monospace, monospace;
-	white-space: pre-wrap;
-	word-break: break-all;
-}
-pre {
-	counter-reset: precode;
-}
-pre code {
-	counter-increment: precode;
-	display: inline-block;
-	margin: 0 -.3em;
-	padding: .4em .5em;
-	border: none;
-	border-bottom: 1px solid #cdc;
-	min-width: calc(100% - .6em);
-	line-height: 1.1em;
-}
-pre code:last-child {
-	border-bottom: none;
-}
-pre code::before {
-	content: counter(precode);
-	-webkit-user-select: none;
-	display: inline-block;
-	text-align: right;
-	font-size: .75em;
-	color: #48a;
-	width: 4em;
-	padding-right: 1.5em;
-	margin-left: -5.5em;
-}
-pre code:hover {
-	background: #fec;
-	color: #360;
-}
-h1, h2 {
-	line-height: 1.5em;
-}
-h1 {
-	font-size: 1.7em;
-	text-align: center;
-	border: 1em solid #777;
-	border-width: .05em 0;
-	margin: 3em 0;
-}
-h2 {
-	font-size: 1.5em;
-	font-weight: normal;
-	background: #f7f7f7;
-	border-top: .07em solid #fff;
-	border-bottom: .07em solid #bbb;
-	border-radius: .5em .5em 0 0;
-	padding-left: .4em;
-	margin-top: 3em;
-}
-h3 {
-	border-bottom: .1em solid #999;
-}
-h1 a, h3 a, h5 a,
-h2 a, h4 a, h6 a {
-	color: inherit;
-	display: block;
-	background: none;
-	border: none;
-	padding: 0;
-	margin: 0;
-}
-#mp ul,
-#mp ol {
-	border-left: .3em solid #ddd;
-}
-#m>ul,
-#m>ol {
-	border-color: #bbb;
-}
-#mp ul>li {
-	list-style-type: disc;
-}
-#mp ul>li,
-#mp ol>li {
-	margin: .7em 0;
-}
-strong {
-	color: #000;
-}
-p>em,
-li>em,
-td>em {
-	color: #c50;
-	padding: .1em;
-	border-bottom: .1em solid #bbb;
-}
-blockquote {
-	font-family: serif;
-	background: #f7f7f7;
-	border: .07em dashed #ccc;
-	padding: 0 2em;
-	margin: 1em 0;
-}
-small {
-	opacity: .8;
+a {
+	text-decoration: none;
 }
 #toc {
 	margin: 0 1em;
@@ -163,7 +60,7 @@ small {
 	z-index: 99;
 	position: relative;
 	display: inline-block;
-	font-family: monospace, monospace;
+	font-family: 'scp', monospace, monospace;
 	font-weight: bold;
 	font-size: 1.3em;
 	line-height: .1em;
@@ -175,14 +72,6 @@ small {
 	color: #6b3;
 	text-shadow: .02em 0 0 #6b3;
 }
-table {
-	border-collapse: collapse;
-	margin: 1em 0;
-}
-th, td {
-	padding: .2em .5em;
-	border: .12em solid #aaa;
-}
 blink {
 	animation: blinker .7s cubic-bezier(.9, 0, .1, 1) infinite;
 }
@@ -195,6 +84,36 @@ blink {
 	}
 }
 
+
+.mdo pre {
+	counter-reset: precode;
+}
+.mdo pre code {
+	counter-increment: precode;
+	display: inline-block;
+	border: none;
+	border-bottom: 1px solid #cdc;
+	min-width: calc(100% - .6em);
+}
+.mdo pre code:last-child {
+	border-bottom: none;
+}
+.mdo pre code::before {
+	content: counter(precode);
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+	display: inline-block;
+	text-align: right;
+	font-size: .75em;
+	color: #48a;
+	width: 4em;
+	padding-right: 1.5em;
+	margin-left: -5.5em;
+}
+
+
 @media screen {
 	html, body {
 		margin: 0;
@@ -211,34 +130,6 @@ blink {
 	#mp {
 		max-width: 52em;
 		margin-bottom: 6em;
-		word-break: break-word;
-		overflow-wrap: break-word;
-		word-wrap: break-word; /*ie*/
-	}
-	a {
-		color: #fff;
-		background: #39b;
-		text-decoration: none;
-		padding: 0 .3em;
-		border: none;
-		border-bottom: .07em solid #079;
-	}
-	h2 {
-		color: #fff;
-		background: #555;
-		margin-top: 2em;
-		border-bottom: .22em solid #999;
-		border-top: none;
-	}
-	h1 {
-		color: #fff;
-		background: #444;
-		font-weight: normal;
-		border-top: .4em solid #fb0;
-		border-bottom: .4em solid #777;
-		border-radius: 0 1em 0 1em;
-		margin: 3em 0 1em 0;
-		padding: .5em 0;
 	}
 	#mn {
 		padding: 1.3em 0 .7em 1em;
@@ -291,6 +182,8 @@ blink {
 		color: #444;
 		background: none;
 		text-decoration: underline;
+		margin: 0 .1em;
+		padding: 0 .3em;
 		border: none;
 	}
 	#mh a:hover {
@@ -339,55 +232,6 @@ blink {
 	html.dark #toc li {
 		border-width: 0;
 	}
-	html.dark #mp a {
-		background: #057;
-	}
-	html.dark #mp h1 a, html.dark #mp h4 a,
-	html.dark #mp h2 a, html.dark #mp h5 a,
-	html.dark #mp h3 a, html.dark #mp h6 a {
-		color: inherit;
-		background: none;
-	}
-	html.dark pre,
-	html.dark code {
-		color: #8c0;
-		background: #1a1a1a;
-		border: .07em solid #333;
-	}
-	html.dark #mp ul,
-	html.dark #mp ol {
-		border-color: #444;
-	}
-	html.dark #m>ul,
-	html.dark #m>ol {
-		border-color: #555;
-	}
-	html.dark strong {
-		color: #fff;
-	}
-	html.dark p>em,
-	html.dark li>em,
-	html.dark td>em {
-		color: #f94;
-		border-color: #666;
-	}
-	html.dark h1 {
-		background: #383838;
-		border-top: .4em solid #b80;
-		border-bottom: .4em solid #4c4c4c;
-	}
-	html.dark h2 {
-		background: #444;
-		border-bottom: .22em solid #555;
-	}
-	html.dark td,
-	html.dark th {
-		border-color: #444;
-	}
-	html.dark blockquote {
-		background: #282828;
-		border: .07em dashed #444;
-	}
 	html.dark #mn a:not(:last-child)::after {
 		border-color: rgba(255,255,255,0.3);
 	}
@@ -493,12 +337,15 @@ blink {
 		mso-footer-margin: .6in;
 		mso-paper-source: 0;
 	}
-	a {
+	.mdo a {
 		color: #079;
 		text-decoration: none;
 		border-bottom: .07em solid #4ac;
 		padding: 0 .3em;
 	}
+	#repl {
+		display: none;
+	}
 	#toc>ul {
 		border-left: .1em solid #84c4dd;
 	}
@@ -523,18 +370,20 @@ blink {
 	a[ctr]::before {
 		content: attr(ctr) '. ';
 	}
-	h1 {
+	.mdo h1 {
 		margin: 2em 0;
 	}
-	h2 {
+	.mdo h2 {
 		margin: 2em 0 0 0;
 	}
-	h1, h2, h3 {
+	.mdo h1,
+	.mdo h2,
+	.mdo h3 {
 		page-break-inside: avoid;
 	}
-	h1::after,
-	h2::after,
-	h3::after {
+	.mdo h1::after,
+	.mdo h2::after,
+	.mdo h3::after {
 		content: 'orz';
 		color: transparent;
 		display: block;
@@ -542,20 +391,20 @@ blink {
 		padding: 4em 0 0 0;
 		margin: 0 0 -5em 0;
 	}
-	p {
+	.mdo p {
 		page-break-inside: avoid;
 	}
-	table {
+	.mdo table {
 		page-break-inside: auto;
 	}
-	tr {
+	.mdo tr {
 		page-break-inside: avoid;
 		page-break-after: auto;
 	}
-	thead {
+	.mdo thead {
 		display: table-header-group;
 	}
-	tfoot {
+	.mdo tfoot {
 		display: table-footer-group;
 	}
 	#mp a.vis::after {
@@ -563,40 +412,32 @@ blink {
 		border-bottom: 1px solid #bbb;
 		color: #444;
 	}
-	blockquote {
+	.mdo blockquote {
 		border-color: #555;
 	}
-	code {
+	.mdo code {
 		border-color: #bbb;
 	}
-	pre, pre code {
+	.mdo pre,
+	.mdo pre code {
 		border-color: #999;
 	}
-	pre code::before {
+	.mdo pre code::before {
 		color: #058;
 	}
 
 
 	
-	html.dark a {
+	html.dark .mdo a {
 		color: #000;
 	}
-	html.dark pre,
-	html.dark code {
+	html.dark .mdo pre,
+	html.dark .mdo code {
 		color: #240;
 	}
-	html.dark p>em,
-	html.dark li>em,
-	html.dark td>em {
+	html.dark .mdo p>em,
+	html.dark .mdo li>em,
+	html.dark .mdo td>em {
 		color: #940;
 	}
 }
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-*/

copyparty/web/md.html

@@ -1,11 +1,12 @@
 <!DOCTYPE html><html><head>
 	<meta charset="utf-8">
-	<title>📝🎉 {{ title }}</title> <!-- 📜 -->
+	<title>📝🎉 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
-	<link href="/.cpr/md.css" rel="stylesheet">
+	<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/md.css?_={{ ts }}">
 	{%- if edit %}
-	<link href="/.cpr/md2.css" rel="stylesheet">
+	<link rel="stylesheet" href="/.cpr/md2.css?_={{ ts }}">
 	{%- endif %}
 </head>
 <body>
@@ -14,9 +15,9 @@
 		<a id="lightswitch" href="#">go dark</a>
 		<a id="navtoggle" href="#">hide nav</a>
 		{%- if edit %}
-			<a id="save" href="?edit">save</a>
-			<a id="sbs" href="#">sbs</a>
-			<a id="nsbs" href="#">editor</a>
+			<a id="save" href="?edit" tt="Hotkey: ctrl-s">save</a>
+			<a id="sbs" href="#" tt="editor and preview side by side">sbs</a>
+			<a id="nsbs" href="#" tt="switch between editor and preview$NHotkey: ctrl-e">editor</a>
 			<div id="toolsbox">
 				<a id="tools" href="#">tools</a>
 				<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
@@ -26,8 +27,8 @@
 				<a id="help" href="#">help</a>
 			</div>
 		{%- else %}
-			<a href="?edit">edit (basic)</a>
-			<a href="?edit2">edit (fancy)</a>
+			<a href="?edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
+			<a href="?edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
 			<a href="?raw">view raw</a>
 		{%- endif %}
 	</div>
@@ -42,8 +43,9 @@
 				if you're still reading this, check that javascript is allowed
 			</div>
 		</div>
-		<div id="mp"></div>
+		<div id="mp" class="mdo"></div>
 	</div>
+	<a href="#" id="repl">π</a>
 	
 	{%- if edit %}
 	<div id="helpbox">
@@ -131,25 +133,25 @@ var md_opt = {
 };
 
 (function () {
-    var btn = document.getElementById("lightswitch");
-    var toggle = function (e) {
-		if (e) e.preventDefault();
-        var dark = !document.documentElement.getAttribute("class");
-        document.documentElement.setAttribute("class", dark ? "dark" : "");
-        btn.innerHTML = "go " + (dark ? "light" : "dark");
-        if (window.localStorage)
-            localStorage.setItem('darkmode', dark ? 1 : 0);
-    };
-    btn.onclick = toggle;
-    if (window.localStorage && localStorage.getItem('darkmode') == 1)
-		toggle();
+    var l = localStorage,
+		drk = l.getItem('lightmode') != 1,
+		btn = document.getElementById("lightswitch"),
+		f = function (e) {
+if (e) { e.preventDefault(); drk = !drk; }
+document.documentElement.setAttribute("class", drk? "dark":"light");
+btn.innerHTML = "go " + (drk ? "light":"dark");
+l.setItem('lightmode', drk? 0:1);
+    	};
+	
+	btn.onclick = f;
+	f();
 })();
 
 	</script>
-    <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.full.js"></script>
-	<script src="/.cpr/md.js"></script>
+    <script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
+	<script src="/.cpr/md.js?_={{ ts }}"></script>
 	{%- if edit %}
-	<script src="/.cpr/md2.js"></script>
+	<script src="/.cpr/md2.js?_={{ ts }}"></script>
 	{%- endif %}
 </body></html>

copyparty/web/md.js

@@ -24,29 +24,12 @@ var dbg = function () { };
 var md_plug = {};
 
 
-function hesc(txt) {
-    return txt.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-}
-
-
-function cls(dom, name, add) {
-    var re = new RegExp('(^| )' + name + '( |$)');
-    var lst = (dom.getAttribute('class') + '').replace(re, "$1$2").replace(/  /, "");
-    dom.setAttribute('class', lst + (add ? ' ' + name : ''));
-}
-
-
-function statify(obj) {
-    return JSON.parse(JSON.stringify(obj));
-}
-
-
 // dodge browser issues
 (function () {
     var ua = navigator.userAgent;
     if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
         // necessary on ff-68.7 at least
-        var s = document.createElement('style');
+        var s = mknod('style');
         s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
         console.log(s.innerHTML);
         document.head.appendChild(s);
@@ -65,7 +48,7 @@ function statify(obj) {
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = hesc(decodeURIComponent(n[a]));
+        var dec = esc(uricom_dec(n[a])[0]);
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -73,6 +56,26 @@ function statify(obj) {
 })();
 
 
+// image load handler
+var img_load = (function () {
+    var r = {};
+    r.callbacks = [];
+
+    function fire() {
+        for (var a = 0; a < r.callbacks.length; a++)
+            r.callbacks[a]();
+    }
+
+    var timeout = null;
+    r.done = function () {
+        clearTimeout(timeout);
+        timeout = setTimeout(fire, 500);
+    };
+
+    return r;
+})();
+
+
 // faster than replacing the entire html (chrome 1.8x, firefox 1.6x)
 function copydom(src, dst, lv) {
     var sc = src.childNodes,
@@ -175,17 +178,17 @@ function md_plug_err(ex, js) {
         msg = "Line " + ln + ", " + msg;
         var lns = js.split('\n');
         if (ln < lns.length) {
-            o = document.createElement('span');
-            o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
+            o = mknod('span');
+            o.style.cssText = "color:#ac2;font-size:.9em;font-family:'scp',monospace,monospace;display:block";
             o.textContent = lns[ln - 1];
         }
     }
-    errbox = document.createElement('div');
+    errbox = mknod('div');
     errbox.setAttribute('id', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;
     errbox.onclick = function () {
-        alert('' + ex.stack);
+        modal.alert('<pre>' + esc(ex.stack) + '</pre>');
     };
     if (o) {
         errbox.appendChild(o);
@@ -356,6 +359,10 @@ function convert_markdown(md_text, dest_dom) {
 
     copydom(md_dom, dest_dom, 0);
 
+    var imgs = dest_dom.getElementsByTagName('img');
+    for (var a = 0, aa = imgs.length; a < aa; a++)
+        imgs[a].onload = img_load.done;
+
     if (ext && ext[0].render2)
         try {
             ext[0].render2(dest_dom);
@@ -490,6 +497,7 @@ function init_toc() {
 // "main" :p
 convert_markdown(dom_src.value, dom_pre);
 var toc = init_toc();
+img_load.callbacks = [toc.refresh];
 
 
 // scroll handler
@@ -530,3 +538,6 @@ dom_navtgl.onclick = function () {
 
 if (sread('hidenav') == 1)
     dom_navtgl.onclick();
+
+if (window['tt'])
+    tt.init();

copyparty/web/md2.css

@@ -1,126 +1,110 @@
 #toc {
-    display: none;
+	display: none;
 }
 #mtw {
-    display: block;
-    position: fixed;
-    left: .5em;
-    bottom: 0;
-    width: calc(100% - 56em);
+	display: block;
+	position: fixed;
+	left: .5em;
+	bottom: 0;
+	width: calc(100% - 56em);
 }
 #mw {
-    left: calc(100% - 55em);
-    overflow-y: auto;
-    position: fixed;
-    bottom: 0;
+	left: calc(100% - 55em);
+	overflow-y: auto;
+	position: fixed;
+	bottom: 0;
 }
 
 
 /* single-screen */
 #mtw.preview,
 #mw.editor {
-    opacity: 0;
-    z-index: 1;
+	opacity: 0;
+	z-index: 1;
 }
 #mw.preview,
 #mtw.editor {
-    z-index: 5;
+	z-index: 5;
 }
 #mtw.single,
 #mw.single {
-    margin: 0;
-    left: 1em;
-    left: max(1em, calc((100% - 56em) / 2));
+	margin: 0;
+	left: 1em;
+	left: max(1em, calc((100% - 56em) / 2));
 }
 #mtw.single {
-    width: 55em;
-    width: min(55em, calc(100% - 2em));
+	width: 55em;
+	width: min(55em, calc(100% - 2em));
 }
 
 
 #mp {
-    position: relative;
+	position: relative;
 }
 #mt, #mtr {
-    width: 100%;
-    height: calc(100% - 1px);
-    color: #444;
-    background: #f7f7f7;
-    border: 1px solid #999;
-    outline: none;
-    padding: 0;
-    margin: 0;
-    font-family: 'consolas', monospace, monospace;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    word-wrap: break-word; /*ie*/
-    overflow-y: scroll;
-    line-height: 1.3em;
-    font-size: .9em;
-    position: relative;
-    scrollbar-color: #eb0 #f7f7f7;
+	width: 100%;
+	height: calc(100% - 1px);
+	color: #444;
+	background: #f7f7f7;
+	border: 1px solid #999;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	font-family: 'scp', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-word;
+	overflow-wrap: break-word;
+	word-wrap: break-word; /*ie*/
+	overflow-y: scroll;
+	line-height: 1.3em;
+	font-size: .9em;
+	position: relative;
+	scrollbar-color: #eb0 #f7f7f7;
 }
 html.dark #mt {
-    color: #eee;
-    background: #222;
-    border: 1px solid #777;
-    scrollbar-color: #b80 #282828;
+	color: #eee;
+	background: #222;
+	border: 1px solid #777;
+	scrollbar-color: #b80 #282828;
 }
 #mtr {
-    position: absolute;
-    top: 0;
-    left: 0;
+	position: absolute;
+	top: 0;
+	left: 0;
 }
 #save.force-save {
-    color: #400;
-    background: #f97;
-    border-radius: .15em;
+	color: #400;
+	background: #f97;
+	border-radius: .15em;
 }
 html.dark #save.force-save {
-    color: #fca;
-    background: #720;
+	color: #fca;
+	background: #720;
 }
 #save.disabled {
-    opacity: .4;
-}
-#helpbox,
-#toast {
-    background: #f7f7f7;
-    border-radius: .4em;
-    z-index: 9001;
+	opacity: .4;
 }
 #helpbox {
-    display: none;
-    position: fixed;
-    padding: 2em;
-    top: 4em;
-    overflow-y: auto;
-    box-shadow: 0 .5em 2em #777;
-    height: calc(100% - 12em);
-    left: calc(50% - 15em);
-    right: 0;
-    width: 30em;
+	background: #f7f7f7;
+	border-radius: .4em;
+	z-index: 9001;
+	display: none;
+	position: fixed;
+	padding: 2em;
+	top: 4em;
+	overflow-y: auto;
+	box-shadow: 0 .5em 2em #777;
+	height: calc(100% - 12em);
+	left: calc(50% - 15em);
+	right: 0;
+	width: 30em;
 }
 #helpclose {
-    display: block;
+	display: block;
 }
 html.dark #helpbox {
-    box-shadow: 0 .5em 2em #444;
-}
-html.dark #helpbox,
-html.dark #toast {
-    background: #222;
-    border: 1px solid #079;
-    border-width: 1px 0;
-}
-#toast {
-    font-weight: bold;
-    text-align: center;
-    padding: .6em 0;
-    position: fixed;
-    z-index: 9001;
-    top: 30%;
-    transition: opacity 0.2s ease-in-out;
-    opacity: 1;
+	box-shadow: 0 .5em 2em #444;
+	background: #222;
+	border: 1px solid #079;
+	border-width: 1px 0;
 }

copyparty/web/md2.js

@@ -16,7 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = document.createElement('div');
+    var d = mknod('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
@@ -71,7 +71,7 @@ var map_src = [];
 var map_pre = [];
 function genmap(dom, oldmap) {
     var find = nlines;
-    while (oldmap && find --> 0) {
+    while (oldmap && find-- > 0) {
         var tmap = genmapq(dom, '*[data-ln="' + find + '"]');
         if (!tmap || !tmap.length)
             continue;
@@ -94,11 +94,11 @@ var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
     function draw_md() {
-        var t0 = new Date().getTime();
+        var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
 
-        var lines = hesc(src).replace(/\r/g, "").split('\n');
+        var lines = esc(src).replace(/\r/g, "").split('\n');
         nlines = lines.length;
         var html = [];
         for (var a = 0; a < lines.length; a++)
@@ -108,9 +108,9 @@ var draw_md = (function () {
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
 
-        cls(ebi('save'), 'disabled', src == server_md);
+        clmod(ebi('save'), 'disabled', src == server_md);
 
-        var t1 = new Date().getTime();
+        var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
     }
 
@@ -127,6 +127,12 @@ var draw_md = (function () {
 })();
 
 
+// discard TOC callback, just regen editor scroll map
+img_load.callbacks = [function () {
+    map_pre = genmap(dom_pre, map_pre);
+}];
+
+
 // resize handler
 redraw = (function () {
     function onresize() {
@@ -136,7 +142,6 @@ redraw = (function () {
         dom_ref.style.width = getComputedStyle(dom_src).offsetWidth + 'px';
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
-        dbg(document.body.clientWidth + 'x' + document.body.clientHeight);
     }
     function setsbs() {
         dom_wrap.setAttribute('class', '');
@@ -236,7 +241,7 @@ function Modpoll() {
 
         var skip = null;
 
-        if (ebi('toast'))
+        if (toast.visible)
             skip = 'toast';
 
         else if (this.skip_one)
@@ -252,7 +257,7 @@ function Modpoll() {
         }
 
         console.log('modpoll...');
-        var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
         var xhr = new XMLHttpRequest();
         xhr.modpoll = this;
         xhr.open('GET', url, true);
@@ -285,16 +290,15 @@ function Modpoll() {
             console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
             this.modpoll.disabled = true;
             var msg = [
-                "The document has changed on the server.<br />" +
+                "The document has changed on the server.",
                 "The changes will NOT be loaded into your editor automatically.",
-
-                "Press F5 or CTRL-R to refresh the page,<br />" +
+                "",
+                "Press F5 or CTRL-R to refresh the page,",
                 "replacing your document with the server copy.",
-
-                "You can click this message to ignore and contnue."
+                "",
+                "You can close this message to ignore and contnue."
             ];
-            return toast(false, "box-shadow:0 1em 2em rgba(64,64,64,0.8);font-weight:normal",
-                36, "<p>" + msg.join('</p>\n<p>') + '</p>');
+            return toast.warn(0, msg.join('\n'));
         }
 
         console.log('modpoll eq');
@@ -323,57 +327,56 @@ function save(e) {
     var save_btn = ebi("save"),
         save_cls = save_btn.getAttribute('class') + '';
 
-    if (save_cls.indexOf('disabled') >= 0) {
-        toast(true, ";font-size:2em;color:#c90", 9, "no changes");
-        return;
-    }
+    if (save_cls.indexOf('disabled') >= 0)
+        return toast.inf(2, "no changes");
 
     var force = (save_cls.indexOf('force-save') >= 0);
-    if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
-        alert('ok, aborted');
-        return;
+    function save2() {
+        var txt = dom_src.value,
+            fd = new FormData();
+
+        fd.append("act", "tput");
+        fd.append("lastmod", (force ? -1 : last_modified));
+        fd.append("body", txt);
+
+        var url = (document.location + '').split('?')[0];
+        var xhr = new XMLHttpRequest();
+        xhr.open('POST', url, true);
+        xhr.responseType = 'text';
+        xhr.onreadystatechange = save_cb;
+        xhr.btn = save_btn;
+        xhr.txt = txt;
+
+        modpoll.skip_one = true;  // skip one iteration while we save
+        xhr.send(fd);
     }
 
-    var txt = dom_src.value;
-
-    var fd = new FormData();
-    fd.append("act", "tput");
-    fd.append("lastmod", (force ? -1 : last_modified));
-    fd.append("body", txt);
-
-    var url = (document.location + '').split('?')[0];
-    var xhr = new XMLHttpRequest();
-    xhr.open('POST', url, true);
-    xhr.responseType = 'text';
-    xhr.onreadystatechange = save_cb;
-    xhr.btn = save_btn;
-    xhr.txt = txt;
-
-    modpoll.skip_one = true;  // skip one iteration while we save
-    xhr.send(fd);
+    if (!force)
+        save2();
+    else
+        modal.confirm('confirm that you wish to lose the changes made on the server since you opened this document', save2, function () {
+            toast.inf(3, 'aborted');
+        });
 }
 
 function save_cb() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var r;
     try {
         r = JSON.parse(this.responseText);
     }
     catch (ex) {
-        alert('Failed to parse reply from server:\n\n' + this.responseText);
-        return;
+        return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
     }
 
     if (!r.ok) {
-        if (!this.btn.classList.contains('force-save')) {
-            this.btn.classList.add('force-save');
+        if (!clgot(this.btn, 'force-save')) {
+            clmod(this.btn, 'force-save', 1);
             var msg = [
                 'This file has been modified since you started editing it!\n',
                 'if you really want to overwrite, press save again.\n',
@@ -383,15 +386,13 @@ function save_cb() {
                 r.lastmod + ' lastmod on the server now,',
                 r.now + ' server time now,\n',
             ];
-            alert(msg.join('\n'));
+            return toast.err(0, msg.join('\n'));
         }
-        else {
-            alert('Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
-        }
-        return;
+        else
+            return toast.err(0, 'Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
     }
 
-    this.btn.classList.remove('force-save');
+    clmod(this.btn, 'force-save');
     //alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
 
     run_savechk(r.lastmod, this.txt, this.btn, 0);
@@ -399,7 +400,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
@@ -415,10 +416,8 @@ function savechk_cb() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var doc1 = this.txt.replace(/\r\n/g, "\n");
     var doc2 = this.responseText.replace(/\r\n/g, "\n");
@@ -431,58 +430,22 @@ function savechk_cb() {
             }, 100);
             return;
         }
-        alert(
+        modal.alert(
             'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
             'Length: yours=' + doc1.length + ', server=' + doc2.length
         );
-        alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
-        alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
+        modal.alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
+        modal.alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
         return;
     }
 
     last_modified = this.lastmod;
     server_md = this.txt;
     draw_md();
-    toast(true, ";font-size:6em;font-family:serif;color:#9b4", 4,
-        'OK✔️<span style="font-size:.2em;color:#999;position:absolute">' + this.ntry + '</span>');
-
+    toast.ok(2, 'save OK' + (this.ntry ? '\nattempt ' + this.ntry : ''));
     modpoll.disabled = false;
 }
 
-function toast(autoclose, style, width, msg) {
-    var ok = ebi("toast");
-    if (ok)
-        ok.parentNode.removeChild(ok);
-
-    style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
-    ok = document.createElement('div');
-    ok.setAttribute('id', 'toast');
-    ok.setAttribute('style', style);
-    ok.innerHTML = msg;
-    var parent = ebi('m');
-    document.documentElement.appendChild(ok);
-
-    var hide = function (delay) {
-        delay = delay || 0;
-
-        setTimeout(function () {
-            ok.style.opacity = 0;
-        }, delay);
-
-        setTimeout(function () {
-            if (ok.parentNode)
-                ok.parentNode.removeChild(ok);
-        }, delay + 250);
-    }
-
-    ok.onclick = function () {
-        hide(0);
-    };
-
-    if (autoclose)
-        hide(500);
-}
-
 
 // firefox bug: initial selection offset isn't cleared properly through js
 var ff_clearsel = (function () {
@@ -761,7 +724,7 @@ function fmt_table(e) {
 
         var ind2 = tab[a].match(re_ind)[0];
         if (ind != ind2 && a != 1)  // the table can be a list entry or something, ignore [0]
-            return alert(err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
+            return toast.err(7, err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
 
         var t = tab[a].slice(ind.length);
         t = t.replace(re_lpipe, "");
@@ -771,7 +734,7 @@ function fmt_table(e) {
         if (a == 0)
             ncols = tab[a].length;
         else if (ncols < tab[a].length)
-            return alert(err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
+            return toast.err(7, err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
 
         // if row has less columns than row2, fill them in
         while (tab[a].length < ncols)
@@ -788,7 +751,7 @@ function fmt_table(e) {
     for (var col = 0; col < tab[1].length; col++) {
         var m = tab[1][col].match(re_align);
         if (!m)
-            return alert(err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
+            return toast.err(7, err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
 
         if (m[2]) {
             if (m[1])
@@ -876,10 +839,9 @@ function mark_uni(e) {
         ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g'),
         mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
 
-    if (txt == mod) {
-        alert('no results;  no modifications were made');
-        return;
-    }
+    if (txt == mod)
+        return toast.inf(5, 'no results;  no modifications were made');
+
     dom_src.value = mod;
 }
 
@@ -893,10 +855,9 @@ function iter_uni(e) {
         re = new RegExp('([^' + js_uni_whitelist + ']+)'),
         m = re.exec(txt.slice(ofs));
 
-    if (!m) {
-        alert('no more hits from cursor onwards');
-        return;
-    }
+    if (!m)
+        return toast.inf(5, 'no more hits from cursor onwards');
+
     ofs += m.index;
 
     dom_src.setSelectionRange(ofs, ofs + m[0].length, "forward");
@@ -911,12 +872,10 @@ function iter_uni(e) {
 function cfg_uni(e) {
     if (e) e.preventDefault();
 
-    var reply = prompt("unicode whitelist", esc_uni_whitelist);
-    if (reply === null)
-        return;
-
-    esc_uni_whitelist = reply;
-    js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+    modal.prompt("unicode whitelist", esc_uni_whitelist, function (reply) {
+        esc_uni_whitelist = reply;
+        js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+    }, null);
 }
 
 
@@ -924,10 +883,9 @@ function cfg_uni(e) {
 (function () {
     function keydown(ev) {
         ev = ev || window.event;
-        var kc = ev.keyCode || ev.which;
-        var ctrl = ev.ctrlKey || ev.metaKey;
-        //console.log(ev.code, kc);
-        if (ctrl && (ev.code == "KeyS" || kc == 83)) {
+        var kc = ev.code || ev.keyCode || ev.which;
+        //console.log(ev.key, ev.code, ev.keyCode, ev.which);
+        if (ctrl(ev) && (ev.code == "KeyS" || kc == 83)) {
             save();
             return false;
         }
@@ -936,23 +894,15 @@ function cfg_uni(e) {
             if (d)
                 d.click();
         }
-        if (document.activeElement == dom_src) {
-            if (ev.code == "Tab" || kc == 9) {
-                md_indent(ev.shiftKey);
-                return false;
-            }
-            if (ctrl && (ev.code == "KeyH" || kc == 72)) {
+        if (document.activeElement != dom_src)
+            return true;
+
+        if (ctrl(ev)) {
+            if (ev.code == "KeyH" || kc == 72) {
                 md_header(ev.shiftKey);
                 return false;
             }
-            if (!ctrl && (ev.code == "Home" || kc == 36)) {
-                md_home(ev.shiftKey);
-                return false;
-            }
-            if (!ctrl && !ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
-                return md_newline();
-            }
-            if (ctrl && (ev.code == "KeyZ" || kc == 90)) {
+            if (ev.code == "KeyZ" || kc == 90) {
                 if (ev.shiftKey)
                     action_stack.redo();
                 else
@@ -960,33 +910,45 @@ function cfg_uni(e) {
 
                 return false;
             }
-            if (ctrl && (ev.code == "KeyY" || kc == 89)) {
+            if (ev.code == "KeyY" || kc == 89) {
                 action_stack.redo();
                 return false;
             }
-            if (!ctrl && !ev.shiftKey && kc == 8) {
-                return md_backspace();
-            }
-            if (ctrl && (ev.code == "KeyK")) {
+            if (ev.code == "KeyK") {
                 fmt_table();
                 return false;
             }
-            if (ctrl && (ev.code == "KeyU")) {
+            if (ev.code == "KeyU") {
                 iter_uni();
                 return false;
             }
-            if (ctrl && (ev.code == "KeyE")) {
+            if (ev.code == "KeyE") {
                 dom_nsbs.click();
-                //fmt_table();
                 return false;
             }
             var up = ev.code == "ArrowUp" || kc == 38;
             var dn = ev.code == "ArrowDown" || kc == 40;
-            if (ctrl && (up || dn)) {
+            if (up || dn) {
                 md_p_jump(dn);
                 return false;
             }
         }
+        else {
+            if (ev.code == "Tab" || kc == 9) {
+                md_indent(ev.shiftKey);
+                return false;
+            }
+            if (ev.code == "Home" || kc == 36) {
+                md_home(ev.shiftKey);
+                return false;
+            }
+            if (!ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
+                return md_newline();
+            }
+            if (!ev.shiftKey && kc == 8) {
+                return md_backspace();
+            }
+        }
     }
     document.onkeydown = keydown;
     ebi('save').onclick = save;
@@ -1049,7 +1011,7 @@ action_stack = (function () {
         var p1 = from.length,
             p2 = to.length;
 
-        while (p1 --> 0 && p2 --> 0)
+        while (p1-- > 0 && p2-- > 0)
             if (from[p1] != to[p2])
                 break;
 
@@ -1129,9 +1091,9 @@ action_stack = (function () {
         ref = newtxt;
         dbg('undos(%d) redos(%d)', hist.un.length, hist.re.length);
         if (hist.un.length > 0)
-            dbg(statify(hist.un.slice(-1)[0]));
+            dbg(jcp(hist.un.slice(-1)[0]));
         if (hist.re.length > 0)
-            dbg(statify(hist.re.slice(-1)[0]));
+            dbg(jcp(hist.re.slice(-1)[0]));
     }
 
     return {
@@ -1142,14 +1104,3 @@ action_stack = (function () {
         _ref: ref
     }
 })();
-
-/*
-ebi('help').onclick = function () {
-    var c1 = getComputedStyle(dom_src).cssText.split(';');
-    var c2 = getComputedStyle(dom_ref).cssText.split(';');
-    var max = Math.min(c1.length, c2.length);
-    for (var a = 0; a < max; a++)
-        if (c1[a] !== c2[a])
-            console.log(c1[a] + '\n' + c2[a]);
-}
-*/

copyparty/web/mde.css

@@ -7,315 +7,149 @@ html .editor-toolbar>button.active { border-color: rgba(0,0,0,0.4); background:
 html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
 html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
 
+
+
 html {
-    line-height: 1.5em;
+	line-height: 1.5em;
 }
 html, body {
-    margin: 0;
-    padding: 0;
-    min-height: 100%;
-    font-family: sans-serif;
-    background: #f7f7f7;
-    color: #333;
+	margin: 0;
+	padding: 0;
+	min-height: 100%;
+	font-family: sans-serif;
+	background: #f7f7f7;
+	color: #333;
 }
+#toast {
+	bottom: auto;
+	top: 1.4em;
+}
+#repl {
+	position: absolute;
+	top: 0;
+	right: .5em;
+	border: none;
+	color: inherit;
+	background: none;
+	text-decoration: none;
+}
+
+
+
 #mn {
-    font-weight: normal;
-    margin: 1.3em 0 .7em 1em;
+	font-weight: normal;
+	margin: 1.3em 0 .7em 1em;
 }
 #mn a {
-    color: #444;
-    margin: 0 0 0 -.2em;
-    padding: 0 0 0 .4em;
-    text-decoration: none;
-    /* ie: */
-    border-bottom: .1em solid #777\9;
-    margin-right: 1em\9;
+	color: #444;
+	margin: 0 0 0 -.2em;
+	padding: 0 0 0 .4em;
+	text-decoration: none;
+	/* ie: */
+	border-bottom: .1em solid #777\9;
+	margin-right: 1em\9;
 }
 #mn a:first-child {
-    padding-left: .5em;
+	padding-left: .5em;
 }
 #mn a:last-child {
-    padding-right: .5em;
+	padding-right: .5em;
 }
 #mn a:not(:last-child):after {
-    content: '';
-    width: 1.05em;
-    height: 1.05em;
-    margin: -.2em .3em -.2em -.4em;
-    display: inline-block;
-    border: 1px solid rgba(0,0,0,0.2);
-    border-width: .2em .2em 0 0;
-    transform: rotate(45deg);
+	content: '';
+	width: 1.05em;
+	height: 1.05em;
+	margin: -.2em .3em -.2em -.4em;
+	display: inline-block;
+	border: 1px solid rgba(0,0,0,0.2);
+	border-width: .2em .2em 0 0;
+	transform: rotate(45deg);
 }
 #mn a:hover {
-    color: #000;
-    text-decoration: underline;
+	color: #000;
+	text-decoration: underline;
 }
 
 html .editor-toolbar>button.disabled {
-    opacity: .35;
-    pointer-events: none;
+	opacity: .35;
+	pointer-events: none;
 }
 html .editor-toolbar>button.save.force-save {
-    background: #f97;
-}
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-.cm-header { font-size: .4em !important }
-*/
-
-
-
-
-
-/* copied from md.css for now */
-.mdo pre,
-.mdo code,
-.mdo a {
-	color: #480;
-	background: #f7f7f7;
-	border: .07em solid #ddd;
-	border-radius: .2em;
-	padding: .1em .3em;
-	margin: 0 .1em;
-}
-.mdo code {
-	font-size: .96em;
-}
-.mdo pre,
-.mdo code {
-	font-family: monospace, monospace;
-	white-space: pre-wrap;
-	word-break: break-all;
-}
-.mdo pre code {
-	display: block;
-	margin: 0 -.3em;
-	padding: .4em .5em;
-	line-height: 1.1em;
-}
-.mdo a {
-    color: #fff;
-    background: #39b;
-    text-decoration: none;
-    padding: 0 .3em;
-    border: none;
-    border-bottom: .07em solid #079;
-}
-.mdo h2 {
-    color: #fff;
-    background: #555;
-    margin-top: 2em;
-    border-bottom: .22em solid #999;
-    border-top: none;
-}
-.mdo h1 {
-    color: #fff;
-    background: #444;
-    font-weight: normal;
-    border-top: .4em solid #fb0;
-    border-bottom: .4em solid #777;
-    border-radius: 0 1em 0 1em;
-    margin: 3em 0 1em 0;
-    padding: .5em 0;
-}
-h1, h2 {
-	line-height: 1.5em;
-}
-h1 {
-	font-size: 1.7em;
-	text-align: center;
-	border: 1em solid #777;
-	border-width: .05em 0;
-	margin: 3em 0;
-}
-h2 {
-	font-size: 1.5em;
-	font-weight: normal;
-	background: #f7f7f7;
-	border-top: .07em solid #fff;
-	border-bottom: .07em solid #bbb;
-	border-radius: .5em .5em 0 0;
-	padding-left: .4em;
-	margin-top: 3em;
-}
-.mdo ul,
-.mdo ol {
-	border-left: .3em solid #ddd;
-}
-.mdo>ul,
-.mdo>ol {
-	border-color: #bbb;
-}
-.mdo ul>li {
-	list-style-type: disc;
-}
-.mdo ul>li,
-.mdo ol>li {
-	margin: .7em 0;
-}
-strong {
-	color: #000;
-}
-p>em,
-li>em,
-td>em {
-	color: #c50;
-	padding: .1em;
-	border-bottom: .1em solid #bbb;
-}
-blockquote {
-	font-family: serif;
-	background: #f7f7f7;
-	border: .07em dashed #ccc;
-	padding: 0 2em;
-	margin: 1em 0;
-}
-small {
-	opacity: .8;
-}
-table {
-	border-collapse: collapse;
-}
-td {
-	padding: .2em .5em;
-	border: .12em solid #aaa;
-}
-th {
-	border: .12em solid #aaa;
-}
-
-
-
-
-
-/* mde support */
-.mdo {
-    padding: 1em;
-    background: #f7f7f7;
-}
-html.dark .mdo {
-    background: #1c1c1c;
+	background: #f97;
 }
 .CodeMirror {
-    background: #f7f7f7;
+	background: #f7f7f7;
 }
 
 
 
-
-
 /* darkmode */
 html.dark .mdo,
 html.dark .CodeMirror {
-    border-color: #222;
+	border-color: #222;
 }
 html.dark,
 html.dark body,
 html.dark .CodeMirror {
-    background: #222;
-    color: #ccc;
+	background: #222;
+	color: #ccc;
 }
 html.dark .CodeMirror-cursor {
-    border-color: #fff;
+	border-color: #fff;
 }
 html.dark .CodeMirror-selected {
-    box-shadow: 0 0 1px #0cf inset;
+	box-shadow: 0 0 1px #0cf inset;
 }
 html.dark .CodeMirror-selected,
 html.dark .CodeMirror-selectedtext {
-    border-radius: .1em;
-    background: #246;
-    color: #fff;
-}
-html.dark .mdo a {
-    background: #057;
-}
-html.dark .mdo h1 a, html.dark .mdo h4 a,
-html.dark .mdo h2 a, html.dark .mdo h5 a,
-html.dark .mdo h3 a, html.dark .mdo h6 a {
-    color: inherit;
-    background: none;
-}
-html.dark pre,
-html.dark code {
-    color: #8c0;
-    background: #1a1a1a;
-    border: .07em solid #333;
-}
-html.dark .mdo ul,
-html.dark .mdo ol {
-    border-color: #444;
-}
-html.dark .mdo>ul,
-html.dark .mdo>ol {
-    border-color: #555;
-}
-html.dark strong {
-    color: #fff;
-}
-html.dark p>em,
-html.dark li>em,
-html.dark td>em {
-    color: #f94;
-    border-color: #666;
-}
-html.dark h1 {
-    background: #383838;
-    border-top: .4em solid #b80;
-    border-bottom: .4em solid #4c4c4c;
-}
-html.dark h2 {
-    background: #444;
-    border-bottom: .22em solid #555;
-}
-html.dark td,
-html.dark th {
-    border-color: #444;
-}
-html.dark blockquote {
-    background: #282828;
-    border: .07em dashed #444;
+	border-radius: .1em;
+	background: #246;
+	color: #fff;
 }
 
 
 
 html.dark #mn a {
-    color: #ccc;
+	color: #ccc;
 }
 html.dark #mn a:not(:last-child):after {
-    border-color: rgba(255,255,255,0.3);
+	border-color: rgba(255,255,255,0.3);
 }
 html.dark .editor-toolbar {
-    border-color: #2c2c2c;
-    background: #1c1c1c;
+	border-color: #2c2c2c;
+	background: #1c1c1c;
 }
 html.dark .editor-toolbar>i.separator {
-    border-left: 1px solid #444;
-    border-right: 1px solid #111;
+	border-left: 1px solid #444;
+	border-right: 1px solid #111;
 }
 html.dark .editor-toolbar>button {
-    margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
-    color: #aaa;
+	margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
+	color: #aaa;
 }
 
 
 
 html.dark .editor-toolbar>button:hover {
-    color: #333;
+	color: #333;
 }
 html.dark .editor-toolbar>button.active {
-    color: #333;
-    border-color: #ec1;
-    background: #c90;
+	color: #333;
+	border-color: #ec1;
+	background: #c90;
 }
 html.dark .editor-toolbar::after,
 html.dark .editor-toolbar::before {
-    background: none;
-}
+	background: none;
+}
+
+
+
+/* ui.css overrides */
+.mdo {
+	padding: 1em;
+	background: #f7f7f7;
+}
+html.dark .mdo {
+	background: #1c1c1c;
+}

copyparty/web/mde.html

@@ -3,9 +3,10 @@
 	<title>📝🎉 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
-	<link href="/.cpr/mde.css" rel="stylesheet">
-	<link href="/.cpr/deps/mini-fa.css" rel="stylesheet">
-	<link href="/.cpr/deps/easymde.css" rel="stylesheet">
+	<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/mde.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/deps/mini-fa.css?_={{ ts }}">
+	<link rel="stylesheet" href="/.cpr/deps/easymde.css?_={{ ts }}">
 </head>
 <body>
 	<div id="mw">
@@ -20,6 +21,7 @@
 			<textarea id="mt" style="display:none" autocomplete="off">{{ md }}</textarea>
 		</div>
 	</div>
+	<a href="#" id="repl">π</a>
 	<script>
 
 var last_modified = {{ lastmod }};
@@ -30,20 +32,19 @@ var md_opt = {
 };
 
 var lightswitch = (function () {
-	var fun = function () {
-		var dark = !!!document.documentElement.getAttribute("class");
-		document.documentElement.setAttribute("class", dark ? "dark" : "");
-		if (window.localStorage)
-			localStorage.setItem('darkmode', dark ? 1 : 0);
-	};
-	if (window.localStorage && localStorage.getItem('darkmode') == 1)
-		fun();
-	
-	return fun;
+	var l = localStorage,
+		drk = l.getItem('lightmode') != 1,
+		f = function (e) {
+if (e) drk = !drk;
+document.documentElement.setAttribute("class", drk? "dark":"light");
+l.setItem('lightmode', drk? 0:1);
+		};
+	f();
+	return f;
 })();
 
 	</script>
-    <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/easymde.js"></script>
-	<script src="/.cpr/mde.js"></script>
+    <script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/deps/easymde.js?_={{ ts }}"></script>
+	<script src="/.cpr/mde.js?_={{ ts }}"></script>
 </body></html>

copyparty/web/mde.js

@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = decodeURIComponent(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+        var dec = uricom_dec(n[a])[0].replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -71,11 +71,11 @@ var mde = (function () {
 })();
 
 function set_jumpto() {
-    document.querySelector('.editor-preview-side').onclick = jumpto;
+    QS('.editor-preview-side').onclick = jumpto;
 }
 
 function jumpto(ev) {
-    var tgt = ev.target || ev.srcElement;
+    var tgt = ev.target;
     var ln = null;
     while (tgt && !ln) {
         ln = tgt.getAttribute('data-ln');
@@ -94,67 +94,63 @@ function md_changed(mde, on_srv) {
         window.md_saved = mde.value();
 
     var md_now = mde.value();
-    var save_btn = document.querySelector('.editor-toolbar button.save');
-
-    if (md_now == window.md_saved)
-        save_btn.classList.add('disabled');
-    else
-        save_btn.classList.remove('disabled');
+    var save_btn = QS('.editor-toolbar button.save');
 
+    clmod(save_btn, 'disabled', md_now == window.md_saved);
     set_jumpto();
 }
 
 function save(mde) {
-    var save_btn = document.querySelector('.editor-toolbar button.save');
-    if (save_btn.classList.contains('disabled')) {
-        alert('there is nothing to save');
-        return;
-    }
-    var force = save_btn.classList.contains('force-save');
-    if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
-        alert('ok, aborted');
-        return;
+    var save_btn = QS('.editor-toolbar button.save');
+    if (clgot(save_btn, 'disabled'))
+        return toast.inf(2, 'no changes');
+
+    var force = clgot(save_btn, 'force-save');
+    function save2() {
+        var txt = mde.value();
+
+        var fd = new FormData();
+        fd.append("act", "tput");
+        fd.append("lastmod", (force ? -1 : last_modified));
+        fd.append("body", txt);
+
+        var url = (document.location + '').split('?')[0];
+        var xhr = new XMLHttpRequest();
+        xhr.open('POST', url, true);
+        xhr.responseType = 'text';
+        xhr.onreadystatechange = save_cb;
+        xhr.btn = save_btn;
+        xhr.mde = mde;
+        xhr.txt = txt;
+        xhr.send(fd);
     }
 
-    var txt = mde.value();
-
-    var fd = new FormData();
-    fd.append("act", "tput");
-    fd.append("lastmod", (force ? -1 : last_modified));
-    fd.append("body", txt);
-
-    var url = (document.location + '').split('?')[0];
-    var xhr = new XMLHttpRequest();
-    xhr.open('POST', url, true);
-    xhr.responseType = 'text';
-    xhr.onreadystatechange = save_cb;
-    xhr.btn = save_btn;
-    xhr.mde = mde;
-    xhr.txt = txt;
-    xhr.send(fd);
+    if (!force)
+        save2();
+    else
+        modal.confirm('confirm that you wish to lose the changes made on the server since you opened this document', save2, function () {
+            toast.inf(3, 'aborted');
+        });
 }
 
 function save_cb() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var r;
     try {
         r = JSON.parse(this.responseText);
     }
     catch (ex) {
-        alert('Failed to parse reply from server:\n\n' + this.responseText);
-        return;
+        return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
     }
 
     if (!r.ok) {
-        if (!this.btn.classList.contains('force-save')) {
-            this.btn.classList.add('force-save');
+        if (!clgot(this.btn, 'force-save')) {
+            clmod(this.btn, 'force-save', 1);
             var msg = [
                 'This file has been modified since you started editing it!\n',
                 'if you really want to overwrite, press save again.\n',
@@ -164,15 +160,13 @@ function save_cb() {
                 r.lastmod + ' lastmod on the server now,',
                 r.now + ' server time now,\n',
             ];
-            alert(msg.join('\n'));
+            return toast.err(0, msg.join('\n'));
         }
-        else {
-            alert('Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
-        }
-        return;
+        else
+            return toast.err(0, 'Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
     }
 
-    this.btn.classList.remove('force-save');
+    clmod(this.btn, 'force-save');
     //alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
 
     // download the saved doc from the server and compare
@@ -192,35 +186,23 @@ function save_chk() {
     if (this.readyState != XMLHttpRequest.DONE)
         return;
 
-    if (this.status !== 200) {
-        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
-        return;
-    }
+    if (this.status !== 200)
+        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
     var doc1 = this.txt.replace(/\r\n/g, "\n");
     var doc2 = this.responseText.replace(/\r\n/g, "\n");
     if (doc1 != doc2) {
-        alert(
+        modal.alert(
             'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
             'Length: yours=' + doc1.length + ', server=' + doc2.length
         );
-        alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
-        alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
+        modal.alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
+        modal.alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
         return;
     }
 
     last_modified = this.lastmod;
     md_changed(this.mde, true);
 
-    var ok = document.createElement('div');
-    ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
-    ok.innerHTML = 'OK✔️';
-    var parent = ebi('m');
-    document.documentElement.appendChild(ok);
-    setTimeout(function () {
-        ok.style.opacity = 0;
-    }, 500);
-    setTimeout(function () {
-        ok.parentNode.removeChild(ok);
-    }, 750);
+    toast.ok(2, 'save OK' + (this.ntry ? '\nattempt ' + this.ntry : ''));
 }

copyparty/web/msg.css

@@ -3,7 +3,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -11,21 +11,19 @@ html {
 	background: #333;
 	font-family: sans-serif;
 	text-shadow: 1px 1px 0px #000;
+	touch-action: manipulation;
 }
 html, body {
 	margin: 0;
 	padding: 0;
 }
-body {
-	padding-bottom: 5em;
-}
 #box {
-    padding: .5em 1em;
-    background: #2c2c2c;
+	padding: .5em 1em;
+	background: #2c2c2c;
 }
 pre {
 	font-family: monospace, monospace;
 }
 a {
 	color: #fc5;
-}
+}

copyparty/web/msg.html

@@ -6,7 +6,7 @@
     <title>copyparty</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/msg.css">
+    <link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
 </head>
 
 <body>

copyparty/web/splash.css

@@ -3,6 +3,9 @@ html, body, #wrap {
 	background: #f7f7f7;
 	font-family: sans-serif;
 }
+html {
+	touch-action: manipulation;
+}
 #wrap {
 	max-width: 40em;
 	margin: 2em auto;
@@ -26,6 +29,32 @@ a {
 	border-radius: .2em;
 	padding: .2em .8em;
 }
+#repl {
+	border: none;
+	background: none;
+	color: inherit;
+	padding: 0;
+}
+table {
+	border-collapse: collapse;
+}
+.vols td,
+.vols th {
+	padding: .3em .6em;
+	text-align: left;
+}
+.num {
+	border-right: 1px solid #bbb;
+}
+.num td {
+	padding: .1em .7em .1em 0;
+}
+.num td:first-child {
+	text-align: right;
+}
+.btns {
+	margin: 1em 0;
+}
 
 
 html.dark,
@@ -50,4 +79,7 @@ html.dark input {
 	border-radius: .5em;
 	padding: .5em .7em;
 	margin: 0 .5em 0 0;
-}
+}
+html.dark .num {
+	border-color: #777;
+}

copyparty/web/splash.html

@@ -6,26 +6,57 @@
     <title>copyparty</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/splash.css">
+    <link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
+    <link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
 </head>
 
 <body>
     <div id="wrap">
         <p>hello {{ this.uname }}</p>
 
+        {%- if avol %}
+        <h1>admin panel:</h1>
+        <table><tr><td> <!-- hehehe -->
+            <table class="num">
+                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
+                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
+                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
+                <tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+            </table>
+        </td><td>
+            <table class="vols">
+                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+                <tbody>
+                    {% for mp in avol %}
+                    {%- if mp in vstate and vstate[mp] %}
+                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                    {%- endif %}
+                    {% endfor %}
+                </tbody>
+            </table>
+        </td></tr></table>
+        <div class="btns">
+            <a href="/?stack">dump stack</a>
+        </div>
+        {%- endif %}
+
+        {%- if rvol %}
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
+        {%- if wvol %}
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
         <h1>login for more:</h1>
         <ul>
@@ -36,11 +67,13 @@
             </form>
         </ul>
     </div>
+	<a href="#" id="repl">π</a>
     <script>
 
-if (window.localStorage && localStorage.getItem('darkmode') == 1)
+if (localStorage.getItem('lightmode') != 1)
     document.documentElement.setAttribute("class", "dark");
 
 </script>
+<script src="/.cpr/util.js?_={{ ts }}"></script>
 </body>
-</html>
+</html>

copyparty/web/ui.css

@@ -0,0 +1,442 @@
+@font-face {
+	font-family: 'scp';
+	src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(/.cpr/deps/scp.woff2) format('woff2');
+}
+html {
+	touch-action: manipulation;
+}
+#tt, #toast {
+	position: fixed;
+	max-width: 34em;
+	max-width: min(34em, 90%);
+	max-width: min(34em, calc(100% - 7em));
+	background: #222;
+	border: 0 solid #777;
+	box-shadow: 0 .2em .5em #222;
+	border-radius: .4em;
+	z-index: 9001;
+}
+#tt {
+	max-width: min(34em, calc(100% - 3.3em));
+	overflow: hidden;
+	margin: .7em 0;
+	padding: 0 1.3em;
+	height: 0;
+	opacity: .1;
+	transition: opacity 0.14s, height 0.14s, padding 0.14s;
+}
+#toast {
+	bottom: 5em;
+	right: -1em;
+	line-height: 1.5em;
+	padding: 1em 1.3em;
+	margin-left: 3em;
+	border-width: .4em 0;
+	overflow-wrap: break-word;
+	transform: translateX(100%);
+	transition:
+		transform .4s cubic-bezier(.2, 1.2, .5, 1),
+		right .4s cubic-bezier(.2, 1.2, .5, 1);
+	text-shadow: 1px 1px 0 #000;
+	color: #fff;
+}
+#toast a {
+	color: inherit;
+	text-shadow: inherit;
+	background: rgba(0, 0, 0, 0.4);
+	border-radius: .3em;
+	padding: .2em .3em;
+}
+#toast a#toastc {
+	display: inline-block;
+	position: absolute;
+	overflow: hidden;
+	left: 0;
+	width: 0;
+	opacity: 0;
+	padding: .3em 0;
+	margin: -.3em 0 0 0;
+	line-height: 1.3em;
+	color: #000;
+	border: none;
+	outline: none;
+	text-shadow: none;
+	border-radius: .5em 0 0 .5em;
+	transition: left .3s, width .3s, padding .3s, opacity .3s;
+}
+#toastb {
+	max-height: 70vh;
+	overflow-y: auto;
+}
+#toast.scroll #toastb {
+	overflow-y: scroll;
+	margin-right: -1.2em;
+	padding-right: .7em;
+}
+#toast pre {
+	margin: 0;
+}
+#toast.vis {
+	right: 1.3em;
+	transform: unset;
+}
+#toast.vis #toastc {
+	left: -2em;
+	width: .4em;
+	padding: .3em .8em;
+	opacity: 1;
+}
+#toast.inf {
+	background: #07a;
+	border-color: #0be;
+}
+#toast.inf #toastc {
+	background: #0be;
+}
+#toast.ok {
+	background: #380;
+	border-color: #8e4;
+}
+#toast.ok #toastc {
+	background: #8e4;
+}
+#toast.warn {
+	background: #960;
+	border-color: #fc0;
+}
+#toast.warn #toastc {
+	background: #fc0;
+}
+#toast.err {
+	background: #900;
+	border-color: #d06;
+}
+#toast.err #toastc {
+	background: #d06;
+}
+#tt.b {
+	padding: 0 2em;
+	border-radius: .5em;
+	box-shadow: 0 .2em 1em #000;
+}
+#tt.show {
+	padding: 1em 1.3em;
+	border-width: .4em 0;
+	height: auto;
+	opacity: 1;
+}
+#tt.show.b {
+	padding: 1.5em 2em;
+	border-width: .5em 0;
+}
+#modalc code,
+#tt code {
+	background: #3c3c3c;
+	padding: .1em .3em;
+	border-top: 1px solid #777;
+	border-radius: .3em;
+	line-height: 1.7em;
+}
+#tt em {
+	color: #f6a;
+}
+html.light #tt {
+	background: #fff;
+	border-color: #888 #000 #777 #000;
+}
+html.light #tt,
+html.light #toast {
+	box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
+}
+#modalc code,
+html.light #tt code {
+	background: #060;
+	color: #fff;
+}
+html.light #tt em {
+	color: #d38;
+}
+#modal {
+	position: fixed;
+    overflow: auto;
+	top: 0;
+	left: 0;
+	right: 0;
+	bottom: 0;
+	width: 100%;
+	height: 100%;
+	z-index: 9001;
+	background: rgba(64,64,64,0.6);
+}
+#modal>table {
+    width: 100%;
+    height: 100%;
+}
+#modal td {
+	text-align: center;
+}
+#modalc {
+	position: relative;
+	display: inline-block;
+	background: #f7f7f7;
+	color: #333;
+	text-shadow: none;
+	text-align: left;
+	margin: 3em;
+	padding: 1em 1.1em;
+	border-radius: .6em;
+    box-shadow: 0 .3em 3em rgba(0,0,0,0.5);
+	max-width: 50em;
+	max-height: 30em;
+	overflow: auto;
+}
+@media (min-width: 40em) {
+    #modalc {
+        min-width: 30em;
+    }
+}
+#modalc li {
+	margin: 1em 0;
+}
+#modalc h6 {
+	font-size: 1.3em;
+	border-bottom: 1px solid #999;
+	margin: 0;
+	padding: .3em;
+	text-align: center;
+}
+#modalb {
+	position: sticky;
+	text-align: right;
+	padding-top: 1em;
+	bottom: 0;
+	right: 0;
+}
+#modalb a {
+	color: #000;
+	background: #ccc;
+	display: inline-block;
+	border-radius: .3em;
+	padding: .5em 1em;
+	outline: none;
+	border: none;
+}
+#modalb a:focus,
+#modalb a:hover {
+	background: #06d;
+	color: #fff;
+}
+#modalb a+a {
+	margin-left: .5em;
+}
+#modali {
+	display: block;
+	background: #fff;
+	color: #000;
+	width: calc(100% - 1.25em);
+    margin: 1em -.1em 0 -.1em;
+	padding: .5em;
+	outline: none;
+	border: .25em solid #ccc;
+	border-radius: .4em;
+}
+#modali:focus {
+	border-color: #06d;
+}
+#repl_pre {
+	max-width: 24em;
+}
+
+
+
+
+
+
+
+
+
+
+
+.mdo pre,
+.mdo code,
+.mdo a {
+	color: #480;
+	background: #f7f7f7;
+	border: .07em solid #ddd;
+	border-radius: .2em;
+	padding: .1em .3em;
+	margin: 0 .1em;
+}
+.mdo pre,
+.mdo code,
+.mdo tt {
+	font-family: 'scp', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-all;
+}
+.mdo code {
+	font-size: .96em;
+}
+.mdo h1,
+.mdo h2 {
+	line-height: 1.5em;
+}
+.mdo h1 {
+	font-size: 1.7em;
+	text-align: center;
+	border: 1em solid #777;
+	border-width: .05em 0;
+	margin: 3em 0;
+}
+.mdo h2 {
+	font-size: 1.5em;
+	font-weight: normal;
+	background: #f7f7f7;
+	border-top: .07em solid #fff;
+	border-bottom: .07em solid #bbb;
+	border-radius: .5em .5em 0 0;
+	padding-left: .4em;
+	margin-top: 3em;
+}
+.mdo h3 {
+	border-bottom: .1em solid #999;
+}
+.mdo h1 a, .mdo h3 a, .mdo h5 a,
+.mdo h2 a, .mdo h4 a, .mdo h6 a {
+	color: inherit;
+	display: block;
+	background: none;
+	border: none;
+	padding: 0;
+	margin: 0;
+}
+.mdo ul,
+.mdo ol {
+	border-left: .3em solid #ddd;
+}
+.mdo ul>li,
+.mdo ol>li {
+	margin: .7em 0;
+	list-style-type: disc;
+}
+.mdo strong {
+	color: #000;
+}
+.mdo p>em,
+.mdo li>em,
+.mdo td>em {
+	color: #c50;
+	padding: .1em;
+	border-bottom: .1em solid #bbb;
+}
+.mdo blockquote {
+	font-family: serif;
+	background: #f7f7f7;
+	border: .07em dashed #ccc;
+	padding: 0 2em;
+	margin: 1em 0;
+}
+.mdo small {
+	opacity: .8;
+}
+.mdo pre code {
+	display: block;
+	margin: 0 -.3em;
+	padding: .4em .5em;
+	line-height: 1.1em;
+}
+.mdo pre code:hover {
+	background: #fec;
+	color: #360;
+}
+.mdo table {
+	border-collapse: collapse;
+	margin: 1em 0;
+}
+.mdo th,
+.mdo td {
+	padding: .2em .5em;
+	border: .12em solid #aaa;
+}
+
+@media screen {
+	.mdo {
+		word-break: break-word;
+		overflow-wrap: break-word;
+		word-wrap: break-word; /*ie*/
+	}
+	html.light .mdo a,
+	.mdo a {
+		color: #fff;
+		background: #39b;
+		text-decoration: none;
+		padding: 0 .3em;
+		border: none;
+		border-bottom: .07em solid #079;
+	}
+	.mdo h1 {
+		color: #fff;
+		background: #444;
+		font-weight: normal;
+		border-top: .4em solid #fb0;
+		border-bottom: .4em solid #777;
+		border-radius: 0 1em 0 1em;
+		margin: 3em 0 1em 0;
+		padding: .5em 0;
+	}
+	.mdo h2 {
+		color: #fff;
+		background: #555;
+		margin-top: 2em;
+		border-bottom: .22em solid #999;
+		border-top: none;
+	}
+
+
+
+	html.dark .mdo a {
+		background: #057;
+	}
+	html.dark .mdo h1 a, html.dark .mdo h4 a,
+	html.dark .mdo h2 a, html.dark .mdo h5 a,
+	html.dark .mdo h3 a, html.dark .mdo h6 a {
+		color: inherit;
+		background: none;
+	}
+	html.dark .mdo pre,
+	html.dark .mdo code {
+		color: #8c0;
+		background: #1a1a1a;
+		border: .07em solid #333;
+	}
+	html.dark .mdo ul,
+	html.dark .mdo ol {
+		border-color: #444;
+	}
+	html.dark .mdo strong {
+		color: #fff;
+	}
+	html.dark .mdo p>em,
+	html.dark .mdo li>em,
+	html.dark .mdo td>em {
+		color: #f94;
+		border-color: #666;
+	}
+	html.dark .mdo h1 {
+		background: #383838;
+		border-top: .4em solid #b80;
+		border-bottom: .4em solid #4c4c4c;
+	}
+	html.dark .mdo h2 {
+		background: #444;
+		border-bottom: .22em solid #555;
+	}
+	html.dark .mdo td,
+	html.dark .mdo th {
+		border-color: #444;
+	}
+	html.dark .mdo blockquote {
+		background: #282828;
+		border: .07em dashed #444;
+	}
+}

copyparty/web/upload.css

@@ -1,223 +0,0 @@
-
-#op_up2k {
-	padding: 0 1em 1em 1em;
-}
-#u2form {
-	position: absolute;
-	top: 0;
-	left: 0;
-	width: 2px;
-	height: 2px;
-	overflow: hidden;
-}
-#u2form input {
-	background: #444;
-	border: 0px solid #444;
-	outline: none;
-}
-#u2err.err {
-	color: #f87;
-	padding: .5em;
-}
-#u2btn {
-	color: #eee;
-	background: #555;
-	background: -moz-linear-gradient(top, #367 0%, #489 50%, #38788a 51%, #367 100%);
-	background: -webkit-linear-gradient(top, #367 0%, #489 50%, #38788a 51%, #367 100%);
-	background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%);
-	filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0);
-	text-decoration: none;
-	line-height: 1.3em;
-	border: 1px solid #222;
-	border-radius: .4em;
-	text-align: center;
-	font-size: 1.5em;
-	margin: .5em auto;
-	padding: .8em 0;
-	width: 16em;
-	cursor: pointer;
-	box-shadow: .4em .4em 0 #111;
-}
-#op_up2k.srch #u2btn {
-	background: linear-gradient(to bottom, #ca3 0%, #fd8 50%, #fc6 51%, #b92 100%);
-	text-shadow: 1px 1px 1px #fc6;
-	color: #333;
-}
-#u2conf #u2btn {
-	margin: -1.5em 0;
-	padding: .8em 0;
-	width: 100%;
-}
-#u2notbtn {
-	display: none;
-	text-align: center;
-	background: #333;
-	padding-top: 1em;
-}
-#u2notbtn * {
-	line-height: 1.3em;
-}
-#u2tab {
-	margin: 3em auto;
-	width: calc(100% - 2em);
-	max-width: 100em;
-}
-#u2form.srch #u2tab {
-	max-width: none;
-}
-#u2tab td {
-	border: 1px solid #ccc;
-	border-width: 0 0px 1px 0;
-	padding: .1em .3em;
-}
-#u2tab td:nth-child(2) {
-	width: 5em;
-}
-#u2tab td:nth-child(3) {
-	width: 40%;
-}
-#u2form.srch #u2tab td:nth-child(3) {
-	font-family: sans-serif;
-	width: auto;
-}
-#u2tab tr+tr:hover td {
-	background: #222;
-}
-#u2conf {
-	margin: 1em auto;
-	width: 30em;
-}
-#u2conf.has_btn {
-	width: 46em;
-}
-#u2conf * {
-	text-align: center;
-	line-height: 1em;
-	margin: 0;
-	padding: 0;
-	border: none;
-	outline: none;
-}
-#u2conf .txtbox {
-	width: 4em;
-	color: #fff;
-	background: #444;
-	border: 1px solid #777;
-	font-size: 1.2em;
-	padding: .15em 0;
-}
-#u2conf a {
-	color: #fff;
-	background: #c38;
-	text-decoration: none;
-	border-radius: .1em;
-	font-size: 1.5em;
-	padding: .1em 0;
-	margin: 0 -.25em;
-	width: 1.5em;
-	height: 1em;
-	display: inline-block;
-	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
-}
-#u2conf input+a {
-	background: #d80;
-}
-#u2conf label {
-	font-size: 1.6em;
-	width: 2em;
-	height: 1em;
-	padding: .4em 0;
-	display: block;
-	user-select: none;
-	border-radius: .25em;
-}
-#u2conf input[type="checkbox"] {
-	position: relative;
-	opacity: .02;
-	top: 2em;
-}
-#u2conf input[type="checkbox"]+label {
-	position: relative;
-	background: #603;
-	border-bottom: .2em solid #a16;
-	box-shadow: 0 .1em .3em #a00 inset;
-}
-#u2conf input[type="checkbox"]:checked+label {
-	background: #6a1;
-	border-bottom: .2em solid #efa;
-	box-shadow: 0 .1em .5em #0c0;
-}
-#u2conf input[type="checkbox"]+label:hover {
-	box-shadow: 0 .1em .3em #fb0;
-	border-color: #fb0;
-}
-#op_up2k.srch #u2conf td:nth-child(1)>*,
-#op_up2k.srch #u2conf td:nth-child(2)>*,
-#op_up2k.srch #u2conf td:nth-child(3)>* {
-	background: #777;
-	border-color: #ccc;
-	box-shadow: none;
-	opacity: .2;
-}
-#u2cdesc {
-	position: absolute;
-	width: 34em;
-	left: calc(50% - 15em);
-	background: #222;
-	border: 0 solid #555;
-	text-align: center;
-	overflow: hidden;
-	margin: 0 -2em;
-	height: 0;
-	padding: 0 1em;
-	opacity: .1;
-    transition: all 0.14s ease-in-out;
-	border-radius: .4em;
-	box-shadow: 0 .2em .5em #222;
-}
-#u2cdesc.show {
-	padding: 1em;
-	height: auto;
-	border-width: .2em 0;
-	opacity: 1;
-}
-#u2foot {
-	color: #fff;
-	font-style: italic;
-}
-#u2footfoot {
-	margin-bottom: -1em;
-}
-.prog {
-	font-family: monospace;
-}
-.prog>div {
-	display: inline-block;
-	position: relative;
-	overflow: hidden;
-	margin: 0;
-	padding: 0;
-	height: 1.1em;
-	margin-bottom: -.15em;
-	box-shadow: -1px -1px 0 inset rgba(255,255,255,0.1);
-}
-.prog>div>div {
-	width: 0%;
-	position: absolute;
-	left: 0;
-	top: 0;
-	bottom: 0;
-	background: #0a0;
-}
-#u2tab a>span {
-	font-weight: bold;
-	font-style: italic;
-	color: #fff;
-	padding-left: .2em;
-}
-#u2cleanup {
-	float: right;
-	margin-bottom: -.3em;
-}

copyparty/web/upload.html

@@ -1,91 +0,0 @@
-
-    <div id="op_bup" class="opview opbox act">
-        <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
-            <input type="hidden" name="act" value="bput" />
-            <input type="file" name="f" multiple><br />
-            <input type="submit" value="start upload">
-        </form>
-    </div>
-
-    <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
-            <input type="hidden" name="act" value="mkdir" />
-            <input type="text" name="name" size="30">
-            <input type="submit" value="mkdir">
-        </form>
-    </div>
-
-    <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
-            <input type="hidden" name="act" value="new_md" />
-            <input type="text" name="name" size="30">
-            <input type="submit" value="create doc">
-        </form>
-    </div>
-
-    <div id="op_msg" class="opview opbox">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
-            <input type="text" name="msg" size="30">
-            <input type="submit" value="send msg">
-        </form>
-    </div>
-
-    <div id="op_up2k" class="opview">
-        <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
-
-            <table id="u2conf">
-                <tr>
-                    <td>parallel uploads</td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="multitask" />
-                        <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="ask_up" />
-                        <label for="ask_up" alt="ask for confirmation befofre upload starts">💭</label>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="flag_en" />
-                        <label for="flag_en" alt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
-                    </td>
-                {%- if have_up2k_idx %}
-                    <td data-perm="read" rowspan="2">
-                        <input type="checkbox" id="fsearch" />
-                        <label for="fsearch" alt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
-                    </td>
-                {%- endif %}
-                    <td data-perm="read" rowspan="2" id="u2btn_cw"></td>
-                </tr>
-                <tr>
-                    <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
-                    </td>
-                </tr>
-            </table>
-
-            <div id="u2cdesc"></div>
-
-            <div id="u2notbtn"></div>
-
-            <div id="u2btn_ct">
-                <div id="u2btn">
-                    <span id="u2bm"></span><br />
-                    drop files here<br />
-                    (or click me)
-                </div>
-            </div>
-
-            <table id="u2tab">
-                <tr>
-                    <td>filename</td>
-                    <td>status</td>
-                    <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
-                </tr>
-            </table>
-
-            <p id="u2foot"></p>
-            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
-    </div>

docs/README.md

@@ -0,0 +1,22 @@
+# example `.epilogue.html`
+save one of these as `.epilogue.html` inside a folder to customize it:
+
+* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+
+
+# example browser-css
+point `--css-browser` to one of these by URL:
+
+* [`browser.css`](browser.css) changes the background
+* [`browser-icons.css`](browser-icons.css) adds filetype icons
+
+
+
+# other stuff
+
+## [`rclone.md`](rclone.md)
+* notes on using rclone as a fuse client/server
+
+## [`example.conf`](example.conf)
+* example config file for `-c` which never really happened

docs/biquad.html

@@ -0,0 +1,95 @@
+<!DOCTYPE html><html><head></head><body><script>
+
+setTimeout(location.reload.bind(location), 700);
+document.documentElement.scrollLeft = 0;
+
+var can = document.createElement('canvas'),
+    cc = can.getContext('2d'),
+    w = 2048,
+    h = 1024;
+
+w = 2048;
+
+can.width = w;
+can.height = h;
+document.body.appendChild(can);
+can.style.cssText = 'width:' + w + 'px;height:' + h + 'px';
+
+cc.fillStyle = '#000';
+cc.fillRect(0, 0, w, h);
+
+var cfg = [ // hz, q, g
+    [31.25 * 0.88, 0, 1.4],  // shelf
+    [31.25 * 1.04, 0.7, 0.96],  // peak
+    [62.5, 0.7, 1],
+    [125, 0.8, 1],
+    [250, 0.9, 1.03],
+    [500, 0.9, 1.1],
+    [1000, 0.9, 1.1],
+    [2000, 0.9, 1.105],
+    [4000, 0.88, 1.05],
+    [8000 * 1.006, 0.73, 1.24],
+    //[16000 * 1.00, 0.5, 1.75],  // peak.v1
+    //[16000 * 1.19, 0, 1.8]  // shelf.v1
+    [16000 * 0.89, 0.7, 1.26],  // peak
+    [16000 * 1.13, 0.82, 1.09],  // peak
+    [16000 * 1.205, 0, 1.9]  // shelf
+];
+
+var freqs = new Float32Array(22000),
+    sum = new Float32Array(freqs.length),
+    ac = new AudioContext(),
+    step = w / freqs.length,
+    colors = [
+        'rgba(255, 0, 0, 0.7)',
+        'rgba(0, 224, 0, 0.7)',
+        'rgba(0, 64, 255, 0.7)'
+    ];
+
+var order = [];
+
+for (var a = 0; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var a = 1; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var ia = 0; ia < order.length; ia++) {
+    var a = order[ia],
+        fi = ac.createBiquadFilter(),
+        mag = new Float32Array(freqs.length),
+        phase = new Float32Array(freqs.length);
+
+    for (var b = 0; b < freqs.length; b++)
+        freqs[b] = b;
+
+    fi.type = a == 0 ? 'lowshelf' : a == cfg.length - 1 ? 'highshelf' : 'peaking';
+    fi.frequency.value = cfg[a][0];
+    fi.Q.value = cfg[a][1];
+    fi.gain.value = 1;
+
+    fi.getFrequencyResponse(freqs, mag, phase);
+    cc.fillStyle = colors[a % colors.length];
+    for (var b = 0; b < sum.length; b++) {
+        mag[b] -= 1;
+        sum[b] += mag[b] * cfg[a][2];
+        var y = h - (mag[b] * h * 3);
+        cc.fillRect(b * step, y, step, h - y);
+        cc.fillRect(b * step - 1, y - 1, 3, 3);
+    }
+}
+
+var min = 999999, max = 0;
+for (var a = 0; a < sum.length; a++) {
+    min = Math.min(min, sum[a]);
+    max = Math.max(max, sum[a]);
+}
+cc.fillStyle = 'rgba(255,255,255,1)';
+for (var a = 0; a < sum.length; a++) {
+    var v = (sum[a] - min) / (max - min);
+    cc.fillRect(a * step, 0, step, v * h / 2);
+}
+
+cc.fillRect(0, 460, w, 1);
+
+</script></body></html>

docs/browser-icons.css

@@ -0,0 +1,71 @@
+/* video, alternative 1:
+   top-left icon, just like the other formats
+=======================================================================
+
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '📺';
+}
+*/
+
+
+
+/* video, alternative 2:
+   play-icon in the middle of the thumbnail
+=======================================================================
+*/
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+) {
+	position: relative;
+	overflow: hidden;
+}
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '▶';
+	opacity: .8;
+	margin: 0;
+	padding: 1em .5em 1em .7em;
+	border-radius: 9em;
+	line-height: 0;
+	color: #fff;
+	text-shadow: none;
+	background: rgba(0, 0, 0, 0.7);
+	left: calc(50% - 1em);
+	top: calc(50% - 1.4em);
+}
+
+
+
+/* audio */
+#ggrid>a:is(
+[href$=".mp3"i],
+[href$=".ogg"i],
+[href$=".opus"i],
+[href$=".flac"i],
+[href$=".m4a"i],
+[href$=".aac"i],
+):before {
+    content: '🎵';
+}
+
+
+
+/* image */
+#ggrid>a:is(
+[href$=".jpg"i],
+[href$=".jpeg"i],
+[href$=".png"i],
+[href$=".gif"i],
+[href$=".webp"i],
+):before {
+    content: '🎨';
+}

docs/browser.css

@@ -0,0 +1,29 @@
+html {
+    background: #333 url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
+}
+#files th {
+    background: rgba(32, 32, 32, 0.9) !important;
+}
+#ops,
+#treeul,
+#files td {
+    background: rgba(32, 32, 32, 0.3) !important;
+}
+
+
+html.light {
+    background: #eee url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
+}
+html.light #files th {
+    background: rgba(255, 255, 255, 0.9) !important;
+}
+html.light #ops,
+html.light #treeul,
+html.light #files td {
+    background: rgba(248, 248, 248, 0.8) !important;
+}
+
+
+#files * {
+    background: transparent !important;
+}

docs/example.conf

@@ -10,19 +10,25 @@ u k:k
 # share "." (the current directory)
 # as "/" (the webroot) for the following users:
 # "r" grants read-access for anyone
-# "a ed" grants read-write to ed
+# "rw ed" grants read-write to ed
 .
 /
 r
-a ed
+rw ed
 
 # custom permissions for the "priv" folder:
-# user "k" can see/read the contents
-# and "ed" gets read-write access
+# user "k" can only see/read the contents
+# user "ed" gets read-write access
 ./priv
 /priv
 r k
-a ed
+rw ed
+
+# this does the same thing:
+./priv
+/priv
+r ed k
+w ed
 
 # share /home/ed/Music/ as /music and let anyone read it
 # (this will replace any folder called "music" in the webroot)
@@ -32,10 +38,14 @@ r
 
 # and a folder where anyone can upload
 # but nobody can see the contents
+# and set the e2d flag to enable the uploads database
+# and set the nodupe flag to reject duplicate uploads
 /home/ed/inc
 /dump
 w
+c e2d
+c nodupe
 
 # this entire config file can be replaced with these arguments:
-# -u ed:123 -u k:k -v .::r:aed -v priv:priv:rk:aed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w
+# -u ed:123 -u k:k -v .::r:a,ed -v priv:priv:r,k:rw,ed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w:c,e2d:c,nodupe
 # but note that the config file always wins in case of conflicts

docs/hls.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html><html lang="en"><head>
+	<meta charset="utf-8">
+	<title>hls-test</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+</head><body>
+
+<video id="vid" controls></video>
+<script src="hls.light.js"></script>
+<script>
+
+var video = document.getElementById('vid');
+var hls = new Hls({
+	debug: true,
+	autoStartLoad: false
+});
+hls.loadSource('live/v.m3u8');
+hls.attachMedia(video);
+hls.on(Hls.Events.MANIFEST_PARSED, function() {
+	hls.startLoad(0);
+});
+hls.on(Hls.Events.MEDIA_ATTACHED, function() {
+	video.muted = true;
+	video.play();
+});
+
+/*
+general good news:
+- doesn't need fixed-length segments; ok to let x264 pick optimal keyframes and slice on those
+- hls.js polls the m3u8 for new segments, scales the duration accordingly, seeking works great
+- the sfx will grow by 66 KiB since that's how small hls.js can get, wait thats not good
+
+# vod, creates m3u8 at the end, fixed keyframes, v bad
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -g 120 -keyint_min 120 -sc_threshold 0 -hls_time 4 -hls_playlist_type vod -hls_segment_filename v%05d.ts v.m3u8
+
+# live, updates m3u8 as it goes, dynamic keyframes, streamable with hls.js
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format mpegts -segment_list_flags live v%05d.ts
+
+# fmp4 (fragmented mp4), doesn't work with hls.js, gets duratoin 149:07:51 (536871s), probably the tkhd/mdhd 0xffffffff (timebase 8000? ok)
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format fmp4 -segment_list_flags live v%05d.mp4
+
+# try 2, works, uses tempfiles for m3u8 updates, good, 6% smaller
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f hls -hls_segment_type fmp4 -hls_list_size 0 -hls_segment_filename v%05d.mp4 v.m3u8
+
+more notes
+- adding -hls_flags single_file makes duration wack during playback (for both fmp4 and ts), ok once finalized and refreshed, gives no size reduction anyways
+- bebop op has good keyframe spacing for testing hls.js, in particular it hops one seg back and immediately resumes if it hits eof with the explicit hls.startLoad(0); otherwise it jumps into the middle of a seg and becomes art
+- can probably -c:v copy most of the time, is there a way to check for cgop? todo
+
+*/
+</script>
+</body></html>

docs/minimal-up2k.html

@@ -0,0 +1,35 @@
+<!--
+  save this as .epilogue.html inside a write-only folder to declutter the UI,  makes it look like
+  https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
+-->
+
+<style>
+
+    /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
+
+    #ops, #tree, #path, #wrap>h2:last-child,  /* main tabs and navigators (tree/breadcrumbs) */
+
+    #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
+
+    #u2cards, #u2etaw  /* and the upload progress tabs */
+
+    {display: none !important}  /* do it! */
+
+
+
+    /* add some margins because now it's weird */
+    .opview {margin-top: 2.5em}
+    #op_up2k {margin-top: 6em}
+
+    /* and embiggen the upload button */
+    #u2conf #u2btn, #u2btn {padding:1.5em 0}
+
+    /* adjust the button area a bit */
+    #u2conf.has_btn {width: 35em !important; margin: 5em auto}
+
+    /* a */
+    #op_up2k {min-height: 0}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

docs/music-analysis.sh

@@ -0,0 +1,242 @@
+#!/bin/bash
+echo please dont actually run this as a scriopt
+exit 1
+
+
+# dependency-heavy, not particularly good fit
+pacman -S llvm10
+python3 -m pip install --user librosa
+git clone https://github.com/librosa/librosa.git
+
+
+# correct bpm for tracks with bad tags
+br='
+/Trip Trip Trip\(Hardcore Edit\).mp3/ {v=176}
+/World!!.BIG_SOS/ {v=175}
+/\/08\..*\(BIG_SOS Bootleg\)\.mp3/ {v=175}
+/もってけ！セーラ服.Asterisk DnB/ {v=175}
+/Rondo\(Asterisk DnB Re.mp3/ {v=175}
+/Ray Nautica 175 Edit/ {v=175;x="thunk"}
+/TOKIMEKI Language.Jauz/ {v=174}
+/YUPPUN Hardcore Remix\).mp3/ {v=174;x="keeps drifting"}
+/(èâAâï.î╧ûδ|バーチャリアル.狐耶)J-Core Remix\).mp3/ {v=172;x="hard"}
+/lucky train..Freezer/ {v=170}
+/Alf zero Bootleg ReMix/ {v=170}
+/Prisoner of Love.Kacky/ {v=170}
+/火炎 .Qota/ {v=170}
+/\(hu-zin Bootleg\)\.mp3/ {v=170}
+/15. STRAIGHT BET\(Milynn Bootleg\)\.mp3/ {v=170}
+/\/13.*\(Milynn Bootleg\)\.mp3/ {v=167;x="way hard"}
+/COLOR PLANET .10SAI . nijikon Remix\)\.mp3/ {v=165}
+/11\. (朝はご飯派|Æ⌐é═é▓ö╤öh)\.mp3/ {v=162}
+/09\. Where.s the core/ {v=160}
+/PLANET\(Koushif Jersey Club Bootleg\)remaster.mp3/ {v=160;x="starts ez turns bs"}
+/kened Soul - Madeon x Angel Beats!.mp3/ {v=160}
+/Dear Moments\(Mother Harlot Bootleg\)\.mp3/ {v=150}
+/POWER.Ringos UKG/ {v=140}
+/ブルー・フィールド\(Ringos UKG Remix\).mp3/ {v=135}
+/プラチナジェット.Ringo Remix..mp3/ {v=131.2}
+/Mirrorball Love \(TKM Bootleg Mix\).mp3/ {v=130}
+/Photon Melodies \(TKM Bootleg Mix\).mp3/ {v=128}
+/Trap of Love \(TKM Bootleg Mix\).mp3/ {v=128}
+/One Step \(TKM Bootleg Mix\)\.mp3/ {v=126}
+/04 (トリカムイ岩|âgâèâJâÇâCèΓ).mp3/ {v=125}
+/Get your Wish \(NAWN REMIX\)\.mp3/ {v=95}
+/Flicker .Nitro Fun/ {v=92}
+/\/14\..*suicat Remix/ {v=85.5;x="tricky"}
+/Yanagi Nagi - Harumodoki \(EO Remix\)\.mp3/ {v=150}
+/Azure - Nicology\.mp3/ {v=128;x="off by 5 how"}
+'
+
+
+# afun host, collects/grades the results
+runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | awk '{v=$1;sub(/[^ ]+ /,"")} '"$br"' {printf "%s %s\n",v,$0}' | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk 'function prt(c) {printf "\033[3%sm%s\033[0m\n",c,$0} $8!="div,"{next} $5!~/^[0-9\.]+/{next} {meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;prt(6);next} idiff==1{n1++;prt(3);next} idiff>10{nx++;prt(1);next} {n10++;prt(5)} END {printf "ok: %d   1off: %2s   (%3s)   10off: %2s   (%3s)   fail: %2s\n",n0,n1,n0+n1,n10,n0+n1+n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; }
+
+
+# ok:   8   1off: 62   ( 70)   10off: 86   (156)   fail: 25   # 105 sec,  librosa @ 8c archvm on 3700x w10
+# ok:   4   1off: 59   ( 63)   10off: 65   (128)   fail: 53   # using original tags (bad)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } runfun
+
+
+# ok: 119   1off:  5   (124)   10off:  8   (132)   fail: 49   # 51 sec,  vamp-example-fixedtempo
+# ok: 109   1off:  4   (113)   10off:  9   (122)   fail: 59   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; }; runfun
+
+
+# ok: 102   1off: 61   (163)   10off: 12   (175)   fail:  6   # 61 sec,  vamp-qm-tempotracker
+# ok:  80   1off: 48   (128)   10off: 11   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; }; runfun
+
+
+# ok: 133   1off: 32   (165)   10off: 12   (177)   fail:  3   # 51 sec,  vamp-beatroot
+# ok: 101   1off: 22   (123)   10off: 16   (139)   fail: 39   # bad-tags
+# note: some tracks fully fail to analyze (unlike the others which always provide a guess)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; }; runfun
+
+
+# ok: 124   1off:  9   (133)   10off: 40   (173)   fail:  8   # 231 sec,  essentia/full
+# ok: 109   1off:  8   (117)   10off: 22   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'import essentia; import essentia.standard as es; fe, fef = es.MusicExtractor(lowlevelStats=["mean", "stdev"], rhythmStats=["mean", "stdev"], tonalStats=["mean", "stdev"])("/dev/shm/'$core'.wav"); print("{:.2f}".format(fe["rhythm.bpm"]))')"; }; runfun
+
+
+# ok: 113   1off: 18   (131)   10off: 46   (177)   fail:  4   # 134 sec,  essentia/re2013
+# ok: 101   1off: 15   (116)   10off: 26   (142)   fail: 39   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'from essentia.standard import *; a=MonoLoader(filename="/dev/shm/'$core'.wav")(); bpm,beats,confidence,_,intervals=RhythmExtractor2013(method="multifeature")(a); print("{:.2f}".format(bpm))')"; }; runfun
+
+
+
+########################################################################
+##
+##  key detectyion
+##
+########################################################################
+
+
+
+# console scriptlet reusing keytabs from browser.js
+var m=''; for (var a=0; a<24; a++) m += 's/\\|(' + maps["traktor_sharps"][a].trim() + "|" + maps["rekobo_classic"][a].trim() + "|" + maps["traktor_musical"][a].trim() + "|" + maps["traktor_open"][a].trim() + ')$/|' + maps["rekobo_alnum"][a].trim() + '/;'; console.log(m);
+
+
+# translate to camelot
+re='s/\|(B|B|B|6d)$/|1B/;s/\|(F#|F#|Gb|7d)$/|2B/;s/\|(C#|Db|Db|8d)$/|3B/;s/\|(G#|Ab|Ab|9d)$/|4B/;s/\|(D#|Eb|Eb|10d)$/|5B/;s/\|(A#|Bb|Bb|11d)$/|6B/;s/\|(F|F|F|12d)$/|7B/;s/\|(C|C|C|1d)$/|8B/;s/\|(G|G|G|2d)$/|9B/;s/\|(D|D|D|3d)$/|10B/;s/\|(A|A|A|4d)$/|11B/;s/\|(E|E|E|5d)$/|12B/;s/\|(G#m|Abm|Abm|6m)$/|1A/;s/\|(D#m|Ebm|Ebm|7m)$/|2A/;s/\|(A#m|Bbm|Bbm|8m)$/|3A/;s/\|(Fm|Fm|Fm|9m)$/|4A/;s/\|(Cm|Cm|Cm|10m)$/|5A/;s/\|(Gm|Gm|Gm|11m)$/|6A/;s/\|(Dm|Dm|Dm|12m)$/|7A/;s/\|(Am|Am|Am|1m)$/|8A/;s/\|(Em|Em|Em|2m)$/|9A/;s/\|(Bm|Bm|Bm|3m)$/|10A/;s/\|(F#m|F#m|Gbm|4m)$/|11A/;s/\|(C#m|Dbm|Dbm|5m)$/|12A/;'
+
+
+# runner/wrapper
+runfun() { cores=8; touch run; tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, key.v from mt key join mt dur on key.w = dur.w where key.k = "key" and dur.k = ".dur" order by dur.w' | uniq -w16 | grep -vE '(Off-Key|None)$' | sed -r "s/ //g;$re" | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done| grep mir/cr | tr \| / | while read key fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); [ "$key" = "$py" ] && c=2 || c=5; printf '%4s sec, %4s orig, \033[3%dm%4s py,\033[0m %s\n' $td "$key" $c "$py" "$fn") || break; done & done; time wait 2>/dev/null; }
+
+
+# ok: 26   1off: 10   2off: 1   fail: 3   #  15 sec, keyfinder
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 -c 'import sys; import keyfinder; print(keyfinder.key(sys.argv[1]).camelot())' "/dev/shm/$core.wav")"; }; runfun
+
+
+# https://github.com/MTG/essentia/raw/master/src/examples/tutorial/example_key_by_steps_streaming.py
+# https://essentia.upf.edu/reference/std_Key.html  # edma edmm braw bgate
+sed -ri 's/^(key = Key\().*/\1profileType="bgate")/' example_key_by_steps_streaming.py
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 example_key_by_steps_streaming.py /dev/shm/$core.{wav,yml} 2>/dev/null | sed -r "s/ major//;s/ minor/m/;s/^/|/;$re;s/.//")"; }; runfun
+
+
+
+########################################################################
+##
+##  misc
+##
+########################################################################
+
+
+
+python3 -m pip install --user vamp
+
+import librosa
+d, r = librosa.load('/dev/shm/0.wav')
+d.dtype
+# dtype('float32')
+d.shape
+# (1323000,)
+d
+# array([-1.9614939e-08,  1.8037968e-08, -1.4106059e-08, ...,
+#         1.2024145e-01,  2.7462116e-01,  1.6202132e-01], dtype=float32)
+
+
+
+import vamp
+c = vamp.collect(d, r, "vamp-example-plugins:fixedtempo")
+c
+# {'list': [{'timestamp':  0.005804988, 'duration':  9.999092971, 'label': '110.0 bpm', 'values': array([109.98116], dtype=float32)}]}
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+d
+array([-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+       -0.0267825 , -0.03564296], dtype=float32)
+
+d = np.reshape(d, [1, -1])
+d
+array([[-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+        -0.0267825 , -0.03564296]], dtype=float32)
+
+
+
+import vampyhost
+print("\n".join(vampyhost.list_plugins()))
+
+mvamp:marsyas_bextract_centroid
+mvamp:marsyas_bextract_lpcc
+mvamp:marsyas_bextract_lsp
+mvamp:marsyas_bextract_mfcc
+mvamp:marsyas_bextract_rolloff
+mvamp:marsyas_bextract_scf
+mvamp:marsyas_bextract_sfm
+mvamp:marsyas_bextract_zero_crossings
+mvamp:marsyas_ibt
+mvamp:zerocrossing
+qm-vamp-plugins:qm-adaptivespectrogram
+qm-vamp-plugins:qm-barbeattracker
+qm-vamp-plugins:qm-chromagram
+qm-vamp-plugins:qm-constantq
+qm-vamp-plugins:qm-dwt
+qm-vamp-plugins:qm-keydetector
+qm-vamp-plugins:qm-mfcc
+qm-vamp-plugins:qm-onsetdetector
+qm-vamp-plugins:qm-segmenter
+qm-vamp-plugins:qm-similarity
+qm-vamp-plugins:qm-tempotracker
+qm-vamp-plugins:qm-tonalchange
+qm-vamp-plugins:qm-transcription
+vamp-aubio:aubiomelenergy
+vamp-aubio:aubiomfcc
+vamp-aubio:aubionotes
+vamp-aubio:aubioonset
+vamp-aubio:aubiopitch
+vamp-aubio:aubiosilence
+vamp-aubio:aubiospecdesc
+vamp-aubio:aubiotempo
+vamp-example-plugins:amplitudefollower
+vamp-example-plugins:fixedtempo
+vamp-example-plugins:percussiononsets
+vamp-example-plugins:powerspectrum
+vamp-example-plugins:spectralcentroid
+vamp-example-plugins:zerocrossing
+vamp-rubberband:rubberband
+
+
+
+plug = vampyhost.load_plugin("vamp-example-plugins:fixedtempo", 22050, 0)
+plug.info
+{'apiVersion': 2, 'pluginVersion': 1, 'identifier': 'fixedtempo', 'name': 'Simple Fixed Tempo Estimator', 'description': 'Study a short section of audio and estimate its tempo, assuming the tempo is constant', 'maker': 'Vamp SDK Example Plugins', 'copyright': 'Code copyright 2008 Queen Mary, University of London.  Freely redistributable (BSD license)'}
+plug = vampyhost.load_plugin("qm-vamp-plugins:qm-tempotracker", 22050, 0)
+from pprint import pprint; pprint(plug.parameters)
+
+
+
+for c in plug.parameters: print("{} \033[36m{}  [\033[33m{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], "\033[36m, \033[33m".join(c["valueNames"]), c["valueNames"][int(c["defaultValue"])])) if "valueNames" in c else print("{} \033[36m{}  [\033[33m{}..{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], c["minValue"], c["maxValue"], c["defaultValue"]))
+
+
+
+beatroot-vamp:beatroot
+cl=c["list"]; 60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"])))
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+# 128 bpm, key 5A Cm
+
+import vamp
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40})
+c["list"][0]["label"]
+# 127.6 bpm
+
+c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150})
+print("\n".join([v["label"] for v in c["list"] if v["label"]]))
+v = [float(x["label"].split(' ')[0]) for x in c["list"] if x["label"]]
+v = list(sorted(v))[len(v)//4:-len(v)//4]
+v = sum(v) / len(v)
+# 128.1 bpm
+

docs/notes.sh

@@ -11,6 +11,13 @@ gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f --
 gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done
 
 
+##
+## detect partial uploads based on file contents
+##  (in case of context loss or old copyparties)
+
+echo; find -type f | while IFS= read -r x; do printf '\033[A\033[36m%s\033[K\033[0m\n' "$x"; tail -c$((1024*1024)) <"$x" | xxd -a | awk 'NR==1&&/^[0: ]+.{16}$/{next} NR==2&&/^\*$/{next} NR==3&&/^[0f]+: [0 ]+65 +.{16}$/{next} {e=1} END {exit e}' || continue; printf '\033[A\033[31msus:\033[33m %s \033[0m\n\n' "$x"; done
+
+
 ##
 ## create a test payload
 
@@ -37,7 +44,7 @@ avg() { awk 'function pr(ncsz) {if (nsmp>0) {printf "%3s %s\n", csz, sum/nsmp} c
 dirs=("$HOME/vfs/ほげ" "$HOME/vfs/ほげ/ぴよ" "$HOME/vfs/$(printf \\xed\\x91)" "$HOME/vfs/$(printf \\xed\\x91/\\xed\\x92)")
 mkdir -p "${dirs[@]}"
 for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qwe,rty;asd fgh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh'; do echo "$dir" > "$dir/$fn.html"; done; done
-
+# qw er+ty%20ui%%20op<as>df&gh&amp;jk#zx'cv"bn`m=qw*er^ty?ui@op,as.df-gh_jk
 
 ##
 ## upload mojibake
@@ -60,6 +67,73 @@ wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:392
 shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*1024*1024)); printf $(tail -c +$((v+1)) "$f" | head -c $((w-v)) | sha512sum | cut -c-64 | sed -r 's/ .*//;s/(..)/\\x\1/g') | base64 -w0 | cut -c-43 | tr '+/' '-_'; v=$w; [ $v -lt $sz ] || break; done; }
 
 
+##
+## poll url for performance issues
+
+command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
+
+
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+# rename all selected songs to <leading-track-number> + <Title> + <extension>
+var sel=msel.getsel(), ci=find_file_col('Title')[0], re=[]; for (var a=0; a<sel.length; a++) { var url=sel[a].vp, tag=ebi(sel[a].id).closest('tr').querySelectorAll('td')[ci].textContent, name=uricom_dec(vsplit(url)[1])[0], m=/^([0-9]+[\. -]+)?.*(\.[^\.]+$)/.exec(name), name2=(m[1]||'')+tag+m[2], url2=vsplit(url)[0]+uricom_enc(name2,false); if (url!=url2) re.push([url, url2]); }
+console.log(JSON.stringify(re, null, '  '));
+function f() { if (!re.length) return treectl.goto(get_evpath()); var [u1,u2] = re.shift(); fetch(u1+'?move='+u2).then((rsp) => {if (rsp.ok) f(); }); }; f();
+
+##
+## bash oneliners
+
+# get the size and video-id of all youtube vids in folder, assuming filename ends with -id.ext, and create a copyparty search query
+find -maxdepth 1 -printf '%s %p\n' | sort -n | awk '!/-([0-9a-zA-Z_-]{11})\.(mkv|mp4|webm)$/{next} {sub(/\.[^\.]+$/,"");n=length($0);v=substr($0,n-10);print $1, v}' | tee /dev/stderr | awk 'BEGIN {p="("} {printf("%s name like *-%s.* ",p,$2);p="or"} END {print ")\n"}' | cat >&2
+
+# unique stacks in a stackdump
+f=a; rm -rf stacks; mkdir stacks; grep -E '^#' $f | while IFS= read -r n; do awk -v n="$n" '!$0{o=0} o; $0==n{o=1}' <$f >stacks/f; h=$(sha1sum <stacks/f | cut -c-16); mv stacks/f stacks/$h-"$n"; done ; find stacks/ | sort | uniq -cw24
+
+
+##
+## sqlite3 stuff
+
+# find dupe metadata keys
+sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = mt2.k and mt1.rowid != mt2.rowid'
+
+# partial reindex by deleting all tags for a list of files
+time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
+cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
+
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
+# unschedule mtp scan for all files somewhere under "enc/"
+sqlite3 -readonly up2k.db 'select substr(up.w,1,16) from up inner join mt on mt.w = substr(up.w,1,16) where rd like "enc/%" and +mt.k = "t:mtp"' > keys; awk '{printf "delete from mt where w = \"%s\" and +k = \"t:mtp\";\n", $0}' <keys | tee /dev/stderr | sqlite3 up2k.db
+
+# compare metadata key "key" between two databases
+sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select w, v from mt where k = "key" order by w' > k2; ok=0; ng=0; while IFS='|' read w k2; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$(sqlite3 -readonly up2k.db.key-full "select * from up where substr(w,1,16) = '$w'" | sed -r 's/\|/ | /g')"; }; done < <(cat k2); echo "match $ok   diff $ng"
+
+# actually this is much better
+sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select mt.w, mt.v, up.rd, up.fn from mt inner join up on mt.w = substr(up.w,1,16) where mt.k = "key" order by up.rd, up.fn' > k2; ok=0; ng=0; while IFS='|' read w k2 path; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$path"; }; done < <(cat k2); echo "match $ok   diff $ng"
+
+
+##
+## media
+
+# split track into test files
+e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
+
+-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
+sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
+
+# generate the sine meme
+for ((f=420;f<1200;f++)); do sz=$(ffmpeg -y -f lavfi -i sine=frequency=$f:duration=2 -vf volume=0.1 -ac 1 -ar 44100 -f s16le /dev/shm/a.wav 2>/dev/null; base64 -w0 </dev/shm/a.wav | gzip -c | wc -c); printf '%d %d\n' $f $sz; done | tee /dev/stderr | sort -nrk2,2
+ffmpeg -y -f lavfi -i sine=frequency=1050:duration=2 -vf volume=0.1 -ac 1 -ar 44100 /dev/shm/a.wav
+
+# play icon calibration pics
+for w in 150 170 190 210 230 250; do for h in 130 150 170 190 210; do /c/Program\ Files/ImageMagick-7.0.11-Q16-HDRI/magick.exe convert -size ${w}x${h} xc:brown -fill orange -draw "circle $((w/2)),$((h/2)) $((w/2)),$((h/3))" $w-$h.png; done; done
+
+
 ##
 ## vscode
 
@@ -89,6 +163,25 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 brew install python@2
 pip install virtualenv
 
+# readme toc
+cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md
+
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
+# determine server version
+git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser,up2k}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser,up2k}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+
+# download all sfx versions
+curl https://api.github.com/repos/9001/copyparty/releases?per_page=100 | jq -r '.[] | .tag_name + " " + .name' | tr -d '\r' | while read v t; do fn="copyparty $v $t.py"; [ -e "$fn" ] || curl https://github.com/9001/copyparty/releases/download/$v/copyparty-sfx.py -Lo "$fn"; done
+
 
 ##
 ## http 206
@@ -114,7 +207,7 @@ Range: bytes=26-         Content-Range: bytes */26
 
 var tsh = [];
 function convert_markdown(md_text, dest_dom) {
-    tsh.push(new Date().getTime());
+    tsh.push(Date.now());
     while (tsh.length > 10)
         tsh.shift();
     if (tsh.length > 1) {
@@ -130,3 +223,4 @@ mk() { rm -rf /tmp/foo; sudo -u ed bash -c 'mkdir /tmp/foo; echo hi > /tmp/foo/b
 mk && t0="$(date)" && while true; do date -s "$(date '+ 1 hour')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
 mk && sudo -u ed flock /tmp/foo sleep 40 & sleep 1; ps aux | grep -E 'sleep 40$' && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
 mk && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; tar -cf/dev/null /tmp/foo; done; echo "$t0"
+

docs/nuitka.txt

@@ -0,0 +1,82 @@
+# recipe for building an exe with nuitka (extreme jank edition)
+#
+# NOTE: win7 and win10 builds both work on win10 but
+#   on win7 they immediately c0000005 in kernelbase.dll
+#
+# first install python-3.6.8-amd64.exe
+#   [x] add to path
+#
+# copypaste the rest of this file into cmd
+
+rem from pypi
+cd \users\ed\downloads
+python -m pip install --user Nuitka-0.6.14.7.tar.gz
+
+rem https://github.com/brechtsanders/winlibs_mingw/releases/download/10.2.0-11.0.0-8.0.0-r5/winlibs-x86_64-posix-seh-gcc-10.2.0-llvm-11.0.0-mingw-w64-8.0.0-r5.zip
+mkdir C:\Users\ed\AppData\Local\Nuitka\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\x86_64\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\x86_64\10.2.0-11.0.0-8.0.0-r5\
+copy c:\users\ed\downloads\winlibs-x86_64-posix-seh-gcc-10.2.0-llvm-11.0.0-mingw-w64-8.0.0-r5.zip C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\x86_64\10.2.0-11.0.0-8.0.0-r5\winlibs-x86_64-posix-seh-gcc-10.2.0-llvm-11.0.0-mingw-w64-8.0.0-r5.zip
+
+rem https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12-windows-32.zip
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\ccache\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\ccache\v3.7.12\
+copy c:\users\ed\downloads\ccache-3.7.12-windows-32.zip C:\Users\ed\AppData\Local\Nuitka\Nuitka\ccache\v3.7.12\ccache-3.7.12-windows-32.zip
+
+rem https://dependencywalker.com/depends22_x64.zip
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\depends\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\depends\x86_64\
+copy c:\users\ed\downloads\depends22_x64.zip C:\Users\ed\AppData\Local\Nuitka\Nuitka\depends\x86_64\depends22_x64.zip
+
+cd \
+rd /s /q %appdata%\..\local\temp\pe-copyparty
+cd \users\ed\downloads
+python copyparty-sfx.py -h
+cd %appdata%\..\local\temp\pe-copyparty\copyparty
+
+python
+import os, re
+os.rename('../dep-j2/jinja2', '../jinja2')
+os.rename('../dep-j2/markupsafe', '../markupsafe')
+
+print("# nuitka dies if .__init__.stuff is imported")
+with open('__init__.py','r',encoding='utf-8') as f:
+ t1 = f.read()
+
+with open('util.py','r',encoding='utf-8') as f:
+ t2 = f.read().split('\n')[3:]
+
+t2 = [x for x in t2 if 'from .__init__' not in x]
+t = t1 + '\n'.join(t2)
+with open('__init__.py','w',encoding='utf-8') as f:
+ f.write('\n')
+
+with open('util.py','w',encoding='utf-8') as f:
+ f.write(t)
+
+print("# local-imports fail, prefix module names")
+ptn = re.compile(r'^( *from )(\.[^ ]+ import .*)')
+for d, _, fs in os.walk('.'):
+ for f in fs:
+  fp = os.path.join(d, f)
+  if not fp.endswith('.py'):
+   continue
+  t = ''
+  with open(fp,'r',encoding='utf-8') as f:
+   for ln in [x.rstrip('\r\n') for x in f]:
+    m = ptn.match(ln)
+    if not m:
+     t += ln + '\n'
+     continue
+    p1, p2 = m.groups()
+    t += "{}copyparty{}\n".format(p1, p2).replace("__init__", "util")
+  with open(fp,'w',encoding='utf-8') as f:
+   f.write(t)
+
+exit()
+
+cd ..
+
+rd /s /q bout & python -m nuitka --standalone --onefile --windows-onefile-tempdir --python-flag=no_site --assume-yes-for-downloads --include-data-dir=copyparty\web=copyparty\web --include-data-dir=copyparty\res=copyparty\res --run --output-dir=bout --mingw64 --include-package=markupsafe --include-package=jinja2 copyparty

docs/tcp-debug.sh

@@ -0,0 +1,32 @@
+(cd ~/dev/copyparty && strace -Tttyyvfs 256 -o strace.strace python3 -um copyparty -i 127.0.0.1 --http-only --stackmon /dev/shm/cpps,10 ) 2>&1 | tee /dev/stderr > ~/log-copyparty-$(date +%Y-%m%d-%H%M%S).txt
+
+14/Jun/2021:16:34:02 1623688447.212405 death
+14/Jun/2021:16:35:02 1623688502.420860 back
+
+tcpdump -nni lo -w /home/ed/lo.pcap
+
+# 16:35:25.324662 IP 127.0.0.1.48632 > 127.0.0.1.3920: Flags [F.], seq 849, ack 544, win 359, options [nop,nop,TS val 809396796 ecr 809396796], length 0
+
+tcpdump -nnr /home/ed/lo.pcap | awk '/ > 127.0.0.1.3920: /{sub(/ > .*/,"");sub(/.*\./,"");print}' | sort -n | uniq | while IFS= read -r port; do echo; tcpdump -nnr /home/ed/lo.pcap 2>/dev/null | grep -E "\.$port( > |: F)" | sed -r 's/ > .*, /, /'; done | grep -E '^16:35:0.*length [^0]' -C50
+
+16:34:02.441732 IP 127.0.0.1.48638, length 0
+16:34:02.441738 IP 127.0.0.1.3920, length 0
+16:34:02.441744 IP 127.0.0.1.48638, length 0
+16:34:02.441756 IP 127.0.0.1.48638, length 791
+16:34:02.441759 IP 127.0.0.1.3920, length 0
+16:35:02.445529 IP 127.0.0.1.48638, length 0
+16:35:02.489194 IP 127.0.0.1.3920, length 0
+16:35:02.515595 IP 127.0.0.1.3920, length 216
+16:35:02.515600 IP 127.0.0.1.48638, length 0
+
+grep 48638 "$(find ~ -maxdepth 1 -name log-copyparty-\*.txt | sort | tail -n 1)"
+
+1623688502.510380 48638 rh
+1623688502.511291 48638 Unrecv direct ...
+1623688502.511827 48638 rh = 791
+16:35:02.518 127.0.0.1 48638       shut(8): [Errno 107] Socket not connected
+Exception in thread httpsrv-0.1-48638:
+
+grep 48638 ~/dev/copyparty/strace.strace
+14561 16:35:02.506310 <... accept4 resumed> {sa_family=AF_INET, sin_port=htons(48638), sin_addr=inet_addr("127.0.0.1")}, [16], SOCK_CLOEXEC) = 8<TCP:[127.0.0.1:3920->127.0.0.1:48638]> <0.000012>
+15230 16:35:02.510725 write(1<pipe:[256639555]>, "1623688502.510380 48638 rh\n", 27 <unfinished ...>

scripts/copyparty-repack.sh

@@ -20,6 +20,7 @@ set -e
 # -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py
 
 
+command -v gnutar && tar() { gnutar "$@"; }
 command -v gtar && tar() { gtar "$@"; }
 command -v gsed && sed() { gsed "$@"; }
 td="$(mktemp -d)"
@@ -29,11 +30,11 @@ pwd
 
 
 dl_text() {
-	command -v curl && exec curl "$@"
+	command -v curl >/dev/null && exec curl "$@"
 	exec wget -O- "$@"
 }
 dl_files() {
-	command -v curl && exec curl -L --remote-name-all "$@"
+	command -v curl >/dev/null && exec curl -L --remote-name-all "$@"
 	exec wget "$@"
 }
 export -f dl_files
@@ -91,20 +92,34 @@ chmod 755 \
   copyparty-extras/copyparty-*/{scripts,bin}/*
 
 
-# extract and repack the sfx with less features enabled
+# extract the sfx
 ( cd copyparty-extras/sfx-full/
 ./copyparty-sfx.py -h
-cd ../copyparty-*/
-./scripts/make-sfx.sh re no-ogv no-cm
 )
 
 
-# put new sfx into copyparty-extras/sfx-lite/,
-# fuse client into copyparty-extras/,
+repack() {
+
+	# do the repack
+	(cd copyparty-extras/copyparty-*/
+	./scripts/make-sfx.sh $2
+	)
+
+	# put new sfx into copyparty-extras/$name/,
+	( cd copyparty-extras/
+	mv copyparty-*/dist/* $1/
+	)
+}
+
+repack sfx-full "re gz no-sh"
+repack sfx-lite "re no-ogv no-cm"
+repack sfx-lite "re no-ogv no-cm gz no-sh"
+
+
+# move fuse client into copyparty-extras/,
 # copy lite-sfx.py to ./copyparty,
 # delete extracted source code
 ( cd copyparty-extras/
-mv copyparty-*/dist/* sfx-lite/
 mv copyparty-*/bin/copyparty-fuse.py .
 cp -pv sfx-lite/copyparty-sfx.py ../copyparty
 rm -rf copyparty-{0..9}*.*.*{0..9}
@@ -118,6 +133,7 @@ true
 
 
 # create the bundle
+printf '\n\n'
 fn=copyparty-$(date +%Y-%m%d-%H%M%S).tgz
 tar -czvf "$od/$fn" *
 cd "$od"

scripts/deps-docker/Dockerfile

@@ -1,8 +1,9 @@
 FROM    alpine:3.13
 WORKDIR /z
 ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
+        ver_hashwasm=4.7.0 \
         ver_marked=1.1.0 \
-        ver_ogvjs=1.8.0 \
+        ver_ogvjs=1.8.4 \
         ver_mde=2.14.0 \
         ver_codemirror=5.59.3 \
         ver_fontawesome=5.13.0 \
@@ -21,7 +22,11 @@ RUN     mkdir -p /z/dist/no-pk \
         && wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
         && wget https://github.com/FortAwesome/Font-Awesome/releases/download/$ver_fontawesome/fontawesome-free-$ver_fontawesome-web.zip -O fontawesome.zip \
         && wget https://github.com/google/zopfli/archive/zopfli-$ver_zopfli.tar.gz -O zopfli.tgz \
+        && wget https://github.com/Daninet/hash-wasm/releases/download/v$ver_hashwasm/hash-wasm@$ver_hashwasm.zip -O hash-wasm.zip \
         && unzip ogvjs.zip \
+        && (mkdir hash-wasm \
+            && cd hash-wasm \
+            && unzip ../hash-wasm.zip) \
         && (tar -xf asmcrypto.tgz \
             && cd asmcrypto.js-$ver_asmcrypto \
             && npm install ) \
@@ -58,29 +63,27 @@ RUN     tar -xf zopfli.tgz \
 RUN     cd asmcrypto.js-$ver_asmcrypto \
         && echo "export { Sha512 } from './hash/sha512/sha512';" > src/entry-export_all.ts \
         && node -r esm build.js \
-        && mv asmcrypto.all.es5.js /z/dist/sha512.js
+        && awk '/HMAC state/{o=1}  /var HEAP/{o=0}  /function hmac_reset/{o=1}  /return \{/{o=0}  /var __extends =/{o=1}  /var Hash =/{o=0}  /hmac_|pbkdf2_/{next}  o{next}  {gsub(/IllegalStateError/,"Exception")}  {sub(/^ +/,"");sub(/^\/\/ .*/,"");sub(/;$/," ;")}  1' < asmcrypto.all.es5.js > /z/dist/sha512.ac.js
+
+
+# build hash-wasm
+RUN     cd hash-wasm \
+        && mv sha512.umd.min.js /z/dist/sha512.hw.js
 
 
 # build ogvjs
 RUN     cd ogvjs-$ver_ogvjs \
         && cp -pv \
-            ogv.js \
             ogv-worker-audio.js \
             ogv-demuxer-ogg-wasm.js \
             ogv-demuxer-ogg-wasm.wasm \
-            ogv-demuxer-webm-wasm.js \
-            ogv-demuxer-webm-wasm.wasm \
             ogv-decoder-audio-opus-wasm.js \
             ogv-decoder-audio-opus-wasm.wasm \
             ogv-decoder-audio-vorbis-wasm.js \
             ogv-decoder-audio-vorbis-wasm.wasm \
-            /z/dist
-
-#            ogv-demuxer-ogg.js \
-#            ogv-demuxer-webm.js \
-#            ogv-decoder-audio-opus.js \
-#            ogv-decoder-audio-vorbis.js \
-#            dynamicaudio.swf \
+            /z/dist \
+        && cp -pv \
+            ogv-es2017.js /z/dist/ogv.js
 
 
 # build marked

scripts/deps-docker/zopfli.makefile

@@ -2,7 +2,7 @@ all: $(addsuffix .gz, $(wildcard *.*))
 
 %.gz: %
 	#brotli -q 11 $<
-	pigz -11 -J 34 -I 573 $<
+	pigz -11 -I 573 $<
 
 # pigz -11 -J 34 -I 100 -F < $< > $@.first
 

scripts/install-githooks.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+[ -e setup.py ] || ..
+[ -e setup.py ] || {
+    echo u wot
+    exit 1
+}
+
+cd .git/hooks
+rm -f pre-commit
+ln -s ../../scripts/run-tests.sh pre-commit

scripts/make-sfx.sh

@@ -2,6 +2,7 @@
 set -e
 echo
 
+help() { exec cat <<'EOF'
 
 # optional args:
 #
@@ -11,12 +12,23 @@ echo
 # `re` does a repack of an sfx which you already executed once
 #   (grabs files from the sfx-created tempdir), overrides `clean`
 #
-# `no-ogv` saves ~500k by removing the opus/vorbis audio codecs
+# `gz` creates a gzip-compressed python sfx instead of bzip2
+#
+# `no-sh` makes just the python sfx, skips the sh/unix sfx
+#
+# `no-ogv` saves ~192k by removing the opus/vorbis audio codecs
 #   (only affects apple devices; everything else has native support)
 #
-# `no-cm` saves ~90k by removing easymde/codemirror
+# `no-cm` saves ~92k by removing easymde/codemirror
 #   (the fancy markdown editor)
+#
+# `no-fnt` saves ~9k by removing the source-code-pro font
+#   (browsers will try to use 'Consolas' instead)
+#
+# `no-dd` saves ~2k by removing the mouse cursor
 
+EOF
+}
 
 # port install gnutar findutils gsed coreutils
 gtar=$(command -v gtar || command -v gnutar) || true
@@ -25,9 +37,23 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
+	shuf() { gshuf "$@"; }
+	nproc() { gnproc; }
+	sha1sum() { shasum "$@"; }
 	unexpand() { gunexpand "$@"; }
 	command -v grealpath >/dev/null &&
 		realpath() { grealpath "$@"; }
+
+	[ -e /opt/local/bin/bzip2 ] &&
+		bzip2() { /opt/local/bin/bzip2 "$@"; }
+}
+
+gawk=$(command -v gawk || command -v gnuawk || command -v awk)
+awk() { $gawk "$@"; }
+
+pybin=$(command -v python3 || command -v python) || {
+	echo need python
+	exit 1
 }
 
 [ -e copyparty/__main__.py ] || cd ..
@@ -38,12 +64,23 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	exit 1
 }
 
+use_gz=
+do_sh=1
+do_py=1
 while [ ! -z "$1" ]; do
-	[ "$1" = clean  ] && clean=1  && shift && continue
-	[ "$1" = re     ] && repack=1 && shift && continue
-	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
-	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
-	break
+	case $1 in
+		clean)  clean=1  ; ;;
+		re)     repack=1 ; ;;
+		gz)     use_gz=1 ; ;;
+		no-ogv) no_ogv=1 ; ;;
+		no-fnt) no_fnt=1 ; ;;
+		no-dd)  no_dd=1  ; ;;
+		no-cm)  no_cm=1  ; ;;
+		no-sh)  do_sh=   ; ;;
+		no-py)  do_py=   ; ;;
+		*)      help     ; ;;
+	esac
+	shift
 done
 
 tmv() {
@@ -51,16 +88,23 @@ tmv() {
 	mv t "$1"
 }
 
+stamp=$(
+	for d in copyparty scripts; do
+		find $d -type f -printf '%TY-%Tm-%Td %TH:%TM:%TS %p\n'
+	done | sort | tail -n 1 | sha1sum | cut -c-16
+)
+
 rm -rf sfx/*
 mkdir -p sfx build
 cd sfx
 
-[ $repack ] && {
-	old="$(
-		printf '%s\n' "$TMPDIR" /tmp |
-		awk '/./ {print; exit}'
-	)/pe-copyparty"
+tmpdir="$(
+	printf '%s\n' "$TMPDIR" /tmp |
+	awk '/./ {print; exit}'
+)"
 
+[ $repack ] && {
+	old="$tmpdir/pe-copyparty"
 	echo "repack of files in $old"
 	cp -pR "$old/"*{dep-j2,copyparty} .
 }
@@ -102,9 +146,10 @@ cd sfx
 }
 
 ver=
+[ -z "$repack" ] &&
 git describe --tags >/dev/null 2>/dev/null && {
 	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
-	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//; s/-g?/./g')";
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//')";
 	t_ver=
 
 	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
@@ -133,7 +178,7 @@ git describe --tags >/dev/null 2>/dev/null && {
 
 [ -z "$ver" ] && 
 	ver="$(awk '/^VERSION *= \(/ {
-		gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < copyparty/__version__.py)"
+		gsub(/[^0-9,a-g-]/,""); gsub(/,/,"."); print; exit}' < copyparty/__version__.py)"
 
 ts=$(date -u +%s)
 hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
@@ -142,15 +187,15 @@ mkdir -p ../dist
 sfx_out=../dist/copyparty-sfx
 
 echo cleanup
-find .. -name '*.pyc' -delete
-find .. -name __pycache__ -delete
+find -name '*.pyc' -delete
+find -name __pycache__ -delete
 
 # especially prevent osx from leaking your lan ip (wtf apple)
-find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
-find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
+find -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
+find -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/dbg-* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -166,53 +211,178 @@ done
 	rm -rf copyparty/web/mde.* copyparty/web/deps/easymde*
 	echo h > copyparty/web/mde.html
 	f=copyparty/web/md.html
-	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
+	sed -r '/edit2">edit \(fancy/d' <$f >t
+	tmv "$f"
 }
 
+[ $no_fnt ] && {
+	rm -f copyparty/web/deps/scp.woff2
+	f=copyparty/web/ui.css
+	gzip -d "$f.gz" || true
+	sed -r "s/src:.*scp.*\)/src:local('Consolas')/" <$f >t
+	tmv "$f"
+}
+
+[ $no_dd ] && {
+	rm -rf copyparty/web/dd
+	f=copyparty/web/browser.css
+	gzip -d "$f.gz" || true
+	sed -r 's/(cursor: ?)url\([^)]+\), ?(pointer)/\1\2/; /[0-9]+% \{cursor:/d; /animation: ?cursor/d' <$f >t
+	tmv "$f"
+}
+
+[ $repack ] ||
 find | grep -E '\.py$' |
   grep -vE '__version__' |
   tr '\n' '\0' |
-  xargs -0 python ../scripts/uncomment.py
+  xargs -0 $pybin ../scripts/uncomment.py
 
 f=dep-j2/jinja2/constants.py
 awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
 tmv "$f"
 
+grep -rLE '^#[^a-z]*coding: utf-8' dep-j2 |
+while IFS= read -r f; do
+	(echo "# coding: utf-8"; cat "$f") >t
+	tmv "$f"
+done
+
 # up2k goes from 28k to 22k laff
-echo entabbening
-find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
+awk 'BEGIN{gensub(//,"",1)}' </dev/null &&
+echo entabbening &&
+find | grep -E '\.css$' | while IFS= read -r f; do
+	awk '{
+		sub(/^[ \t]+/,"");
+		sub(/[ \t]+$/,"");
+		$0=gensub(/^([a-z-]+) *: *(.*[^ ]) *;$/,"\\1:\\2;","1");
+		sub(/ +\{$/,"{");
+		gsub(/, /,",")
+	}
+	!/\}$/ {printf "%s",$0;next}
+	1
+	' <$f | sed 's/;\}$/}/' >t
+	tmv "$f"
+done
+unexpand -h 2>/dev/null &&
+find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
 
+gzres() {
+	command -v pigz &&
+		pk='pigz -11 -I 2560' ||
+		pk='gzip'
+
+	np=$(nproc)
+	echo "$pk #$np"
+
+	while IFS=' ' read -r _ f; do
+		while true; do
+			na=$(ps auxwww | grep -F "$pk" | wc -l)
+			[ $na -le $np ] && break
+			sleep 0.2
+		done
+		echo -n .
+		$pk "$f" &
+	done < <(
+		find -printf '%s %p\n' |
+		grep -E '\.(js|css)$' |
+		grep -vF /deps/ |
+		sort -nr
+	)
+	wait
+	echo
+}
+
+
+zdir="$tmpdir/cpp-mksfx"
+[ -e "$zdir/$stamp" ] || rm -rf "$zdir"
+mkdir -p "$zdir"
+echo a > "$zdir/$stamp"
+nf=$(ls -1 "$zdir"/arc.* | wc -l)
+[ $nf -ge 2 ] && [ ! $repack ] && use_zdir=1 || use_zdir=
+
+[ $use_zdir ] || {
+	echo "$nf alts += 1"
+	gzres
+	[ $repack ] ||
+		tar -cf "$zdir/arc.$(date +%s)" copyparty/web/*.gz
+}
+[ $use_zdir ] && {
+	arcs=("$zdir"/arc.*)
+	arc="${arcs[$RANDOM % ${#arcs[@]} ] }"
+	echo "using $arc"
+	tar -xf "$arc"
+	for f in copyparty/web/*.gz; do
+		rm "${f%.*}"
+	done
+}
+
+
+echo gen tarlist
+for d in copyparty dep-j2; do find $d -type f; done |
+sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
+sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
+
+(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | shuf) >list || true
+
 echo creating tar
 args=(--owner=1000 --group=1000)
 [ "$OSTYPE" = msys ] &&
 	args=()
 
-tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
+tar -cf tar "${args[@]}" --numeric-owner -T list
+
+pc=bzip2
+pe=bz2
+[ $use_gz ] && pc=gzip && pe=gz
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2
-for n in {2..9}; do cp tar t.$n;  xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz
-rm t.*
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2; }
+[ $do_sh ] && { for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
+rm t.* || true
+exts=()
 
+
+[ $do_sh ] && {
+exts+=(.sh)
 echo creating unix sfx
 (
 	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
 	grep -E '^sfx_eof$' -B 9001;
 	cat tar.xz
 ) >$sfx_out.sh
+}
 
-echo creating generic sfx
-python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
-mv sfx.out $sfx_out.py
-chmod 755 $sfx_out.*
+
+[ $do_py ] && {
+	echo creating generic sfx
+
+	py=../scripts/sfx.py
+	suf=
+	[ $use_gz ] && {
+		sed -r 's/"r:bz2"/"r:gz"/' <$py >$py.t
+		py=$py.t
+		suf=-gz
+	}
+
+	$pybin $py --sfx-make tar.bz2 $ver $ts
+	mv sfx.out $sfx_out$suf.py
+	
+	exts+=($suf.py)
+	[ $use_gz ] &&
+		rm $py
+}
+
+
+chmod 755 $sfx_out*
 
 printf "done:\n"
-printf "  %s\n" "$(realpath $sfx_out)."{sh,py}
-# rm -rf *
+for ext in ${exts[@]}; do
+	printf "  %s\n" "$(realpath $sfx_out)"$ext
+done
 
-# tar -tvf ../sfx/tar | sed -r 's/(.* ....-..-.. ..:.. )(.*)/\2 `` \1/' | sort | sed -r 's/(.*) `` (.*)/\2 \1/'| less
-# for n in {1..9}; do tar -tf tar | grep -vE '/$' | sed -r 's/(.*)\.(.*)/\2.\1/' | sort | sed -r 's/([^\.]+)\.(.*)/\2.\1/' | tar -cT- | bzip2 -c$n | wc -c; done 
+# apk add bash python3 tar xz bzip2
+# while true; do ./make-sfx.sh; for f in ..//dist/copyparty-sfx.{sh,py}; do mv $f $f.$(wc -c <$f | awk '{print$1}'); done; done

scripts/profile.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import sys
+
+sys.path.insert(0, ".")
+cmd = sys.argv[1]
+
+if cmd == "cpp":
+    from copyparty.__main__ import main
+
+    argv = ["__main__", "-v", "srv::r", "-v", "../../yt:yt:r"]
+    main(argv=argv)
+
+elif cmd == "test":
+    from unittest import main
+
+    argv = ["__main__", "discover", "-s", "tests"]
+    main(module=None, argv=argv)
+
+else:
+    raise Exception()
+
+# import dis; print(dis.dis(main))
+
+
+# macos:
+#   option1) python3.9 -m pip install --user -U vmprof==0.4.9
+#   option2) python3.9 -m pip install --user -U https://github.com/vmprof/vmprof-python/archive/refs/heads/master.zip
+#
+# python -m vmprof -o prof --lines ./scripts/profile.py test
+
+# linux: ~/.local/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+#   win: %appdata%\..\Roaming\Python\Python39\Scripts\vmprofshow.exe prof tree

scripts/run-tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+pids=()
+for py in python{2,3}; do
+    nice $py -m unittest discover -s tests >/dev/null &
+    pids+=($!)
+done
+
+python3 scripts/test/smoketest.py &
+pids+=($!)
+
+for pid in ${pids[@]}; do
+    wait $pid
+done

scripts/sfx.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python
-# coding: utf-8
+# coding: latin-1
 from __future__ import print_function, unicode_literals
 
-import os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
+import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform, tempfile, traceback
 import subprocess as sp
 
 """
-run me with any version of python, i will unpack and run copyparty
+to edit this file, use HxD or "vim -b"
+  (there is compressed stuff at the end)
 
-(but please don't edit this file with a text editor
-  since that would probably corrupt the binary stuff at the end)
+run me with any version of python, i will unpack and run copyparty
 
 there's zero binaries! just plaintext python scripts all the way down
   so you can easily unpack the archive and inspect it for shady stuff
@@ -27,21 +27,21 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
+WINDOWS = sys.platform in ["win32", "msys"]
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
-cpp = None
 
 
-def eprint(*args, **kwargs):
-    kwargs["file"] = sys.stderr
-    print(*args, **kwargs)
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    print(*a, **ka)
 
 
-def msg(*args, **kwargs):
-    if args:
-        args = ["[SFX]", args[0]] + list(args[1:])
+def msg(*a, **ka):
+    if a:
+        a = ["[SFX]", a[0]] + list(a[1:])
 
-    eprint(*args, **kwargs)
+    eprint(*a, **ka)
 
 
 # skip 1
@@ -156,6 +156,9 @@ def encode(data, size, cksum, ver, ts):
                 skip = True
                 continue
 
+            if ln.strip().startswith("# fmt: "):
+                continue
+
             unpk += ln + "\n"
 
         for k, v in [
@@ -209,11 +212,11 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    hasher = hashlib.md5()
+    h = hashlib.md5()
     for block in yieldfile(fn):
-        hasher.update(block)
+        h.update(block)
 
-    return hasher.hexdigest()
+    return h.hexdigest()
 
 
 def unpack():
@@ -222,9 +225,10 @@ def unpack():
     tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
-    final = os.path.join(top, name)
-    mine = os.path.join(top, withpid)
-    tar = os.path.join(mine, "tar")
+    opj = os.path.join
+    final = opj(top, name)
+    mine = opj(top, withpid)
+    tar = opj(mine, "tar")
 
     try:
         if tag in os.listdir(final):
@@ -233,28 +237,24 @@ def unpack():
     except:
         pass
 
-    nwrite = 0
+    sz = 0
     os.mkdir(mine)
     with open(tar, "wb") as f:
         for buf in get_payload():
-            nwrite += len(buf)
+            sz += len(buf)
             f.write(buf)
 
-    if nwrite != SIZE:
-        t = "\n\n  bad file:\n    expected {} bytes, got {}\n".format(SIZE, nwrite)
-        raise Exception(t)
-
-    cksum = hashfile(tar)
-    if cksum != CKSUM:
-        t = "\n\n  bad file:\n    {} expected,\n    {} obtained\n".format(CKSUM, cksum)
-        raise Exception(t)
+    ck = hashfile(tar)
+    if ck != CKSUM:
+        t = "\n\nexpected {} ({} byte)\nobtained {} ({} byte)\nsfx corrupt"
+        raise Exception(t.format(CKSUM, SIZE, ck, sz))
 
     with tarfile.open(tar, "r:bz2") as tf:
         tf.extractall(mine)
 
     os.remove(tar)
 
-    with open(os.path.join(mine, tag), "wb") as f:
+    with open(opj(mine, tag), "wb") as f:
         f.write(b"h\n")
 
     try:
@@ -272,25 +272,25 @@ def unpack():
     except:
         pass
 
+    for fn in u8(os.listdir(top)):
+        if fn.startswith(name) and fn != withpid:
+            try:
+                old = opj(top, fn)
+                if time.time() - os.path.getmtime(old) > 86400:
+                    shutil.rmtree(old)
+            except:
+                pass
+
     try:
         os.symlink(mine, final)
     except:
         try:
             os.rename(mine, final)
+            return final
         except:
             msg("reloc fail,", mine)
-            return mine
 
-    for fn in u8(os.listdir(top)):
-        if fn.startswith(name) and fn not in [name, withpid]:
-            try:
-                old = os.path.join(top, fn)
-                if time.time() - os.path.getmtime(old) > 10:
-                    shutil.rmtree(old)
-            except:
-                pass
-
-    return final
+    return mine
 
 
 def get_payload():
@@ -307,96 +307,119 @@ def get_payload():
         if ofs < 0:
             raise Exception("could not find archive marker")
 
-        # start reading from the final b"\n"
+        # start at final b"\n"
         fpos = ofs + len(ptn) - 3
-        # msg("tar found at", fpos)
         f.seek(fpos)
         dpos = 0
-        leftovers = b""
+        rem = b""
         while True:
             rbuf = f.read(1024 * 32)
             if rbuf:
-                buf = leftovers + rbuf
+                buf = rem + rbuf
                 ofs = buf.rfind(b"\n")
                 if len(buf) <= 4:
-                    leftovers = buf
+                    rem = buf
                     continue
 
                 if ofs >= len(buf) - 4:
-                    leftovers = buf[ofs:]
+                    rem = buf[ofs:]
                     buf = buf[:ofs]
                 else:
-                    leftovers = b"\n# "
+                    rem = b"\n# "
             else:
-                buf = leftovers
+                buf = rem
 
             fpos += len(buf) + 1
-            buf = (
-                buf.replace(b"\n# ", b"")
-                .replace(b"\n#r", b"\r")
-                .replace(b"\n#n", b"\n")
-            )
-            dpos += len(buf) - 1
+            for a, b in [[b"\n# ", b""], [b"\n#r", b"\r"], [b"\n#n", b"\n"]]:
+                buf = buf.replace(a, b)
 
+            dpos += len(buf) - 1
             yield buf
 
             if not rbuf:
                 break
 
 
-def confirm():
+def utime(top):
+    i = 0
+    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    while WINDOWS:
+        t = int(time.time())
+        if i:
+            msg("utime {}, {}".format(i, t))
+
+        for f in files:
+            os.utime(f, (t, t))
+
+        i += 1
+        time.sleep(78123)
+
+
+def confirm(rv):
     msg()
+    msg("retcode", rv if rv else traceback.format_exc())
     msg("*** hit enter to exit ***")
     try:
         raw_input() if PY2 else input()
     except:
         pass
 
+    sys.exit(rv or 1)
 
-def run(tmp, j2ver):
-    global cpp
 
-    msg("jinja2:", j2ver or "bundled")
+def run(tmp, j2):
+    msg("jinja2:", j2 or "bundled")
     msg("sfxdir:", tmp)
+    msg()
 
-    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    # block systemd-tmpfiles-clean.timer
     try:
         import fcntl
 
         fd = os.open(tmp, os.O_RDONLY)
         fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
-    except:
-        pass
+    except Exception as ex:
+        if not WINDOWS:
+            msg("\033[31mflock:{!r}\033[0m".format(ex))
+
+    t = threading.Thread(target=utime, args=(tmp,))
+    t.daemon = True
+    t.start()
 
     ld = [tmp, os.path.join(tmp, "dep-j2")]
-    if j2ver:
+    if j2:
         del ld[-1]
 
-    cmd = (
-        "import sys, runpy; "
-        + "".join(['sys.path.insert(0, r"' + x + '"); ' for x in ld])
-        + 'runpy.run_module("copyparty", run_name="__main__")'
-    )
-    cmd = [sys.executable, "-c", cmd] + list(sys.argv[1:])
-
-    cmd = [str(x) for x in cmd]
-    msg("\n", cmd, "\n")
-    cpp = sp.Popen(cmd)
-    try:
-        cpp.wait()
-    except:
-        cpp.wait()
-
-    if cpp.returncode != 0:
-        confirm()
-
-    sys.exit(cpp.returncode)
+    if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
+        run_s(ld)
+    else:
+        run_i(ld)
 
 
-def bye(sig, frame):
-    if cpp is not None:
-        cpp.terminate()
+def run_i(ld):
+    for x in ld:
+        sys.path.insert(0, x)
+
+    from copyparty.__main__ import main as p
+
+    p()
+
+
+def run_s(ld):
+    # fmt: off
+    c = "import sys,runpy;" + "".join(['sys.path.insert(0,r"' + x + '");' for x in ld]) + 'runpy.run_module("copyparty",run_name="__main__")'
+    c = [str(x) for x in [sys.executable, "-c", c] + list(sys.argv[1:])]
+    # fmt: on
+    msg("\n", c, "\n")
+    p = sp.Popen(c)
+
+    def bye(*a):
+        p.send_signal(signal.SIGINT)
+
+    signal.signal(signal.SIGTERM, bye)
+    p.wait()
+
+    raise SystemExit(p.returncode)
 
 
 def main():
@@ -430,16 +453,23 @@ def main():
 
     # skip 0
 
-    signal.signal(signal.SIGTERM, bye)
-
-    tmp = unpack()
+    tmp = os.path.realpath(unpack())
 
     try:
-        from jinja2 import __version__ as j2ver
+        from jinja2 import __version__ as j2
     except:
-        j2ver = None
+        j2 = None
 
-    return run(tmp, j2ver)
+    try:
+        run(tmp, j2)
+    except SystemExit as ex:
+        c = ex.code
+        if c not in [0, -15]:
+            confirm(ex.code)
+    except KeyboardInterrupt:
+        pass
+    except:
+        confirm(0)
 
 
 if __name__ == "__main__":

scripts/sfx.sh

@@ -47,7 +47,7 @@ grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
 	printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
 	$_py -c 'import jinja2' 2>/dev/null || continue
 	printf '%s\n' "$_py"
-	mv $dir/{,x.}jinja2
+	mv $dir/{,x.}dep-j2
 	break
 done)"
 

scripts/speedtest-fs.py

@@ -17,14 +17,15 @@ __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
 
-def get_spd(nbyte, nsec):
+def get_spd(nbyte, nfiles, nsec):
     if not nsec:
-        return "0.000 MB   0.000 sec   0.000 MB/s"
+        return "0.000 MB   0 files   0.000 sec   0.000 MB/s   0.000 f/s"
 
     mb = nbyte / (1024 * 1024.0)
     spd = mb / nsec
+    nspd = nfiles / nsec
 
-    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+    return f"{mb:.3f} MB   {nfiles} files   {nsec:.3f} sec   {spd:.3f} MB/s   {nspd:.3f} f/s"
 
 
 class Inf(object):
@@ -36,6 +37,7 @@ class Inf(object):
         self.mtx_reports = threading.Lock()
 
         self.n_byte = 0
+        self.n_file = 0
         self.n_sec = 0
         self.n_done = 0
         self.t0 = t0
@@ -63,7 +65,8 @@ class Inf(object):
                 continue
 
             msgs = msgs[-64:]
-            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            spd = get_spd(self.n_byte, len(self.reports), self.n_sec)
+            msgs = [f"{spd}   {x}" for x in msgs]
             print("\n".join(msgs))
 
     def report(self, fn, n_byte, n_sec):
@@ -131,8 +134,9 @@ def main():
 
     num_threads = 8
     read_sz = 32 * 1024
+    targs = (q, inf, read_sz)
     for _ in range(num_threads):
-        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr = threading.Thread(target=worker, args=targs)
         thr.daemon = True
         thr.start()
 
@@ -151,14 +155,14 @@ def main():
     log = inf.reports
     log.sort()
     for nbyte, nsec, fn in log[-64:]:
-        print(f"{get_spd(nbyte, nsec)}   {fn}")
+        spd = get_spd(nbyte, len(log), nsec)
+        print(f"{spd}   {fn}")
 
     print()
     print("\n".join(inf.errors))
 
-    print(get_spd(inf.n_byte, t2 - t0))
+    print(get_spd(inf.n_byte, len(log), t2 - t0))
 
 
 if __name__ == "__main__":
     main()
-

scripts/test/race.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import json
+import threading
+import http.client
+
+
+class Conn(object):
+    def __init__(self, ip, port):
+        self.s = http.client.HTTPConnection(ip, port, timeout=260)
+        self.st = []
+
+    def get(self, vpath):
+        self.st = [time.time()]
+
+        self.s.request("GET", vpath)
+        self.st.append(time.time())
+
+        ret = self.s.getresponse()
+        self.st.append(time.time())
+
+        if ret.status < 200 or ret.status >= 400:
+            raise Exception(ret.status)
+
+        ret = ret.read()
+        self.st.append(time.time())
+
+        return ret
+
+    def get_json(self, vpath):
+        ret = self.get(vpath)
+        return json.loads(ret)
+
+
+class CState(threading.Thread):
+    def __init__(self, cs):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.cs = cs
+        self.start()
+
+    def run(self):
+        colors = [5, 1, 3, 2, 7]
+        remotes = []
+        remotes_ok = False
+        while True:
+            time.sleep(0.001)
+            if not remotes_ok:
+                remotes = []
+                remotes_ok = True
+                for conn in self.cs:
+                    try:
+                        remotes.append(conn.s.sock.getsockname()[1])
+                    except:
+                        remotes.append("?")
+                        remotes_ok = False
+
+            m = []
+            for conn, remote in zip(self.cs, remotes):
+                stage = len(conn.st)
+                m.append(f"\033[3{colors[stage]}m{remote}")
+
+            m = " ".join(m)
+            print(f"{m}\033[0m\n\033[A", end="")
+
+
+def allget(cs, urls):
+    thrs = []
+    for c, url in zip(cs, urls):
+        t = threading.Thread(target=c.get, args=(url,))
+        t.start()
+        thrs.append(t)
+
+    for t in thrs:
+        t.join()
+
+
+def main():
+    os.system("")
+
+    ip, port = sys.argv[1].split(":")
+    port = int(port)
+
+    cs = []
+    for _ in range(64):
+        cs.append(Conn(ip, 3923))
+
+    CState(cs)
+
+    urlbase = "/doujin/c95"
+    j = cs[0].get_json(f"{urlbase}?ls")
+    urls = []
+    for d in j["dirs"]:
+        urls.append(f"{urlbase}/{d['href']}?th=w")
+
+    for n in range(100):
+        print(n)
+        allget(cs, urls)
+
+
+if __name__ == "__main__":
+    main()