v0.10.4

idk forgot to pull

.eslintrc.json

@@ -0,0 +1,12 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 12
+    },
+    "rules": {
+    }
+}

.gitattributes

@@ -1,4 +1,6 @@
 * text eol=lf
 
+*.reg text eol=crlf
+
 *.png binary
 *.gif binary

.gitignore

@@ -8,17 +8,15 @@ copyparty.egg-info/
 buildenv/
 build/
 dist/
-*.rst
-.env/
+sfx/
+.venv/
 
-# sublime
+# ide
 *.sublime-workspace
 
 # winmerge
 *.bak
 
-# other licenses
-contrib/
-
-# deps
-copyparty/web/deps
+# derived
+copyparty/web/deps/
+srv/

.vscode/launch.json

@@ -9,15 +9,26 @@
             "console": "integratedTerminal",
             "cwd": "${workspaceFolder}",
             "args": [
-                "-j",
-                "0",
                 //"-nw",
+                "-ed",
+                "-emp",
+                "-e2dsa",
+                "-e2ts",
                 "-a",
                 "ed:wark",
                 "-v",
-                "/home/ed/inc:inc:r:aed"
+                "srv::r:aed:cnodupe",
+                "-v",
+                "dist:dist:r"
             ]
         },
+        {
+            "name": "No debug",
+            "preLaunchTask": "no_dbg",
+            "type": "python",
+            //"request": "attach", "port": 42069
+            // fork: nc -l 42069 </dev/null
+        },
         {
             "name": "Run active unit test",
             "type": "python",

.vscode/settings.json

@@ -37,7 +37,7 @@
     "python.linting.banditEnabled": true,
     "python.linting.flake8Args": [
         "--max-line-length=120",
-        "--ignore=E722,F405,E203,W503,W293",
+        "--ignore=E722,F405,E203,W503,W293,E402",
     ],
     "python.linting.banditArgs": [
         "--ignore=B104"
@@ -50,11 +50,9 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "editor.codeActionsOnSaveTimeout": 9001,
-    "editor.formatOnSaveTimeout": 9001,
-    //
-    //  things you may wanna edit:
-    //
-    "python.pythonPath": ".env/bin/python",
-    //"python.linting.enabled": true,
+    "python.formatting.blackArgs": [
+        "-t",
+        "py27"
+    ],
+    "python.linting.enabled": true,
 }

.vscode/tasks.json

@@ -0,0 +1,17 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "pre",
+            "command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
+            "type": "shell"
+        },
+        {
+            "label": "no_dbg",
+            "type": "shell",
+            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -e2ts -a ed:wark -v srv::r:aed:cnodupe -v dist:dist:r ;exit 1"
+            // -v ~/Music/mt:mt:r:cmtp=.bpm=~/dev/copyparty/bin/mtag/audio-bpm.py:cmtp=key=~/dev/copyparty/bin/mtag/audio-key.py:ce2tsr 
+            // -v ~/Music/mt:mt:r:cmtp=.bpm=~/dev/copyparty/bin/mtag/audio-bpm.py:ce2tsr
+        }
+    ]
+}

README.md

@@ -1,6 +1,6 @@
 # ⇆🎉 copyparty
 
-* http file sharing hub (py2/py3)
+* http file sharing hub (py2/py3) [(on PyPI)](https://pypi.org/project/copyparty/)
 * MIT-Licensed, 2019-05-26, ed @ irc.rizon.net
 
 
@@ -8,61 +8,280 @@
 
 turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
 
-* server runs on anything with `py2.7` or `py3.2+`
+* server runs on anything with `py2.7` or `py3.3+`
 * *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
 * code standard: `black`
 
 
+## readme toc
+
+* top
+    * [quickstart](#quickstart)
+    * [notes](#notes)
+    * [status](#status)
+* [bugs](#bugs)
+* [usage](#usage)
+    * [zip downloads](#zip-downloads)
+* [searching](#searching)
+    * [search configuration](#search-configuration)
+    * [metadata from audio files](#metadata-from-audio-files)
+    * [file parser plugins](#file-parser-plugins)
+    * [complete examples](#complete-examples)
+* [client examples](#client-examples)
+* [dependencies](#dependencies)
+    * [optional gpl stuff](#optional-gpl-stuff)
+* [sfx](#sfx)
+    * [sfx repack](#sfx-repack)
+* [install on android](#install-on-android)
+* [dev env setup](#dev-env-setup)
+* [how to release](#how-to-release)
+* [todo](#todo)
+
+
+## quickstart
+
+download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and you're all set!
+
+running the sfx without arguments (for example doubleclicking it on Windows) will let anyone access the current folder; see `-h` for help if you want accounts and volumes etc
+
+you may also want these, especially on servers:
+* [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
+* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for legit https)
+
+
 ## notes
 
 * iPhone/iPad: use Firefox to download files
 * Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
 * Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
 * Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
+  * because no browsers currently implement the media-query to do this properly orz
 
 
 ## status
 
-* [x] sanic multipart parser
-* [x] load balancer (multiprocessing)
-* [x] upload (plain multipart, ie6 support)
-* [x] upload (js, resumable, multithreaded)
-* [x] download
-* [x] browser
-* [x] media player
-* [ ] thumbnails
-* [ ] download as zip
-* [x] volumes
-* [x] accounts
+* backend stuff
+  * ☑ sanic multipart parser
+  * ☑ load balancer (multiprocessing)
+  * ☑ volumes (mountpoints)
+  * ☑ accounts
+* upload
+  * ☑ basic: plain multipart, ie6 support
+  * ☑ up2k: js, resumable, multithreaded
+  * ☑ stash: simple PUT filedropper
+  * ☑ symlink/discard existing files (content-matching)
+* download
+  * ☑ single files in browser
+  * ☑ folders as zip / tar files
+  * ☑ FUSE client (read-only)
+* browser
+  * ☑ tree-view
+  * ☑ media player
+  * ✖ thumbnails
+  * ✖ SPA (browse while uploading)
+    * currently safe using the file-tree on the left only, not folders in the file list
+* server indexing
+  * ☑ locate files by contents
+  * ☑ search by name/path/date/size
+  * ☑ search by ID3-tags etc.
+* markdown
+  * ☑ viewer
+  * ☑ editor (sure why not)
 
-summary: close to beta
+summary: it works! you can use it! (but technically not even close to beta)
+
+
+# bugs
+
+* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
+* Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
+* probably more, pls let me know
+
+
+# usage
+
+the browser has the following hotkeys
+* `0..9` jump to 10%..90%
+* `U/O` skip 10sec back/forward
+* `J/L` prev/next song
+* `I/K` prev/next folder
+* `P` parent folder
+
+
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+
+# searching
+
+when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
+* make search queries by `size`/`date`/`directory-path`/`filename`, or...
+* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+
+path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
+* path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
+* name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9)
+
+add `-e2ts` to also scan/index tags from music files:
+
+
+## search configuration
+
+searching relies on two databases, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`). Configuration can be done through arguments, volume flags, or a mix of both.
+
+through arguments:
+* `-e2d` enables file indexing on upload
+* `-e2ds` scans writable folders on startup
+* `-e2dsa` scans all mounted volumes (including readonly ones)
+* `-e2t` enables metadata indexing on upload
+* `-e2ts` scans for tags in all files that don't have tags yet
+* `-e2tsr` deletes all existing tags, so a full reindex
+
+the same arguments can be set as volume flags, in addition to `d2d` and `d2t` for disabling:
+* `-v ~/music::r:ce2dsa:ce2tsr` does a full reindex of everything on startup
+* `-v ~/music::r:cd2d` disables **all** indexing, even if any `-e2*` are on
+* `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
+
+`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+
+
+## metadata from audio files
+
+`-mte` decides which tags to index and display in the browser (and also the display order), this can be changed per-volume:
+* `-v ~/music::r:cmte=title,artist` indexes and displays *title* followed by *artist*
+
+if you add/remove a tag from `mte` you will need to run with `-e2tsr` once to rebuild the database, otherwise only new files will be affected
+
+`-mtm` can be used to add or redefine a metadata mapping, say you have media files with `foo` and `bar` tags and you want them to display as `qux` in the browser (preferring `foo` if both are present), then do `-mtm qux=foo,bar` and now you can `-mte artist,title,qux`
+
+tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric value
+
+see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
+
+`--no-mutagen` disables mutagen and uses ffprobe instead, which...
+* is about 20x slower than mutagen
+* catches a few tags that mutagen doesn't
+* avoids pulling any GPL code into copyparty
+* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
+
+
+## file parser plugins
+
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (as argument or volume flag), there is a default timeout of 30sec
+
+* `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
+* `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
+* `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
+
+
+## complete examples
+
+* read-only music server with bpm and key scanning  
+  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+
+
+# client examples
+
+* javascript: dump some state into a file (two separate examples)
+  * `await fetch('https://127.0.0.1:3923/', {method:"PUT", body: JSON.stringify(foo)});`
+  * `var xhr = new XMLHttpRequest(); xhr.open('POST', 'https://127.0.0.1:3923/msgs?raw'); xhr.send('foo');`
+
+* curl/wget: upload some files (post=file, chunk=stdin)
+  * `post(){ curl -b cppwd=wark http://127.0.0.1:3923/ -F act=bput -F f=@"$1";}`  
+    `post movie.mkv`
+  * `post(){ wget --header='Cookie: cppwd=wark' http://127.0.0.1:3923/?raw --post-file="$1" -O-;}`  
+    `post movie.mkv`
+  * `chunk(){ curl -b cppwd=wark http://127.0.0.1:3923/ -T-;}`  
+    `chunk <movie.mkv`
+
+* FUSE: mount a copyparty server as a local filesystem
+  * cross-platform python client available in [./bin/](bin/)
+  * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
+
+copyparty returns a truncated sha512sum of your PUT/POST as base64; you can generate the same checksum locally to verify uplaods:
+
+    b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|head -c43;}
+    b512 <movie.mkv
 
 
 # dependencies
 
-* `jinja2`
-  * pulls in `markupsafe` as of v2.7; use jinja 2.6 on py3.2
+* `jinja2` (is built into the SFX)
 
-optional, enables thumbnails:
+**optional,** enables music tags:
+* either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
+* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
+
+**optional,** will eventually enable thumbnails:
 * `Pillow` (requires py2.7 or py3.5+)
 
 
+## optional gpl stuff
+
+some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
+
+these are standalone and will never be imported / evaluated by copyparty
+
+
+# sfx
+
+currently there are two self-contained binaries:
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
+
+launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
+
+pls note that `copyparty-sfx.sh` will fail if you rename `copyparty-sfx.py` to `copyparty.py` and keep it in the same folder because `sys.path` is funky
+
+
+## sfx repack
+
+if you don't need all the features you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except for either msys2 or WSL if you're on windows)
+* `724K` original size as of v0.4.0
+* `256K` after `./scripts/make-sfx.sh re no-ogv`
+* `164K` after `./scripts/make-sfx.sh re no-ogv no-cm`
+
+the features you can opt to drop are
+* `ogv`.js, the opus/vorbis decoder which is needed by apple devices to play foss audio files
+* `cm`/easymde, the "fancy" markdown editor
+
+for the `re`pack to work, first run one of the sfx'es once to unpack it
+
+**note:** you can also just download and run [scripts/copyparty-repack.sh](scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a `no-ogv no-cm` repack; works on linux/macos (and windows with msys2 or WSL)
+
+
 # install on android
 
 install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
 ```sh
-apt update && apt -y full-upgrade && termux-setup-storage && apt -y install curl && cd && curl -L https://github.com/9001/copyparty/raw/master/scripts/copyparty-android.sh > copyparty-android.sh && chmod 755 copyparty-android.sh && ./copyparty-android.sh -h
+apt update && apt -y full-upgrade && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install -U copyparty
 echo $?
 ```
 
-after the initial setup (and restarting bash), you can launch copyparty at any time by running "copyparty" in Termux
+after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
 
 
 # dev env setup
 
 ```sh
-python3 -m venv .env
-. .env/bin/activate
+python3 -m venv .venv
+. .venv/bin/activate
 pip install jinja2  # mandatory deps
 pip install Pillow  # thumbnail deps
 pip install black bandit pylint flake8  # vscode tooling
@@ -74,18 +293,25 @@ pip install black bandit pylint flake8  # vscode tooling
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies
+* `git tag v1.2.3 && git push origin --tags`
 * create github release with `make-tgz-release.sh`
 * upload to pypi with `make-pypi-release.(sh|bat)`
+* create sfx with `make-sfx.sh`
 
 
 # todo
 
 roughly sorted by priority
 
-* look into android thumbnail cache file format
+* reduce up2k roundtrips
+  * start from a chunk index and just go
+  * terminate client on bad data
+* drop onto folders
+* `os.copy_file_range` for up2k cloning
+* up2k partials ui
 * support pillow-simd
 * cache sha512 chunks on client
-* symlink existing files on upload
 * comment field
+* ~~look into android thumbnail cache file format~~ bad idea
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||

bin/README.md

@@ -0,0 +1,47 @@
+# [`copyparty-fuse.py`](copyparty-fuse.py)
+* mount a copyparty server as a local filesystem (read-only)
+* **supports Windows!** -- expect `194 MiB/s` sequential read
+* **supports Linux** -- expect `117 MiB/s` sequential read
+* **supports macos** -- expect `85 MiB/s` sequential read
+
+filecache is default-on for windows and macos;
+* macos readsize is 64kB, so speed ~32 MiB/s without the cache
+* windows readsize varies by software; explorer=1M, pv=32k
+
+note that copyparty should run with `-ed` to enable dotfiles (hidden otherwise)
+
+also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x performance
+
+
+## to run this on windows:
+* install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)
+  * [x] add python 3.x to PATH (it asks during install)
+* `python -m pip install --user fusepy`
+* `python ./copyparty-fuse.py n: http://192.168.1.69:3923/`
+
+10% faster in [msys2](https://www.msys2.org/), 700% faster if debug prints are enabled:
+* `pacman -S mingw64/mingw-w64-x86_64-python{,-pip}`
+* `/mingw64/bin/python3 -m pip install --user fusepy`
+* `/mingw64/bin/python3 ./copyparty-fuse.py [...]`
+
+you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releases/latest), let me know if you [figure out how](https://github.com/dokan-dev/dokany/wiki/FUSE)  
+(winfsp's sshfs leaks, doesn't look like winfsp itself does, should be fine)
+
+
+
+# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
+* mount a copyparty server as a local filesystem (read-only)
+* does the same thing except more correct, `samba` approves
+* **supports Linux** -- expect `18 MiB/s` (wait what)
+* **supports Macos** -- probably
+
+
+
+# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
+* pretend this doesn't exist
+
+
+
+# [`mtag/`](mtag/)
+* standalone programs which perform misc. file analysis
+* copyparty can Popen programs like these during file indexing to collect additional metadata

bin/copyparty-fuseb.py

@@ -0,0 +1,592 @@
+#!/usr/bin/env python3
+from __future__ import print_function, unicode_literals
+
+"""copyparty-fuseb: remote copyparty as a local filesystem"""
+__author__ = "ed <copyparty@ocv.me>"
+__copyright__ = 2020
+__license__ = "MIT"
+__url__ = "https://github.com/9001/copyparty/"
+
+import re
+import os
+import sys
+import time
+import stat
+import errno
+import struct
+import threading
+import http.client  # py2: httplib
+import urllib.parse
+from datetime import datetime
+from urllib.parse import quote_from_bytes as quote
+
+try:
+    import fuse
+    from fuse import Fuse
+
+    fuse.fuse_python_api = (0, 2)
+    if not hasattr(fuse, "__version__"):
+        raise Exception("your fuse-python is way old")
+except:
+    print(
+        "\n  could not import fuse; these may help:\n    python3 -m pip install --user fuse-python\n    apt install libfuse\n    modprobe fuse\n"
+    )
+    raise
+
+
+"""
+mount a copyparty server (local or remote) as a filesystem
+
+usage:
+  python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas
+
+dependencies:
+  sudo apk add fuse-dev python3-dev
+  python3 -m pip install --user fuse-python
+
+fork of copyparty-fuse.py based on fuse-python which
+  appears to be more compliant than fusepy? since this works with samba
+    (probably just my garbage code tbh)
+"""
+
+
+def threadless_log(msg):
+    print(msg + "\n", end="")
+
+
+def boring_log(msg):
+    msg = "\033[36m{:012x}\033[0m {}\n".format(threading.current_thread().ident, msg)
+    print(msg[4:], end="")
+
+
+def rice_tid():
+    tid = threading.current_thread().ident
+    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
+
+
+def fancy_log(msg):
+    print("{} {}\n".format(rice_tid(), msg), end="")
+
+
+def null_log(msg):
+    pass
+
+
+info = fancy_log
+log = fancy_log
+dbg = fancy_log
+log = null_log
+dbg = null_log
+
+
+def get_tid():
+    return threading.current_thread().ident
+
+
+def html_dec(txt):
+    return (
+        txt.replace("&lt;", "<")
+        .replace("&gt;", ">")
+        .replace("&quot;", '"')
+        .replace("&amp;", "&")
+    )
+
+
+class CacheNode(object):
+    def __init__(self, tag, data):
+        self.tag = tag
+        self.data = data
+        self.ts = time.time()
+
+
+class Stat(fuse.Stat):
+    def __init__(self):
+        self.st_mode = 0
+        self.st_ino = 0
+        self.st_dev = 0
+        self.st_nlink = 1
+        self.st_uid = 1000
+        self.st_gid = 1000
+        self.st_size = 0
+        self.st_atime = 0
+        self.st_mtime = 0
+        self.st_ctime = 0
+
+
+class Gateway(object):
+    def __init__(self, base_url):
+        self.base_url = base_url
+
+        ui = urllib.parse.urlparse(base_url)
+        self.web_root = ui.path.strip("/")
+        try:
+            self.web_host, self.web_port = ui.netloc.split(":")
+            self.web_port = int(self.web_port)
+        except:
+            self.web_host = ui.netloc
+            if ui.scheme == "http":
+                self.web_port = 80
+            elif ui.scheme == "https":
+                raise Exception("todo")
+            else:
+                raise Exception("bad url?")
+
+        self.conns = {}
+
+    def quotep(self, path):
+        # TODO: mojibake support
+        path = path.encode("utf-8", "ignore")
+        return quote(path, safe="/")
+
+    def getconn(self, tid=None):
+        tid = tid or get_tid()
+        try:
+            return self.conns[tid]
+        except:
+            info("new conn [{}] [{}]".format(self.web_host, self.web_port))
+
+            conn = http.client.HTTPConnection(self.web_host, self.web_port, timeout=260)
+
+            self.conns[tid] = conn
+            return conn
+
+    def closeconn(self, tid=None):
+        tid = tid or get_tid()
+        try:
+            self.conns[tid].close()
+            del self.conns[tid]
+        except:
+            pass
+
+    def sendreq(self, *args, **kwargs):
+        tid = get_tid()
+        try:
+            c = self.getconn(tid)
+            c.request(*list(args), **kwargs)
+            return c.getresponse()
+        except:
+            self.closeconn(tid)
+            c = self.getconn(tid)
+            c.request(*list(args), **kwargs)
+            return c.getresponse()
+
+    def listdir(self, path):
+        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
+        r = self.sendreq("GET", web_path)
+        if r.status != 200:
+            self.closeconn()
+            raise Exception(
+                "http error {} reading dir {} in {}".format(
+                    r.status, web_path, rice_tid()
+                )
+            )
+
+        return self.parse_html(r)
+
+    def download_file_range(self, path, ofs1, ofs2):
+        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
+        hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
+        log("downloading {}".format(hdr_range))
+
+        r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        if r.status != http.client.PARTIAL_CONTENT:
+            self.closeconn()
+            raise Exception(
+                "http error {} reading file {} range {} in {}".format(
+                    r.status, web_path, hdr_range, rice_tid()
+                )
+            )
+
+        return r.read()
+
+    def parse_html(self, datasrc):
+        ret = []
+        remainder = b""
+        ptn = re.compile(
+            r"^<tr><td>(-|DIR)</td><td><a [^>]+>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$"
+        )
+
+        while True:
+            buf = remainder + datasrc.read(4096)
+            # print('[{}]'.format(buf.decode('utf-8')))
+            if not buf:
+                break
+
+            remainder = b""
+            endpos = buf.rfind(b"\n")
+            if endpos >= 0:
+                remainder = buf[endpos + 1 :]
+                buf = buf[:endpos]
+
+            lines = buf.decode("utf-8").split("\n")
+            for line in lines:
+                m = ptn.match(line)
+                if not m:
+                    # print(line)
+                    continue
+
+                ftype, fname, fsize, fdate = m.groups()
+                fname = html_dec(fname)
+                ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
+                sz = int(fsize)
+                if ftype == "-":
+                    ret.append([fname, self.stat_file(ts, sz), 0])
+                else:
+                    ret.append([fname, self.stat_dir(ts, sz), 0])
+
+        return ret
+
+    def stat_dir(self, ts, sz=4096):
+        ret = Stat()
+        ret.st_mode = stat.S_IFDIR | 0o555
+        ret.st_nlink = 2
+        ret.st_size = sz
+        ret.st_atime = ts
+        ret.st_mtime = ts
+        ret.st_ctime = ts
+        return ret
+
+    def stat_file(self, ts, sz):
+        ret = Stat()
+        ret.st_mode = stat.S_IFREG | 0o444
+        ret.st_size = sz
+        ret.st_atime = ts
+        ret.st_mtime = ts
+        ret.st_ctime = ts
+        return ret
+
+
+class CPPF(Fuse):
+    def __init__(self, *args, **kwargs):
+        Fuse.__init__(self, *args, **kwargs)
+
+        self.url = None
+
+        self.dircache = []
+        self.dircache_mtx = threading.Lock()
+
+        self.filecache = []
+        self.filecache_mtx = threading.Lock()
+
+    def init2(self):
+        # TODO figure out how python-fuse wanted this to go
+        self.gw = Gateway(self.url)  # .decode('utf-8'))
+        info("up")
+
+    def clean_dircache(self):
+        """not threadsafe"""
+        now = time.time()
+        cutoff = 0
+        for cn in self.dircache:
+            if now - cn.ts > 1:
+                cutoff += 1
+            else:
+                break
+
+        if cutoff > 0:
+            self.dircache = self.dircache[cutoff:]
+
+    def get_cached_dir(self, dirpath):
+        # with self.dircache_mtx:
+        if True:
+            self.clean_dircache()
+            for cn in self.dircache:
+                if cn.tag == dirpath:
+                    return cn
+
+        return None
+
+    """
+            ,-------------------------------,  g1>=c1, g2<=c2
+            |cache1                   cache2|  buf[g1-c1:(g1-c1)+(g2-g1)]
+            `-------------------------------'
+                    ,---------------,
+                    |get1       get2|
+                    `---------------'
+    __________________________________________________________________________
+
+            ,-------------------------------,  g2<=c2, (g2>=c1)
+            |cache1                   cache2|  cdr=buf[:g2-c1]
+            `-------------------------------'  dl car; g1-512K:c1
+    ,---------------,
+    |get1       get2|
+    `---------------'
+    __________________________________________________________________________
+
+            ,-------------------------------,  g1>=c1, (g1<=c2)
+            |cache1                   cache2|  car=buf[c2-g1:]
+            `-------------------------------'  dl cdr; c2:c2+1M
+                                    ,---------------,
+                                    |get1       get2|
+                                    `---------------'
+    """
+
+    def get_cached_file(self, path, get1, get2, file_sz):
+        car = None
+        cdr = None
+        ncn = -1
+        # with self.filecache_mtx:
+        if True:
+            dbg("cache request from {} to {}, size {}".format(get1, get2, file_sz))
+            for cn in self.filecache:
+                ncn += 1
+
+                cache_path, cache1 = cn.tag
+                if cache_path != path:
+                    continue
+
+                cache2 = cache1 + len(cn.data)
+                if get2 <= cache1 or get1 >= cache2:
+                    continue
+
+                if get1 >= cache1 and get2 <= cache2:
+                    # keep cache entry alive by moving it to the end
+                    self.filecache = (
+                        self.filecache[:ncn] + self.filecache[ncn + 1 :] + [cn]
+                    )
+                    buf_ofs = get1 - cache1
+                    buf_end = buf_ofs + (get2 - get1)
+                    dbg(
+                        "found all ({}, {} to {}, len {}) [{}:{}] = {}".format(
+                            ncn,
+                            cache1,
+                            cache2,
+                            len(cn.data),
+                            buf_ofs,
+                            buf_end,
+                            buf_end - buf_ofs,
+                        )
+                    )
+                    return cn.data[buf_ofs:buf_end]
+
+                if get2 < cache2:
+                    x = cn.data[: get2 - cache1]
+                    if not cdr or len(cdr) < len(x):
+                        dbg(
+                            "found car ({}, {} to {}, len {}) [:{}-{}] = [:{}] = {}".format(
+                                ncn,
+                                cache1,
+                                cache2,
+                                len(cn.data),
+                                get2,
+                                cache1,
+                                get2 - cache1,
+                                len(x),
+                            )
+                        )
+                        cdr = x
+
+                    continue
+
+                if get1 > cache1:
+                    x = cn.data[-(cache2 - get1) :]
+                    if not car or len(car) < len(x):
+                        dbg(
+                            "found cdr ({}, {} to {}, len {}) [-({}-{}):] = [-{}:] = {}".format(
+                                ncn,
+                                cache1,
+                                cache2,
+                                len(cn.data),
+                                cache2,
+                                get1,
+                                cache2 - get1,
+                                len(x),
+                            )
+                        )
+                        car = x
+
+                    continue
+
+                raise Exception("what")
+
+        if car and cdr:
+            dbg("<cache> have both")
+
+            ret = car + cdr
+            if len(ret) == get2 - get1:
+                return ret
+
+            raise Exception("{} + {} != {} - {}".format(len(car), len(cdr), get2, get1))
+
+        elif cdr:
+            h_end = get1 + (get2 - get1) - len(cdr)
+            h_ofs = h_end - 512 * 1024
+
+            if h_ofs < 0:
+                h_ofs = 0
+
+            buf_ofs = (get2 - get1) - len(cdr)
+
+            dbg(
+                "<cache> cdr {}, car {}-{}={} [-{}:]".format(
+                    len(cdr), h_ofs, h_end, h_end - h_ofs, buf_ofs
+                )
+            )
+
+            buf = self.gw.download_file_range(path, h_ofs, h_end)
+            ret = buf[-buf_ofs:] + cdr
+
+        elif car:
+            h_ofs = get1 + len(car)
+            h_end = h_ofs + 1024 * 1024
+
+            if h_end > file_sz:
+                h_end = file_sz
+
+            buf_ofs = (get2 - get1) - len(car)
+
+            dbg(
+                "<cache> car {}, cdr {}-{}={} [:{}]".format(
+                    len(car), h_ofs, h_end, h_end - h_ofs, buf_ofs
+                )
+            )
+
+            buf = self.gw.download_file_range(path, h_ofs, h_end)
+            ret = car + buf[:buf_ofs]
+
+        else:
+            h_ofs = get1 - 256 * 1024
+            h_end = get2 + 1024 * 1024
+
+            if h_ofs < 0:
+                h_ofs = 0
+
+            if h_end > file_sz:
+                h_end = file_sz
+
+            buf_ofs = get1 - h_ofs
+            buf_end = buf_ofs + get2 - get1
+
+            dbg(
+                "<cache> {}-{}={} [{}:{}]".format(
+                    h_ofs, h_end, h_end - h_ofs, buf_ofs, buf_end
+                )
+            )
+
+            buf = self.gw.download_file_range(path, h_ofs, h_end)
+            ret = buf[buf_ofs:buf_end]
+
+        cn = CacheNode([path, h_ofs], buf)
+        # with self.filecache_mtx:
+        if True:
+            if len(self.filecache) > 6:
+                self.filecache = self.filecache[1:] + [cn]
+            else:
+                self.filecache.append(cn)
+
+        return ret
+
+    def _readdir(self, path):
+        path = path.strip("/")
+        log("readdir {}".format(path))
+
+        ret = self.gw.listdir(path)
+
+        # with self.dircache_mtx:
+        if True:
+            cn = CacheNode(path, ret)
+            self.dircache.append(cn)
+            self.clean_dircache()
+
+        return ret
+
+    def readdir(self, path, offset):
+        for e in self._readdir(path)[offset:]:
+            # log("yield [{}]".format(e[0]))
+            yield fuse.Direntry(e[0])
+
+    def open(self, path, flags):
+        if (flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)) != os.O_RDONLY:
+            return -errno.EACCES
+
+        st = self.getattr(path)
+        try:
+            if st.st_nlink > 0:
+                return st
+        except:
+            return st  # -int(os.errcode)
+
+    def read(self, path, length, offset, fh=None, *args):
+        if args:
+            log("unexpected args [" + "] [".join(repr(x) for x in args) + "]")
+            raise Exception()
+
+        path = path.strip("/")
+
+        ofs2 = offset + length
+        log("read {} @ {} len {} end {}".format(path, offset, length, ofs2))
+
+        st = self.getattr(path)
+        try:
+            file_sz = st.st_size
+        except:
+            return st  # -int(os.errcode)
+
+        if ofs2 > file_sz:
+            ofs2 = file_sz
+            log("truncate to len {} end {}".format(ofs2 - offset, ofs2))
+
+        if file_sz == 0 or offset >= ofs2:
+            return b""
+
+        # toggle cache here i suppose
+        # return self.get_cached_file(path, offset, ofs2, file_sz)
+        return self.gw.download_file_range(path, offset, ofs2)
+
+    def getattr(self, path):
+        log("getattr [{}]".format(path))
+
+        path = path.strip("/")
+        try:
+            dirpath, fname = path.rsplit("/", 1)
+        except:
+            dirpath = ""
+            fname = path
+
+        if not path:
+            ret = self.gw.stat_dir(time.time())
+            dbg("=root")
+            return ret
+
+        cn = self.get_cached_dir(dirpath)
+        if cn:
+            log("cache ok")
+            dents = cn.data
+        else:
+            log("cache miss")
+            dents = self._readdir(dirpath)
+
+        for cache_name, cache_stat, _ in dents:
+            if cache_name == fname:
+                dbg("=file")
+                return cache_stat
+
+        log("=404")
+        return -errno.ENOENT
+
+
+def main():
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+
+    server = CPPF()
+    server.parser.add_option(mountopt="url", metavar="BASE_URL", default=None)
+    server.parse(values=server, errex=1)
+    if not server.url or not str(server.url).startswith("http"):
+        print("\nerror:")
+        print("  need argument: -o url=<...>")
+        print("  need argument: mount-path")
+        print("example:")
+        print(
+            "  ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas"
+        )
+        sys.exit(1)
+
+    server.init2()
+    threading.Thread(target=server.main, daemon=True).start()
+    while True:
+        time.sleep(9001)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/README.md

@@ -0,0 +1,34 @@
+standalone programs which take an audio file as argument
+
+some of these rely on libraries which are not MIT-compatible
+
+* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
+* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
+
+
+# dependencies
+
+run [`install-deps.sh`](install-deps.sh) to build/install most dependencies required by these programs (supports windows/linux/macos)
+
+*alternatively* (or preferably) use packages from your distro instead, then you'll need at least these:
+
+* from distro: `numpy vamp-plugin-sdk beatroot-vamp mixxx-keyfinder ffmpeg`
+* from pypy: `keyfinder vamp`
+
+
+# usage from copyparty
+
+`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
+
+* `f,` makes the detected value replace any existing values
+* the `.` in `.bpm` indicates numeric value
+* assumes the python files are in the folder you're launching copyparty from, replace the filename with a relative/absolute path if that's not the case
+* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
+
+
+## usage with volume-flags
+
+instead of affecting all volumes, you can set the options for just one volume like so:
+```
+copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
+```

bin/mtag/audio-bpm.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import vamp
+import tempfile
+import numpy as np
+import subprocess as sp
+
+from copyparty.util import fsenc
+
+"""
+dep: vamp
+dep: beatroot-vamp
+dep: ffmpeg
+"""
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-ss", "13",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-ac", "1",
+        "-ar", "22050",
+        "-t", "300",
+        "-f", "f32le",
+        tf
+    ])
+    # fmt: on
+
+    with open(tf, "rb") as f:
+        d = np.fromfile(f, dtype=np.float32)
+        try:
+            # 98% accuracy on jcore
+            c = vamp.collect(d, 22050, "beatroot-vamp:beatroot")
+            cl = c["list"]
+        except:
+            # fallback; 73% accuracy
+            plug = "vamp-example-plugins:fixedtempo"
+            c = vamp.collect(d, 22050, plug, parameters={"maxdflen": 40})
+            print(c["list"][0]["label"].split(" ")[0])
+            return
+
+        # throws if detection failed:
+        bpm = float(cl[-1]["timestamp"] - cl[1]["timestamp"])
+        bpm = round(60 * ((len(cl) - 1) / bpm), 2)
+        print(f"{bpm:.2f}")
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import sys
+import keyfinder
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+
+note: cannot fsenc
+"""
+
+
+try:
+    print(keyfinder.key(sys.argv[1]).camelot())
+except:
+    pass

bin/mtag/install-deps.sh

@@ -0,0 +1,265 @@
+#!/bin/bash
+set -e
+
+
+# install dependencies for audio-*.py
+#
+# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
+# win64: requires msys2-mingw64 environment
+# macos: requires macports
+#
+# has the following manual dependencies, especially on mac:
+#   https://www.vamp-plugins.org/pack.html
+#
+# installs stuff to the following locations:
+#   ~/pe/
+#   whatever your python uses for --user packages
+#
+# does the following terrible things:
+#   modifies the keyfinder python lib to load the .so in ~/pe
+
+
+linux=1
+
+win=
+[ ! -z "$MSYSTEM" ] || [ -e /msys2.exe ] && {
+	[ "$MSYSTEM" = MINGW64 ] || {
+		echo windows detected, msys2-mingw64 required
+		exit 1
+	}
+	pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	win=1
+	linux=
+}
+
+mac=
+[ $(uname -s) = Darwin ] && {
+	#pybin="$(printf '%s\n' /opt/local/bin/python* | (sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) | (sort -nr || cat) | (sed -E 's/([^ ]*) (.*)/\2\1/' || cat) | grep -E '/(python|pypy)[0-9\.-]*$' | head -n 1)"
+	pybin=/opt/local/bin/python3.9
+	[ -e "$pybin" ] || {
+		echo mac detected, python3 from macports required
+		exit 1
+	}
+	pkgs='ffmpeg python39 py39-wheel'
+	ninst=$(port installed | awk '/^  /{print$1}' | sort | uniq | grep -E '^('"$(echo "$pkgs" | tr ' ' '|')"')$' | wc -l)
+	[ $ninst -eq 3 ] || {
+		sudo port install $pkgs
+	}
+	mac=1
+	linux=
+}
+
+hash -r
+
+[ $mac ] || {
+	command -v python3 && pybin=python3 || pybin=python
+}
+
+$pybin -m pip install --user numpy
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+
+
+need() {
+	command -v $1 >/dev/null || {
+		echo need $1
+		exit 1
+	}
+}
+need cmake
+need ffmpeg
+need $pybin
+#need patchelf
+
+
+td="$(mktemp -d)"
+cln() {
+	rm -rf "$td"
+}
+trap cln EXIT
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	local yolo= ex=
+	[ $1 = "yolo" ] && yolo=1 && ex=k && shift
+	command -v curl >/dev/null && exec curl -${ex}JOL "$@"
+	
+	[ $yolo ] && ex=--no-check-certificate
+	exec wget --trust-server-names $ex "$@"
+}
+export -f dl_files
+
+
+github_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.tarball_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"tarball_url": "/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+gitlab_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.[0].assets.sources[]|select(.format|test("tar.gz")).url' ||
+
+		# fallback to abomination
+		tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	tee links |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+install_keyfinder() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
+	
+	github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
+
+	tar -xf mixxxdj-libkeyfinder-*
+	rm -- *.tar.gz
+	cd mixxxdj-libkeyfinder*
+	
+	h="$HOME"
+	so="lib/libkeyfinder.so"
+	memes=()
+
+	[ $win ] &&
+		so="bin/libkeyfinder.dll" &&
+		h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
+		memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
+	
+	[ $mac ] &&
+		so="lib/libkeyfinder.dylib"
+
+	cmake -DCMAKE_INSTALL_PREFIX="$h/pe/keyfinder" "${memes[@]}" -S . -B build
+	cmake --build build --parallel $(nproc || echo 4)
+	cmake --install build
+
+	libpath="$h/pe/keyfinder/$so"
+	[ $linux ] && [ ! -e "$libpath" ] &&
+		so=lib64/libkeyfinder.so
+	
+	libpath="$h/pe/keyfinder/$so"
+	[ -e "$libpath" ] || {
+		echo "so not found at $sop"
+		exit 1
+	}
+	
+	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
+	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
+	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
+	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
+	$pybin -m pip install --user keyfinder
+
+	pypath="$($pybin -c 'import keyfinder; print(keyfinder.__file__)')"
+	for pyso in "${pypath%/*}"/*.so; do
+		[ -e "$pyso" ] || break
+		patchelf --set-rpath "${libpath%/*}" "$pyso" ||
+			echo "WARNING: patchelf failed (only fatal on musl-based distros)"
+	done
+	
+	mv "$pypath"{,.bak}
+	(
+		printf 'import ctypes\nctypes.cdll.LoadLibrary("%s")\n' "$libpath"
+		cat "$pypath.bak"
+	) >"$pypath"
+
+	echo
+	echo libkeyfinder successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+have_beatroot() {
+	$pybin -c 'import vampyhost, sys; plugs = vampyhost.list_plugins(); sys.exit(0 if "beatroot-vamp:beatroot" in plugs else 1)'
+}
+
+
+install_vamp() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	
+	$pybin -m pip install --user vamp
+
+	have_beatroot || {
+		printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
+		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
+		sha512sum -c <(
+			echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874  -"
+		) <beatroot-vamp-v1.0.tar.gz
+		tar -xf beatroot-vamp-v1.0.tar.gz 
+		cd beatroot-vamp-v1.0
+		make -f Makefile.linux -j4
+		# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
+		mkdir ~/vamp
+		cp -pv beatroot-vamp.* ~/vamp/
+	}
+	
+	have_beatroot &&
+		printf '\033[32mfound the vamp beatroot plugin, nice\033[0m\n' ||
+		printf '\033[31mWARNING: could not find the vamp beatroot plugin, please install it for optimal results\033[0m\n'
+}
+
+
+# not in use because it kinda segfaults, also no windows support
+install_soundtouch() {
+	gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
+	
+	tar -xvf soundtouch-*
+	rm -- *.tar.gz
+	cd soundtouch-*
+	
+	# https://github.com/jrising/pysoundtouch
+	./bootstrap
+	./configure --enable-integer-samples CXXFLAGS="-fPIC" --prefix="$HOME/pe/soundtouch"
+	make -j$(nproc || echo 4)
+	make install
+	
+	CFLAGS=-I$HOME/pe/soundtouch/include/ \
+	LDFLAGS=-L$HOME/pe/soundtouch/lib \
+	$pybin -m pip install --user git+https://github.com/snowxmas/pysoundtouch.git
+	
+	pypath="$($pybin -c 'import importlib; print(importlib.util.find_spec("soundtouch").origin)')"
+	libpath="$(echo "$HOME/pe/soundtouch/lib/")"
+	patchelf --set-rpath "$libpath" "$pypath"
+
+	echo
+	echo soundtouch successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+[ "$1" = keyfinder ] && { install_keyfinder; exit $?; }
+[ "$1" = soundtouch ] && { install_soundtouch; exit $?; }
+[ "$1" = vamp ] && { install_vamp; exit $?; }
+
+echo no args provided, installing keyfinder and vamp
+install_keyfinder
+install_vamp

bin/mtag/sleep.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+
+import time
+import random
+
+v = random.random() * 6
+time.sleep(v)
+print(f"{v:.2f}")

bin/up2k.sh

@@ -118,7 +118,7 @@ printf ']}' >> /dev/shm/$salt.hs
 
 printf '\033[36m'
 
-#curl "http://$target:1234$posturl/handshake.php" -H "Content-Type: text/plain;charset=UTF-8" -H "Cookie: cppwd=$passwd" --data "$(cat "/dev/shm/$salt.hs")" | tee /dev/shm/$salt.res
+#curl "http://$target:3923$posturl/handshake.php" -H "Content-Type: text/plain;charset=UTF-8" -H "Cookie: cppwd=$passwd" --data "$(cat "/dev/shm/$salt.hs")" | tee /dev/shm/$salt.res
 
 {
     {
@@ -135,7 +135,7 @@ EOF
     cat /dev/shm/$salt.hs
 } |
 tee /dev/shm/$salt.hsb |
-ncat $target 1234 |
+ncat $target 3923 |
 tee /dev/shm/$salt.hs1r
 
 wark="$(cat /dev/shm/$salt.hs1r | getwark)"
@@ -190,7 +190,7 @@ EOF
     nchunk=$((nchunk+1))
 
 done |
-ncat $target 1234 |
+ncat $target 3923 |
 tee /dev/shm/$salt.pr
 
 t=$(date +%s.%N)
@@ -201,7 +201,7 @@ t=$(date +%s.%N)
 
 printf '\033[36m'
 
-ncat $target 1234 < /dev/shm/$salt.hsb |
+ncat $target 3923 < /dev/shm/$salt.hsb |
 tee /dev/shm/$salt.hs2r |
 grep -E '"hash": ?\[ *\]'
 

contrib/README.md

@@ -0,0 +1,27 @@
+### [`copyparty.bat`](copyparty.bat)
+* launches copyparty with no arguments (anon read+write within same folder)
+* intended for windows machines with no python.exe in PATH
+* works on windows, linux and macos
+* assumes `copyparty-sfx.py` was renamed to `copyparty.py` in the same folder as `copyparty.bat`
+
+### [`index.html`](index.html)
+* drop-in redirect from an httpd to copyparty
+* assumes the webserver and copyparty is running on the same server/IP
+* modify `10.13.1.1` as necessary if you wish to support browsers without javascript
+
+### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
+* disables thumbnails and folder-type detection in windows explorer
+* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+
+### [`cfssl.sh`](cfssl.sh)
+* creates CA and server certificates using cfssl
+* give a 3rd argument to install it to your copyparty config
+
+# OS integration
+init-scripts to start copyparty as a service
+* [`systemd/copyparty.service`](systemd/copyparty.service)
+* [`openrc/copyparty`](openrc/copyparty)
+
+# Reverse-proxy
+copyparty has basic support for running behind another webserver
+* [`nginx/copyparty.conf`](nginx/copyparty.conf)

contrib/cfssl.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e
+
+# ca-name and server-name
+ca_name="$1"
+srv_name="$2"
+
+[ -z "$srv_name" ] && {
+	echo "need arg 1: ca name"
+	echo "need arg 2: server name"
+	exit 1
+}
+
+
+gen_ca() {
+	(tee /dev/stderr <<EOF
+{"CN": "$ca_name ca",
+"CA": {"expiry":"87600h", "pathlen":0},
+"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name ca"}]}
+EOF
+	)|
+	cfssl gencert -initca - |
+	cfssljson -bare ca
+	
+	mv ca-key.pem ca.key
+	rm ca.csr
+}
+
+
+gen_srv() {
+	(tee /dev/stderr <<EOF
+{"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name - $srv_name"}]}
+EOF
+	)|
+	cfssl gencert -ca ca.pem -ca-key ca.key \
+		-profile=www -hostname="$srv_name.$ca_name" - |
+	cfssljson -bare "$srv_name"
+
+	mv "$srv_name-key.pem" "$srv_name.key"
+	rm "$srv_name.csr"
+}
+
+
+# create ca if not exist
+[ -e ca.key ] ||
+	gen_ca
+
+# always create server cert
+gen_srv
+
+
+# dump cert info
+show() {
+	openssl x509 -text -noout -in $1 |
+	awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
+}
+show ca.pem
+show "$srv_name.pem"
+
+
+# write cert into copyparty config
+[ -z "$3" ] || {
+	mkdir -p ~/.config/copyparty
+	cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
+}
+
+
+# rm *.key *.pem
+# cfssl print-defaults config
+# cfssl print-defaults csr

contrib/copyparty.bat

@@ -0,0 +1,33 @@
+exec python "$(dirname "$0")"/copyparty.py
+
+@rem on linux, the above will execute and the script will terminate
+@rem on windows, the rest of this script will run
+
+@echo off
+cls
+
+set py=
+for /f %%i in ('where python 2^>nul') do (
+    set "py=%%i"
+    goto c1
+)
+:c1
+
+if [%py%] == [] (
+    for /f %%i in ('where /r "%localappdata%\programs\python" python 2^>nul') do (
+        set "py=%%i"
+        goto c2
+    )
+)
+:c2
+
+if [%py%] == [] set "py=c:\python27\python.exe"
+
+if not exist "%py%" (
+    echo could not find python
+    echo(
+    pause
+    exit /b
+)
+
+start cmd /c %py% "%~dp0\copyparty.py"

contrib/explorer-nothumbs-nofoldertypes.reg

@@ -0,0 +1,31 @@
+Windows Registry Editor Version 5.00
+
+; this will do 3 things, all optional:
+;  1) disable thumbnails
+;  2) delete all existing folder type settings/detections
+;  3) disable folder type detection (force default columns)
+;
+; this makes the file explorer way faster,
+; especially on slow/networked locations
+
+
+; =====================================================================
+; 1) disable thumbnails
+
+[HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced]
+"IconsOnly"=dword:00000001
+
+
+; =====================================================================
+; 2) delete all existing folder type settings/detections
+
+[-HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\Bags]
+
+[-HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\BagMRU]
+
+
+; =====================================================================
+; 3) disable folder type detection
+
+[HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\Bags\AllFolders\Shell]
+"FolderType"="NotSpecified"

contrib/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>⇆🎉 redirect</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<style>
+
+html, body {
+	font-family: sans-serif;
+}
+body {
+	padding: 1em 2em;
+	font-size: 1.5em;
+}
+a {
+	font-size: 1.2em;
+	padding: .1em;
+}
+
+</style>
+</head>
+<body>
+	<span id="desc">you probably want</span> <a id="redir" href="//10.13.1.1:3923/">copyparty</a>
+	<script>
+
+var a = document.getElementById('redir'),
+	proto = window.location.protocol.indexOf('https') === 0 ? 'https' : 'http',
+	loc = window.location.hostname || '127.0.0.1',
+	port = a.getAttribute('href').split(':').pop().split('/')[0],
+	url = proto + '://' + loc + ':' + port + '/';
+
+a.setAttribute('href', url);
+document.getElementById('desc').innerHTML = 'redirecting to';
+
+setTimeout(function() {
+	window.location.href = url;
+}, 500);
+
+</script>
+</body>
+</html>

contrib/nginx/copyparty.conf

@@ -0,0 +1,26 @@
+upstream cpp {
+	server 127.0.0.1:3923;
+	keepalive 120;
+}
+server {
+	listen 443 ssl;
+	listen [::]:443 ssl;
+
+	server_name fs.example.com;
+	
+	location / {
+		proxy_pass http://cpp;
+		proxy_redirect off;
+		# disable buffering (next 4 lines)
+		proxy_http_version 1.1;
+		client_max_body_size 0;
+		proxy_buffering off;
+		proxy_request_buffering off;
+
+		proxy_set_header   Host              $host;
+		proxy_set_header   X-Real-IP         $remote_addr;
+		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+		proxy_set_header   X-Forwarded-Proto $scheme;
+		proxy_set_header   Connection        "Keep-Alive";
+	}
+}

contrib/openrc/copyparty

@@ -0,0 +1,18 @@
+#!/sbin/openrc-run
+
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   cp -pv copyparty /etc/init.d && rc-update add copyparty
+#
+# you may want to:
+#   change '/usr/bin/python' to another interpreter
+#   change '/mnt::a' to another location or permission-set
+
+name="$SVCNAME"
+command_background=true
+pidfile="/var/run/$SVCNAME.pid"
+
+command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
+command_args="-q -v /mnt::a"

contrib/systemd/copyparty.service

@@ -0,0 +1,19 @@
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
+#
+# you may want to:
+#   change '/usr/bin/python' to another interpreter
+#   change '/mnt::a' to another location or permission-set
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
+
+[Install]
+WantedBy=multi-user.target

copyparty/__init__.py

@@ -5,11 +5,19 @@ import platform
 import sys
 import os
 
-WINDOWS = platform.system() == "Windows"
 PY2 = sys.version_info[0] == 2
 if PY2:
     sys.dont_write_bytecode = True
 
+WINDOWS = False
+if platform.system() == "Windows":
+    WINDOWS = [int(x) for x in platform.version().split(".")]
+
+VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
+# introduced in anniversary update
+
+MACOS = platform.system() == "Darwin"
+
 
 class EnvParams(object):
     def __init__(self):
@@ -24,6 +32,7 @@ class EnvParams(object):
                 + "/copyparty"
             )
 
+        self.cfg = self.cfg.replace("\\", "/")
         try:
             os.makedirs(self.cfg)
         except:

copyparty/__main__.py

@@ -8,17 +8,29 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
+import re
 import os
+import sys
+import time
+import signal
 import shutil
 import filecmp
 import locale
 import argparse
+import threading
+import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS
-from .__version__ import S_VERSION, S_BUILD_DT
+from .__init__ import E, WINDOWS, VT100, PY2
+from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc
+from .util import py_desc, align_tab, IMPLICATIONS
+
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
 
 
 class RiceFormatter(argparse.HelpFormatter):
@@ -28,7 +40,7 @@ class RiceFormatter(argparse.HelpFormatter):
         except the help += [...] line now has colors
         """
         fmt = "\033[36m (default: \033[35m%(default)s\033[36m)\033[0m"
-        if WINDOWS:
+        if not VT100:
             fmt = " (default: %(default)s)"
 
         help = action.help
@@ -44,6 +56,10 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+def warn(msg):
+    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+
+
 def ensure_locale():
     for x in [
         "en_US.UTF-8",
@@ -84,12 +100,111 @@ def ensure_cert():
     # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
 
 
-def main():
-    f = "\033[36mcopyparty v{} ({})\n   python v{}\033[0m\n"
-    print(f.format(S_VERSION, S_BUILD_DT, py_desc()))
+def configure_ssl_ver(al):
+    def terse_sslver(txt):
+        txt = txt.lower()
+        for c in ["_", "v", "."]:
+            txt = txt.replace(c, "")
+
+        return txt.replace("tls10", "tls1")
+
+    # oh man i love openssl
+    # check this out
+    # hold my beer
+    ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
+    sslver = terse_sslver(al.ssl_ver).split(",")
+    flags = [k for k in ssl.__dict__ if ptn.match(k)]
+    # SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
+    if "help" in sslver:
+        avail = [terse_sslver(x[6:]) for x in flags]
+        avail = " ".join(sorted(avail) + ["all"])
+        print("\navailable ssl/tls versions:\n  " + avail)
+        sys.exit(0)
+
+    al.ssl_flags_en = 0
+    al.ssl_flags_de = 0
+    for flag in sorted(flags):
+        ver = terse_sslver(flag[6:])
+        num = getattr(ssl, flag)
+        if ver in sslver:
+            al.ssl_flags_en |= num
+        else:
+            al.ssl_flags_de |= num
+
+    if sslver == ["all"]:
+        x = al.ssl_flags_en
+        al.ssl_flags_en = al.ssl_flags_de
+        al.ssl_flags_de = x
+
+    for k in ["ssl_flags_en", "ssl_flags_de"]:
+        num = getattr(al, k)
+        print("{}: {:8x} ({})".format(k, num, num))
+
+    # think i need that beer now
+
+
+def configure_ssl_ciphers(al):
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    if al.ssl_ver:
+        ctx.options &= ~al.ssl_flags_en
+        ctx.options |= al.ssl_flags_de
+
+    is_help = al.ciphers == "help"
+
+    if al.ciphers and not is_help:
+        try:
+            ctx.set_ciphers(al.ciphers)
+        except:
+            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+
+    if not hasattr(ctx, "get_ciphers"):
+        print("cannot read cipher list: openssl or python too old")
+    else:
+        ciphers = [x["description"] for x in ctx.get_ciphers()]
+        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+
+    if is_help:
+        sys.exit(0)
+
+
+def sighandler(signal=None, frame=None):
+    msg = [""] * 5
+    for th in threading.enumerate():
+        msg.append(str(th))
+        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
+
+    msg.append("\n")
+    print("\n".join(msg))
+
+
+def main(argv=None):
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if WINDOWS:
+        os.system("rem")  # enables colors
+
+    if argv is None:
+        argv = sys.argv
+
+    desc = py_desc().replace("[", "\033[1;30m[")
+
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
+    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
-    ensure_cert()
+    if HAVE_SSL:
+        ensure_cert()
+
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        argv[idx] = nk
+        time.sleep(2)
 
     ap = argparse.ArgumentParser(
         formatter_class=RiceFormatter,
@@ -98,38 +213,120 @@ def main():
         epilog=dedent(
             """
             -a takes username:password,
-            -v takes src:dst:permset:permset:... where "permset" is
-               accesslevel followed by username (no separator)
+            -v takes src:dst:permset:permset:cflag:cflag:...
+               where "permset" is accesslevel followed by username (no separator)
+               and "cflag" is config flags to set on this volume
             
+            list of cflags:
+              "cnodupe" rejects existing files (instead of symlinking them)
+
             example:\033[35m
-              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed  \033[36m
+              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
               mount current directory at "/" with
                * r (read-only) for everyone
                * a (read+write) for ed
               mount ../inc at "/dump" with
                * w (write-only) for everyone
-               * a (read+write) for ed  \033[0m
+               * a (read+write) for ed
+               * reject duplicate files  \033[0m
             
             if no accounts or volumes are configured,
             current folder will be read/write for everyone
 
             consider the config file for more flexible account/volume management,
             including dynamic reload at runtime (and being more readable w)
+
+            values for --urlform:
+              "stash" dumps the data to file and returns length + checksum
+              "save,get" dumps to file and returns the page like a GET
+              "print,get" prints the data in the log and returns GET
+              (leave out the ",get" to return an error instead)
+
+            --ciphers help = available ssl/tls ciphers,
+            --ssl-ver help = available ssl/tls versions,
+              default is what python considers safe, usually >= TLS1
             """
         ),
     )
-    ap.add_argument(
-        "-c", metavar="PATH", type=str, action="append", help="add config file"
-    )
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind")
-    ap.add_argument("-p", metavar="PORT", type=int, default=1234, help="port to bind")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=16, help="max num clients")
-    ap.add_argument("-j", metavar="CORES", type=int, help="max num cpu cores")
+    # fmt: off
+    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
+    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
+    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
     ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
     ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
     ap.add_argument("-q", action="store_true", help="quiet")
-    ap.add_argument("-nw", action="store_true", help="benchmark: disable writing")
-    al = ap.parse_args()
+    ap.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
+    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
+    ap.add_argument("-nih", action="store_true", help="no info hostname")
+    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
+    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
+    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
+    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+
+    ap2 = ap.add_argument_group('database options')
+    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
+    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
+    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
+    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
+    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
+    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
+    ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
+    ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
+    ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
+
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="ssl/tls versions to allow")
+    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+    
+    al = ap.parse_args(args=argv[1:])
+    # fmt: on
+
+    # propagate implications
+    for k1, k2 in IMPLICATIONS:
+        if getattr(al, k1):
+            setattr(al, k2, True)
+
+    al.i = al.i.split(",")
+    try:
+        if "-" in al.p:
+            lo, hi = [int(x) for x in al.p.split("-")]
+            al.p = list(range(lo, hi + 1))
+        else:
+            al.p = [int(x) for x in al.p.split(",")]
+    except:
+        raise Exception("invalid value for -p")
+
+    if HAVE_SSL:
+        if al.ssl_ver:
+            configure_ssl_ver(al)
+
+        if al.ciphers:
+            configure_ssl_ciphers(al)
+    else:
+        warn("ssl module does not exist; cannot enable https")
+
+    if PY2 and WINDOWS and al.e2d:
+        warn(
+            "windows py2 cannot do unicode filenames with -e2d\n"
+            + "  (if you crash with codec errors then that is why)"
+        )
+
+    # signal.signal(signal.SIGINT, sighandler)
 
     SvcHub(al).run()
 

copyparty/__version__.py

@@ -1,7 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 2, 3)
-BUILD_DT = (2020, 1, 19)
+VERSION = (0, 10, 4)
+CODENAME = "zip it"
+BUILD_DT = (2021, 3, 29)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -1,23 +1,39 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
 import re
+import os
+import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
-from .util import undot, Pebkac, fsdec, fsenc
+from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint
 
 
 class VFS(object):
     """single level in the virtual fs"""
 
-    def __init__(self, realpath, vpath, uread=[], uwrite=[]):
+    def __init__(self, realpath, vpath, uread=[], uwrite=[], flags={}):
         self.realpath = realpath  # absolute path on host filesystem
         self.vpath = vpath  # absolute path in the virtual filesystem
         self.uread = uread  # users who can read this
         self.uwrite = uwrite  # users who can write this
+        self.flags = flags  # config switches
         self.nodes = {}  # child nodes
+        self.all_vols = {vpath: self}  # flattened recursive
+
+    def __repr__(self):
+        return "VFS({})".format(
+            ", ".join(
+                "{}={!r}".format(k, self.__dict__[k])
+                for k in "realpath vpath uread uwrite flags".split()
+            )
+        )
+
+    def _trk(self, vol):
+        self.all_vols[vol.vpath] = vol
+        return vol
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""
@@ -29,16 +45,18 @@ class VFS(object):
             name, dst = dst.split("/", 1)
             if name in self.nodes:
                 # exists; do not manipulate permissions
-                return self.nodes[name].add(src, dst)
+                return self._trk(self.nodes[name].add(src, dst))
 
             vn = VFS(
                 "{}/{}".format(self.realpath, name),
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
                 self.uwrite,
+                self.flags,
             )
+            self._trk(vn)
             self.nodes[name] = vn
-            return vn.add(src, dst)
+            return self._trk(vn.add(src, dst))
 
         if dst in self.nodes:
             # leaf exists; return as-is
@@ -48,7 +66,7 @@ class VFS(object):
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
         self.nodes[dst] = vn
-        return vn
+        return self._trk(vn)
 
     def _find(self, vpath):
         """return [vfs,remainder]"""
@@ -95,29 +113,89 @@ class VFS(object):
 
         return fsdec(os.path.realpath(fsenc(rp)))
 
-    def ls(self, rem, uname):
+    def ls(self, rem, uname, scandir, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
+        virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
-        real = [fsdec(x) for x in os.listdir(fsenc(abspath))]
+        real = list(statdir(nuprint, scandir, lstat, abspath))
         real.sort()
-        if rem:
-            virt_vis = []
-        else:
-            virt_all = []  # all nodes that exist
-            virt_vis = []  # nodes readable by user
+        if not rem:
             for name, vn2 in sorted(self.nodes.items()):
-                virt_all.append(name)
-                if uname in vn2.uread:
-                    virt_vis.append(name)
+                if uname in vn2.uread or "*" in vn2.uread:
+                    virt_vis[name] = vn2
 
-            for name in virt_all:
-                try:
-                    real.remove(name)
-                except ValueError:
-                    pass
+            # no vfs nodes in the list of real inodes
+            real = [x for x in real if x[0] not in self.nodes]
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+                rd = [x for x in rd if x[0] in flt]
+                vd = {x: y for x, y in vd.items() if x in flt}
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)
@@ -134,22 +212,21 @@ class VFS(object):
 class AuthSrv(object):
     """verifies users against given paths"""
 
-    def __init__(self, args, log_func):
-        self.log_func = log_func
+    def __init__(self, args, log_func, warn_anonwrite=True):
         self.args = args
-
-        self.warn_anonwrite = True
+        self.log_func = log_func
+        self.warn_anonwrite = warn_anonwrite
 
         if WINDOWS:
-            self.re_vol = re.compile(r'^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)')
+            self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
         else:
-            self.re_vol = re.compile(r'^([^:]*):([^:]*):(.*)')
+            self.re_vol = re.compile(r"^([^:]*):([^:]*):(.*)$")
 
         self.mutex = threading.Lock()
         self.reload()
 
-    def log(self, msg):
-        self.log_func("auth", msg)
+    def log(self, msg, c=0):
+        self.log_func("auth", msg, c)
 
     def invert(self, orig):
         if PY2:
@@ -167,7 +244,7 @@ class AuthSrv(object):
 
         yield prev, True
 
-    def _parse_config_file(self, fd, user, mread, mwrite, mount):
+    def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount):
         vol_src = None
         vol_dst = None
         for ln in [x.decode("utf-8").strip() for x in fd]:
@@ -197,13 +274,43 @@ class AuthSrv(object):
                 mount[vol_dst] = vol_src
                 mread[vol_dst] = []
                 mwrite[vol_dst] = []
+                mflags[vol_dst] = {}
                 continue
 
             lvl, uname = ln.split(" ")
-            if lvl in "ra":
-                mread[vol_dst].append(uname)
-            if lvl in "wa":
-                mwrite[vol_dst].append(uname)
+            self._read_vol_str(
+                lvl, uname, mread[vol_dst], mwrite[vol_dst], mflags[vol_dst]
+            )
+
+    def _read_vol_str(self, lvl, uname, mr, mw, mf):
+        if lvl == "c":
+            cval = True
+            if "=" in uname:
+                uname, cval = uname.split("=", 1)
+
+            self._read_volflag(mf, uname, cval, False)
+            return
+
+        if uname == "":
+            uname = "*"
+
+        if lvl in "ra":
+            mr.append(uname)
+
+        if lvl in "wa":
+            mw.append(uname)
+
+    def _read_volflag(self, flags, name, value, is_list):
+        if name not in ["mtp"]:
+            flags[name] = value
+            return
+
+        if not is_list:
+            value = [value]
+        elif not value:
+            return
+
+        flags[name] = flags.get(name, []) + value
 
     def reload(self):
         """
@@ -216,6 +323,7 @@ class AuthSrv(object):
         user = {}  # username:password
         mread = {}  # mountpoint:[username]
         mwrite = {}  # mountpoint:[username]
+        mflags = {}  # mountpoint:[flag]
         mount = {}  # dst:src (mountpoint:realpath)
 
         if self.args.a:
@@ -225,39 +333,37 @@ class AuthSrv(object):
 
         if self.args.v:
             # list of src:dst:permset:permset:...
-            # permset is [rwa]username
-            for vol_match in [self.re_vol.match(x) for x in self.args.v]:
-                try:
-                    src, dst, perms = vol_match.groups()
-                except:
-                    raise Exception('invalid -v argument')
+            # permset is [rwa]username or [c]flag
+            for v_str in self.args.v:
+                m = self.re_vol.match(v_str)
+                if not m:
+                    raise Exception("invalid -v argument: [{}]".format(v_str))
 
+                src, dst, perms = m.groups()
+                # print("\n".join([src, dst, perms]))
                 src = fsdec(os.path.abspath(fsenc(src)))
                 dst = dst.strip("/")
                 mount[dst] = src
                 mread[dst] = []
                 mwrite[dst] = []
+                mflags[dst] = {}
 
                 perms = perms.split(":")
                 for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
-                    if uname == "":
-                        uname = "*"
-                    if lvl in "ra":
-                        mread[dst].append(uname)
-                    if lvl in "wa":
-                        mwrite[dst].append(uname)
+                    self._read_vol_str(lvl, uname, mread[dst], mwrite[dst], mflags[dst])
 
         if self.args.c:
             for cfg_fn in self.args.c:
                 with open(cfg_fn, "rb") as f:
-                    self._parse_config_file(f, user, mread, mwrite, mount)
+                    self._parse_config_file(f, user, mread, mwrite, mflags, mount)
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(os.path.abspath("."), "", [], [])
+            vfs = VFS(os.path.abspath("."), "")
+            vfs.flags["d2d"] = True
 
         maxdepth = 0
         for dst in sorted(mount.keys(), key=lambda x: (x.count("/"), len(x))):
@@ -267,22 +373,117 @@ class AuthSrv(object):
 
             if dst == "":
                 # rootfs was mapped; fully replaces the default CWD vfs
-                vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst])
+                vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst], mflags[dst])
                 continue
 
             v = vfs.add(mount[dst], dst)
             v.uread = mread[dst]
             v.uwrite = mwrite[dst]
+            v.flags = mflags[dst]
+
+        missing_users = {}
+        for d in [mread, mwrite]:
+            for _, ul in d.items():
+                for usr in ul:
+                    if usr != "*" and usr not in user:
+                        missing_users[usr] = 1
+
+        if missing_users:
+            self.log(
+                "you must -a the following users: "
+                + ", ".join(k for k in sorted(missing_users)),
+                c=1,
+            )
+            raise Exception("invalid config")
+
+        all_mte = {}
+        errors = False
+        for vol in vfs.all_vols.values():
+            if (self.args.e2ds and vol.uwrite) or self.args.e2dsa:
+                vol.flags["e2ds"] = True
+
+            if self.args.e2d or "e2ds" in vol.flags:
+                vol.flags["e2d"] = True
+
+            for k in ["e2t", "e2ts", "e2tsr"]:
+                if getattr(self.args, k):
+                    vol.flags[k] = True
+
+            for k1, k2 in IMPLICATIONS:
+                if k1 in vol.flags:
+                    vol.flags[k2] = True
+
+            # default tag-list if unset
+            if "mte" not in vol.flags:
+                vol.flags["mte"] = self.args.mte
+
+            # append parsers from argv to volume-flags
+            self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
+
+            # d2d drops all database features for a volume
+            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"]]:
+                if not vol.flags.get(grp, False):
+                    continue
+
+                vol.flags["d2t"] = True
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # mt* needs e2t so drop those too
+            for grp, rm in [["e2t", "mt"]]:
+                if vol.flags.get(grp, False):
+                    continue
+
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # verify tags mentioned by -mt[mp] are used by -mte
+            local_mtp = {}
+            local_only_mtp = {}
+            for a in vol.flags.get("mtp", []) + vol.flags.get("mtm", []):
+                a = a.split("=")[0]
+                local_mtp[a] = True
+                local = True
+                for b in self.args.mtp or []:
+                    b = b.split("=")[0]
+                    if a == b:
+                        local = False
+
+                if local:
+                    local_only_mtp[a] = True
+
+            local_mte = {}
+            for a in vol.flags.get("mte", "").split(","):
+                local = True
+                all_mte[a] = True
+                local_mte[a] = True
+                for b in self.args.mte.split(","):
+                    if not a or not b:
+                        continue
+
+                    if a == b:
+                        local = False
+
+            for mtp in local_only_mtp.keys():
+                if mtp not in local_mte:
+                    m = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)'
+                    self.log(m.format(vol.vpath, mtp), 1)
+                    errors = True
+
+        for mtp in self.args.mtp or []:
+            mtp = mtp.split("=")[0]
+            if mtp not in all_mte:
+                m = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volume-flag)'
+                self.log(m.format(mtp), 1)
+                errors = True
+
+        if errors:
+            sys.exit(1)
 
         try:
-            vfs.get("/", "*", False, True)
-            if self.warn_anonwrite:
+            v, _ = vfs.get("/", "*", False, True)
+            if self.warn_anonwrite and os.getcwd() == v.realpath:
                 self.warn_anonwrite = False
-                self.log(
-                    "\033[31manyone can read/write the current directory: {}\033[0m".format(
-                        os.getcwd()
-                    )
-                )
+                msg = "anyone can read/write the current directory: {}"
+                self.log(msg.format(v.realpath), c=1)
         except Pebkac:
             self.warn_anonwrite = True
 

copyparty/broker_mp.py

@@ -4,7 +4,7 @@ from __future__ import print_function, unicode_literals
 import time
 import threading
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, VT100
 from .broker_util import try_exec
 from .broker_mpw import MpWorker
 from .util import mp
@@ -29,7 +29,7 @@ class BrokerMp(object):
         self.mutex = threading.Lock()
 
         cores = self.args.j
-        if cores is None:
+        if not cores:
             cores = mp.cpu_count()
 
         self.log("broker", "booting {} subprocesses".format(cores))
@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()
@@ -141,7 +141,7 @@ class BrokerMp(object):
 
     def debug_load_balancer(self):
         fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
-        if WINDOWS:
+        if not VT100:
             fmt = "({}{:4})"
 
         last = ""

copyparty/broker_mpw.py

@@ -49,11 +49,11 @@ class MpWorker(object):
         # print('k')
         pass
 
-    def log(self, src, msg):
-        self.q_yield.put([0, "log", [src, msg]])
+    def log(self, src, msg, c=0):
+        self.q_yield.put([0, "log", [src, msg, c]])
 
-    def logw(self, msg):
-        self.log("mp{}".format(self.n), msg)
+    def logw(self, msg, c=0):
+        self.log("mp{}".format(self.n), msg, c)
 
     def httpdrop(self, addr):
         self.q_yield.put([0, "httpdrop", [addr]])
@@ -73,7 +73,9 @@ class MpWorker(object):
                 if PY2:
                     sck = pickle.loads(sck)  # nosec
 
-                self.log(str(addr), "-" * 4 + "C-qpop")
+                if self.args.log_conn:
+                    self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+                
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:

copyparty/broker_thr.py

@@ -28,7 +28,9 @@ class BrokerThr(object):
     def put(self, want_retval, dest, *args):
         if dest == "httpconn":
             sck, addr = args
-            self.log(str(addr), "-" * 4 + "C-qpop")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+
             self.httpsrv.accept(sck, addr)
 
         else:

copyparty/broker_util.py

@@ -4,13 +4,7 @@ from __future__ import print_function, unicode_literals
 
 import traceback
 
-from .__init__ import PY2
-from .util import Pebkac
-
-if not PY2:
-    from queue import Queue
-else:
-    from Queue import Queue  # pylint: disable=import-error,no-name-in-module
+from .util import Pebkac, Queue
 
 
 class ExceptionalQueue(Queue, object):

copyparty/httpconn.py

@@ -2,13 +2,20 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import ssl
+import sys
+import time
 import socket
-import jinja2
+
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
 
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
+from .u2idx import U2idx
 
 
 class HttpConn(object):
@@ -26,28 +33,45 @@ class HttpConn(object):
         self.auth = hsrv.auth
         self.cert_path = hsrv.cert_path
 
+        self.t0 = time.time()
+        self.nbyte = 0
         self.workload = 0
+        self.u2idx = None
         self.log_func = hsrv.log
-        self.log_src = "{} \033[36m{}".format(addr[0], addr[1]).ljust(26)
+        self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
+    def set_rproxy(self, ip=None):
+        if ip is None:
+            color = 36
+            ip = self.addr[0]
+            self.rproxy = None
+        else:
+            color = 34
+            self.rproxy = ip
+
+        self.ip = ip
+        self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
+        return self.log_src
 
     def respath(self, res_name):
         return os.path.join(E.mod, "web", res_name)
 
-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)
 
-    def run(self):
+    def get_u2idx(self):
+        if not self.u2idx:
+            self.u2idx = U2idx(self.args, self.log_func)
+
+        return self.u2idx
+
+    def _detect_https(self):
         method = None
-        self.sr = None
         if self.cert_path:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
+            except socket.timeout:
+                return
             except AttributeError:
                 # jython does not support msg_peek; forget about https
                 method = self.s.recv(4)
@@ -63,20 +87,64 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
-                self.log(err)
+                if method:
+                    self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 
-        if method not in [None, b"GET ", b"HEAD", b"POST"]:
+        return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
+
+    def run(self):
+        self.sr = None
+        if self.args.https_only:
+            is_https = True
+        elif self.args.http_only or not HAVE_SSL:
+            is_https = False
+        else:
+            is_https = self._detect_https()
+
+        if is_https:
             if self.sr:
-                self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
+                self.log("TODO: cannot do https in jython", c="1;31")
                 return
 
             self.log_src = self.log_src.replace("[36m", "[35m")
             try:
-                self.s = ssl.wrap_socket(
-                    self.s, server_side=True, certfile=self.cert_path
-                )
+                ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+                ctx.load_cert_chain(self.cert_path)
+                if self.args.ssl_ver:
+                    ctx.options &= ~self.args.ssl_flags_en
+                    ctx.options |= self.args.ssl_flags_de
+                    # print(repr(ctx.options))
+
+                if self.args.ssl_log:
+                    try:
+                        ctx.keylog_filename = self.args.ssl_log
+                    except:
+                        self.log("keylog failed; openssl or python too old")
+
+                if self.args.ciphers:
+                    ctx.set_ciphers(self.args.ciphers)
+
+                self.s = ctx.wrap_socket(self.s, server_side=True)
+                msg = [
+                    "\033[1;3{:d}m{}".format(c, s)
+                    for c, s in zip([0, 5, 0], self.s.cipher())
+                ]
+                self.log(" ".join(msg) + "\033[0m")
+
+                if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
+                    overlap = [y[::-1] for y in self.s.shared_ciphers()]
+                    lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
+                    self.log("\n".join(lines))
+                    for k, v in [
+                        ["compression", self.s.compression()],
+                        ["ALPN proto", self.s.selected_alpn_protocol()],
+                        ["NPN proto", self.s.selected_npn_protocol()],
+                    ]:
+                        self.log("TLS {}: {}".format(k, v or "nah"))
+
             except Exception as ex:
                 em = str(ex)
 
@@ -89,7 +157,7 @@ class HttpConn(object):
                     pass
 
                 else:
-                    self.log("\033[35mhandshake\033[0m " + em)
+                    self.log("handshake\033[0m " + em, c=5)
 
                 return
 

copyparty/httpsrv.py

@@ -2,11 +2,29 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
-from .__init__ import E
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
+from .__init__ import E, MACOS
 from .httpconn import HttpConn
 from .authsrv import AuthSrv
 
@@ -30,6 +48,13 @@ class HttpSrv(object):
         self.workload_thr_alive = False
         self.auth = AuthSrv(self.args, self.log)
 
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "msg", "md", "mde"]
+        }
+
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
             self.cert_path = cert_path
@@ -38,7 +63,9 @@ class HttpSrv(object):
 
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log(str(addr), "-" * 5 + "C-cthr")
+        if self.args.log_conn:
+            self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+
         thr = threading.Thread(target=self.thr_client, args=(sck, addr))
         thr.daemon = True
         thr.start()
@@ -66,19 +93,31 @@ class HttpSrv(object):
                 thr.start()
 
         try:
-            self.log(str(addr), "-" * 6 + "C-crun")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+
             cli.run()
 
         finally:
-            self.log(str(addr), "-" * 7 + "C-done")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+
             try:
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
             except (OSError, socket.error) as ex:
-                if ex.errno not in [107, 57, 9]:
-                    # 107 Transport endpoint not connected
-                    #  57 Socket is not connected
-                    #   9 Bad file descriptor
+                if not MACOS:
+                    self.log(
+                        "%s %s" % addr,
+                        "shut({}): {}".format(sck.fileno(), ex),
+                        c="1;30",
+                    )
+                if ex.errno not in [10038, 10054, 107, 57, 9]:
+                    # 10038 No longer considered a socket
+                    # 10054 Foribly closed by remote
+                    #   107 Transport endpoint not connected
+                    #    57 Socket is not connected
+                    #     9 Bad file descriptor
                     raise
             finally:
                 with self.mutex:

copyparty/mtag.py

@@ -0,0 +1,347 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import re
+import os
+import sys
+import shutil
+import subprocess as sp
+
+from .__init__ import PY2, WINDOWS
+from .util import fsenc, fsdec, REKOBO_LKEY
+
+if not PY2:
+    unicode = str
+
+
+class MTag(object):
+    def __init__(self, log_func, args):
+        self.log_func = log_func
+        self.usable = True
+        self.prefer_mt = False
+        mappings = args.mtm
+        self.backend = "ffprobe" if args.no_mutagen else "mutagen"
+        or_ffprobe = " or ffprobe"
+
+        if self.backend == "mutagen":
+            self.get = self.get_mutagen
+            try:
+                import mutagen
+            except:
+                self.log("could not load mutagen, trying ffprobe instead", c=3)
+                self.backend = "ffprobe"
+
+        if self.backend == "ffprobe":
+            self.get = self.get_ffprobe
+            self.prefer_mt = True
+            # about 20x slower
+            if PY2:
+                cmd = [b"ffprobe", b"-version"]
+                try:
+                    sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+                except:
+                    self.usable = False
+            else:
+                if not shutil.which("ffprobe"):
+                    self.usable = False
+
+            if self.usable and WINDOWS and sys.version_info < (3, 8):
+                self.usable = False
+                or_ffprobe = " or python >= 3.8"
+                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                self.log(msg, c=1)
+
+        if not self.usable:
+            msg = "need mutagen{} to read media tags so please run this:\n  {} -m pip install --user mutagen"
+            self.log(msg.format(or_ffprobe, os.path.basename(sys.executable)), c=1)
+            return
+
+        # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
+        tagmap = {
+            "album": ["album", "talb", "\u00a9alb", "original-album", "toal"],
+            "artist": [
+                "artist",
+                "tpe1",
+                "\u00a9art",
+                "composer",
+                "performer",
+                "arranger",
+                "\u00a9wrt",
+                "tcom",
+                "tpe3",
+                "original-artist",
+                "tope",
+            ],
+            "title": ["title", "tit2", "\u00a9nam"],
+            "circle": [
+                "album-artist",
+                "tpe2",
+                "aart",
+                "conductor",
+                "organization",
+                "band",
+            ],
+            ".tn": ["tracknumber", "trck", "trkn", "track"],
+            "genre": ["genre", "tcon", "\u00a9gen"],
+            "date": [
+                "original-release-date",
+                "release-date",
+                "date",
+                "tdrc",
+                "\u00a9day",
+                "original-date",
+                "original-year",
+                "tyer",
+                "tdor",
+                "tory",
+                "year",
+                "creation-time",
+            ],
+            ".bpm": ["bpm", "tbpm", "tmpo", "tbp"],
+            "key": ["initial-key", "tkey", "key"],
+            "comment": ["comment", "comm", "\u00a9cmt", "comments", "description"],
+        }
+
+        if mappings:
+            for k, v in [x.split("=") for x in mappings]:
+                tagmap[k] = v.split(",")
+
+        self.tagmap = {}
+        for k, vs in tagmap.items():
+            vs2 = []
+            for v in vs:
+                if "-" not in v:
+                    vs2.append(v)
+                    continue
+
+                vs2.append(v.replace("-", " "))
+                vs2.append(v.replace("-", "_"))
+                vs2.append(v.replace("-", ""))
+
+            self.tagmap[k] = vs2
+
+        self.rmap = {
+            v: [n, k] for k, vs in self.tagmap.items() for n, v in enumerate(vs)
+        }
+        # self.get = self.compare
+
+    def log(self, msg, c=0):
+        self.log_func("mtag", msg, c)
+
+    def normalize_tags(self, ret, md):
+        for k, v in dict(md).items():
+            if not v:
+                continue
+
+            k = k.lower().split("::")[0].strip()
+            mk = self.rmap.get(k)
+            if not mk:
+                continue
+
+            pref, mk = mk
+            if mk not in ret or ret[mk][0] > pref:
+                ret[mk] = [pref, v[0]]
+
+        # take first value
+        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
+
+        # track 3/7 => track 3
+        for k, v in ret.items():
+            if k[0] == ".":
+                v = v.split("/")[0].strip().lstrip("0")
+                ret[k] = v or 0
+
+        # normalize key notation to rkeobo
+        okey = ret.get("key")
+        if okey:
+            key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
+            ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
+
+        return ret
+
+    def compare(self, abspath):
+        if abspath.endswith(".au"):
+            return {}
+
+        print("\n" + abspath)
+        r1 = self.get_mutagen(abspath)
+        r2 = self.get_ffprobe(abspath)
+
+        keys = {}
+        for d in [r1, r2]:
+            for k in d.keys():
+                keys[k] = True
+
+        diffs = []
+        l1 = []
+        l2 = []
+        for k in sorted(keys.keys()):
+            if k in [".q", ".dur"]:
+                continue  # lenient
+
+            v1 = r1.get(k)
+            v2 = r2.get(k)
+            if v1 == v2:
+                print("  ", k, v1)
+            elif v1 != "0000":  # ffprobe date=0
+                diffs.append(k)
+                print(" 1", k, v1)
+                print(" 2", k, v2)
+                if v1:
+                    l1.append(k)
+                if v2:
+                    l2.append(k)
+
+        if diffs:
+            raise Exception()
+
+        return r1
+
+    def get_mutagen(self, abspath):
+        import mutagen
+
+        try:
+            md = mutagen.File(abspath, easy=True)
+            x = md.info.length
+        except Exception as ex:
+            return {}
+
+        ret = {}
+        try:
+            dur = int(md.info.length)
+            try:
+                q = int(md.info.bitrate / 1024)
+            except:
+                q = int((os.path.getsize(abspath) / dur) / 128)
+
+            ret[".dur"] = [0, dur]
+            ret[".q"] = [0, q]
+        except:
+            pass
+
+        return self.normalize_tags(ret, md)
+
+    def get_ffprobe(self, abspath):
+        cmd = [b"ffprobe", b"-hide_banner", b"--", fsenc(abspath)]
+        p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+        r = p.communicate()
+        txt = r[1].decode("utf-8", "replace")
+        txt = [x.rstrip("\r") for x in txt.split("\n")]
+
+        """
+        note:
+          tags which contain newline will be truncated on first \n,
+          ffprobe emits \n and spacepads the : to align visually
+        note:
+          the Stream ln always mentions Audio: if audio
+          the Stream ln usually has kb/s, is more accurate
+          the Duration ln always has kb/s
+          the Metadata: after Chapter may contain BPM info,
+            title : Tempo: 126.0
+
+        Input #0, wav,
+          Metadata:
+            date : <OK>
+          Duration:
+            Chapter #
+            Metadata:
+              title : <NG>
+
+        Input #0, mp3,
+          Metadata:
+            album : <OK>
+          Duration:
+            Stream #0:0: Audio:
+            Stream #0:1: Video:
+            Metadata:
+              comment : <NG>
+        """
+
+        ptn_md_beg = re.compile("^( +)Metadata:$")
+        ptn_md_kv = re.compile("^( +)([^:]+) *: (.*)")
+        ptn_dur = re.compile("^ *Duration: ([^ ]+)(, |$)")
+        ptn_br1 = re.compile("^ *Duration: .*, bitrate: ([0-9]+) kb/s(, |$)")
+        ptn_br2 = re.compile("^ *Stream.*: Audio:.* ([0-9]+) kb/s(, |$)")
+        ptn_audio = re.compile("^ *Stream .*: Audio: ")
+        ptn_au_parent = re.compile("^ *(Input #|Stream .*: Audio: )")
+
+        ret = {}
+        md = {}
+        in_md = False
+        is_audio = False
+        au_parent = False
+        for ln in txt:
+            m = ptn_md_kv.match(ln)
+            if m and in_md and len(m.group(1)) == in_md:
+                _, k, v = [x.strip() for x in m.groups()]
+                if k != "" and v != "":
+                    md[k] = [v]
+                continue
+            else:
+                in_md = False
+
+            m = ptn_md_beg.match(ln)
+            if m and au_parent:
+                in_md = len(m.group(1)) + 2
+                continue
+
+            au_parent = bool(ptn_au_parent.search(ln))
+
+            if ptn_audio.search(ln):
+                is_audio = True
+
+            m = ptn_dur.search(ln)
+            if m:
+                sec = 0
+                tstr = m.group(1)
+                if tstr.lower() != "n/a":
+                    try:
+                        tf = tstr.split(",")[0].split(".")[0].split(":")
+                        for f in tf:
+                            sec *= 60
+                            sec += int(f)
+                    except:
+                        self.log("invalid timestr from ffprobe: [{}]".format(tstr), c=3)
+
+                ret[".dur"] = sec
+                m = ptn_br1.search(ln)
+                if m:
+                    ret[".q"] = m.group(1)
+
+            m = ptn_br2.search(ln)
+            if m:
+                ret[".q"] = m.group(1)
+
+        if not is_audio:
+            return {}
+
+        ret = {k: [0, v] for k, v in ret.items()}
+
+        return self.normalize_tags(ret, md)
+
+    def get_bin(self, parsers, abspath):
+        pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
+        pypath = str(os.pathsep.join(pypath))
+        env = os.environ.copy()
+        env["PYTHONPATH"] = pypath
+
+        ret = {}
+        for tagname, (binpath, timeout) in parsers.items():
+            try:
+                cmd = [sys.executable, binpath, abspath]
+                args = {"env": env, "timeout": timeout}
+
+                if WINDOWS:
+                    args["creationflags"] = 0x4000
+                else:
+                    cmd = ["nice"] + cmd
+
+                cmd = [fsenc(x) for x in cmd]
+                v = sp.check_output(cmd, **args).strip()
+                if v:
+                    ret[tagname] = v.decode("utf-8")
+            except:
+                pass
+
+        return ret

copyparty/star.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen)
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/stolen/surrogateescape.py

@@ -10,11 +10,12 @@ Original source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/m
 
 # This code is released under the Python license and the BSD 2-clause license
 
+import platform
 import codecs
 import sys
 
 PY3 = sys.version_info[0] > 2
-
+WINDOWS = platform.system() == "Windows"
 FS_ERRORS = "surrogateescape"
 
 
@@ -168,6 +169,11 @@ FS_ENCODING = sys.getfilesystemencoding()
 # FS_ENCODING = 'UTF-8'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]')
 
 
+if WINDOWS and not PY3:
+    # py2 thinks win* is mbcs, probably a bug? anyways this works
+    FS_ENCODING = 'utf-8'
+
+
 # normalize the filesystem encoding name.
 # For example, we expect "utf-8", not "UTF8".
 FS_ENCODING = codecs.lookup(FS_ENCODING).name

copyparty/sutil.py

@@ -0,0 +1,25 @@
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/svchub.py

@@ -8,7 +8,7 @@ import threading
 from datetime import datetime, timedelta
 import calendar
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, MACOS, VT100
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
 from .util import mp
@@ -65,16 +65,16 @@ class SvcHub(object):
             self.broker.shutdown()
             print("nailed it")
 
-    def _log_disabled(self, src, msg):
+    def _log_disabled(self, src, msg, c=0):
         pass
 
-    def _log_enabled(self, src, msg):
+    def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
         with self.log_mutex:
             now = time.time()
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
-                print("\033[36m{}\033[0m".format(dt.strftime("%Y-%m-%d")))
+                print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
 
                 # unix timestamp of next 00:00:00 (leap-seconds safe)
                 day_now = dt.day
@@ -84,22 +84,30 @@ class SvcHub(object):
                 dt = dt.replace(hour=0, minute=0, second=0)
                 self.next_day = calendar.timegm(dt.utctimetuple())
 
-            ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
-
-            if not WINDOWS:
-                fmt = "\033[36m{} \033[33m{:21} \033[0m{}"
-            else:
-                fmt = "{} {:21} {}"
+            fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
+            if not VT100:
+                fmt = "{} {:21} {}\n"
                 if "\033" in msg:
                     msg = self.ansi_re.sub("", msg)
                 if "\033" in src:
                     src = self.ansi_re.sub("", src)
+            elif c:
+                if isinstance(c, int):
+                    msg = "\033[3{}m{}".format(c, msg)
+                elif "\033" not in c:
+                    msg = "\033[{}m{}\033[0m".format(c, msg)
+                else:
+                    msg = "{}{}\033[0m".format(c, msg)
 
+            ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             msg = fmt.format(ts, src, msg)
             try:
-                print(msg)
+                print(msg, end="")
             except UnicodeEncodeError:
-                print(msg.encode("utf-8", "replace").decode())
+                try:
+                    print(msg.encode("utf-8", "replace").decode(), end="")
+                except:
+                    print(msg.encode("ascii", "replace").decode(), end="")
 
     def check_mp_support(self):
         vmin = sys.version_info[1]
@@ -110,6 +118,8 @@ class SvcHub(object):
                 return msg
             elif vmin < 3:
                 return msg
+        elif MACOS:
+            return "multiprocessing is wonky on mac osx;"
         else:
             msg = "need python 2.7 or 3.3+ for multiprocessing;"
             if not PY2 and vmin < 3:
@@ -126,8 +136,11 @@ class SvcHub(object):
         return None
 
     def check_mp_enable(self):
-        if self.args.j == 0:
-            self.log("root", "multiprocessing disabled by argument -j 0;")
+        if self.args.j == 1:
+            self.log("root", "multiprocessing disabled by argument -j 1;")
+            return False
+
+        if mp.cpu_count() <= 1:
             return False
 
         try:

copyparty/szip.py

@@ -0,0 +1,271 @@
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, "/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import time
 import socket
+import select
 
 from .util import chkcmd, Counter
 
@@ -23,55 +24,80 @@ class TcpSrv(object):
 
         ip = "127.0.0.1"
         eps = {ip: "local only"}
-        if self.args.i != ip:
-            eps = self.detect_interfaces(self.args.i) or eps
+        nonlocals = [x for x in self.args.i if x != ip]
+        if nonlocals:
+            eps = self.detect_interfaces(self.args.i)
+            if not eps:
+                for x in nonlocals:
+                    eps[x] = "external"
 
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
-            self.log(
-                "tcpsrv",
-                "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
-                    ip, self.args.p, desc
-                ),
-            )
+            for port in sorted(self.args.p):
+                self.log(
+                    "tcpsrv",
+                    "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
+                        ip, port, desc
+                    ),
+                )
 
-        self.srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.srv = []
+        for ip in self.args.i:
+            for port in self.args.p:
+                self.srv.append(self._listen(ip, port))
+
+    def _listen(self, ip, port):
+        srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         try:
-            self.srv.bind((self.args.i, self.args.p))
+            srv.bind((ip, port))
+            return srv
         except (OSError, socket.error) as ex:
-            if ex.errno == 98:
-                raise Exception(
-                    "\033[1;31mport {} is busy on interface {}\033[0m".format(
-                        self.args.p, self.args.i
-                    )
-                )
-
-            if ex.errno == 99:
-                raise Exception(
-                    "\033[1;31minterface {} does not exist\033[0m".format(self.args.i)
-                )
+            if ex.errno in [98, 48]:
+                e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
+            elif ex.errno in [99, 49]:
+                e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
+            else:
+                raise
+            raise Exception(e)
 
     def run(self):
-        self.srv.listen(self.args.nc)
-
-        self.log("tcpsrv", "listening @ {0}:{1}".format(self.args.i, self.args.p))
+        for srv in self.srv:
+            srv.listen(self.args.nc)
+            ip, port = srv.getsockname()
+            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
 
         while True:
-            self.log("tcpsrv", "-" * 1 + "C-ncli")
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
+
             if self.num_clients.v >= self.args.nc:
                 time.sleep(0.1)
                 continue
 
-            self.log("tcpsrv", "-" * 2 + "C-acc1")
-            sck, addr = self.srv.accept()
-            self.log(str(addr), "-" * 3 + "C-acc2")
-            self.num_clients.add()
-            self.hub.broker.put(False, "httpconn", sck, addr)
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
+
+            ready, _, _ = select.select(self.srv, [], [])
+            for srv in ready:
+                sck, addr = srv.accept()
+                sip, sport = srv.getsockname()
+                if self.args.log_conn:
+                    self.log(
+                        "%s %s" % addr,
+                        "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                            "-" * 3, sip, sport % 8, sport
+                        ),
+                        c="1;30",
+                    )
+
+                self.num_clients.add()
+                self.hub.broker.put(False, "httpconn", sck, addr)
 
     def shutdown(self):
         self.log("tcpsrv", "ok bye")
 
-    def detect_interfaces(self, ext_ip):
+    def detect_interfaces(self, listen_ips):
         eps = {}
 
         # get all ips and their interfaces
@@ -85,29 +111,41 @@ class TcpSrv(object):
             for ln in ip_addr.split("\n"):
                 try:
                     ip, dev = r.match(ln.rstrip()).groups()
-                    if ext_ip in ["0.0.0.0", ip]:
-                        eps[ip] = dev
+                    for lip in listen_ips:
+                        if lip in ["0.0.0.0", ip]:
+                            eps[ip] = dev
                 except:
                     pass
 
-        # get ip with default route
+        default_route = None
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        try:
-            s.connect(("10.255.255.255", 1))
-            ip = s.getsockname()[0]
-        except (OSError, socket.error) as ex:
-            if ex.errno not in [101, 10065]:
-                raise
-
-            return None
+        for ip in [
+            "10.255.255.255",
+            "172.31.255.255",
+            "192.168.255.255",
+            "239.255.255.255",
+            # could add 1.1.1.1 as a final fallback
+            # but external connections is kinshi
+        ]:
+            try:
+                s.connect((ip, 1))
+                # raise OSError(13, "a")
+                default_route = s.getsockname()[0]
+                break
+            except (OSError, socket.error) as ex:
+                if ex.errno == 13:
+                    self.log("tcpsrv", "eaccess {} (trying next)".format(ip))
+                elif ex.errno not in [101, 10065, 10051]:
+                    self.log("tcpsrv", "route lookup failed; err {}".format(ex.errno))
 
         s.close()
 
-        if ext_ip in ["0.0.0.0", ip]:
-            desc = "\033[32mexternal"
-            try:
-                eps[ip] += ", " + desc
-            except:
-                eps[ip] = desc
+        for lip in listen_ips:
+            if default_route and lip in ["0.0.0.0", default_route]:
+                desc = "\033[32mexternal"
+                try:
+                    eps[default_route] += ", " + desc
+                except:
+                    eps[default_route] = desc
 
         return eps

copyparty/u2idx.py

@@ -0,0 +1,281 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import re
+import os
+import time
+import threading
+from datetime import datetime
+
+from .util import u8safe, s3dec, html_escape, Pebkac
+from .up2k import up2k_wark_from_hashlist
+
+
+try:
+    HAVE_SQLITE3 = True
+    import sqlite3
+except:
+    HAVE_SQLITE3 = False
+
+
+class U2idx(object):
+    def __init__(self, args, log_func):
+        self.args = args
+        self.log_func = log_func
+        self.timeout = args.srch_time
+
+        if not HAVE_SQLITE3:
+            self.log("could not load sqlite3; searchign wqill be disabled")
+            return
+
+        self.cur = {}
+        self.mem_cur = sqlite3.connect(":memory:")
+        self.mem_cur.execute(r"create table a (b text)")
+
+        self.p_end = None
+        self.p_dur = 0
+
+    def log(self, msg, c=0):
+        self.log_func("u2idx", msg, c)
+
+    def fsearch(self, vols, body):
+        """search by up2k hashlist"""
+        if not HAVE_SQLITE3:
+            return []
+
+        fsize = body["size"]
+        fhash = body["hash"]
+        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
+
+        uq = "substr(w,1,16) = ? and w = ?"
+        uv = [wark[:16], wark]
+
+        try:
+            return self.run_query(vols, uq, uv, {})[0]
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
+
+    def get_cur(self, ptop):
+        cur = self.cur.get(ptop)
+        if cur:
+            return cur
+
+        cur = _open(ptop)
+        if not cur:
+            return None
+
+        self.cur[ptop] = cur
+        return cur
+
+    def search(self, vols, body):
+        """search by query params"""
+        if not HAVE_SQLITE3:
+            return []
+
+        qobj = {}
+        _conv_sz(qobj, body, "sz_min", "up.sz >= ?")
+        _conv_sz(qobj, body, "sz_max", "up.sz <= ?")
+        _conv_dt(qobj, body, "dt_min", "up.mt >= ?")
+        _conv_dt(qobj, body, "dt_max", "up.mt <= ?")
+        for seg, dk in [["path", "up.rd"], ["name", "up.fn"]]:
+            if seg in body:
+                _conv_txt(qobj, body, seg, dk)
+
+        uq, uv = _sqlize(qobj)
+
+        qobj = {}
+        if "tags" in body:
+            _conv_txt(qobj, body, "tags", "mt.v")
+
+        if "adv" in body:
+            _conv_adv(qobj, body, "adv")
+
+        try:
+            return self.run_query(vols, uq, uv, qobj)
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
+
+    def run_query(self, vols, uq, uv, targs):
+        self.log("qs: {} {} ,  {}".format(uq, repr(uv), repr(targs)))
+
+        done_flag = []
+        self.active_id = "{:.6f}_{}".format(
+            time.time(), threading.current_thread().ident
+        )
+        thr = threading.Thread(
+            target=self.terminator,
+            args=(
+                self.active_id,
+                done_flag,
+            ),
+        )
+        thr.daemon = True
+        thr.start()
+
+        if not targs:
+            if not uq:
+                q = "select * from up"
+                v = ()
+            else:
+                q = "select * from up where " + uq
+                v = tuple(uv)
+        else:
+            q = "select up.* from up"
+            keycmp = "substr(up.w,1,16)"
+            where = []
+            v = []
+            ctr = 0
+            for tq, tv in sorted(targs.items()):
+                ctr += 1
+                tq = tq.split("\n")[0]
+                keycmp2 = "mt{}.w".format(ctr)
+                q += " inner join mt mt{} on {} = {}".format(ctr, keycmp, keycmp2)
+                keycmp = keycmp2
+                where.append(tq.replace("mt.", keycmp[:-1]))
+                v.append(tv)
+
+            if uq:
+                where.append(uq)
+                v.extend(uv)
+
+            q += " where " + (" and ".join(where))
+
+        # self.log("q2: {} {}".format(q, repr(v)))
+
+        ret = []
+        lim = 1000
+        taglist = {}
+        for (vtop, ptop, flags) in vols:
+            cur = self.get_cur(ptop)
+            if not cur:
+                continue
+
+            self.active_cur = cur
+
+            sret = []
+            c = cur.execute(q, v)
+            for hit in c:
+                w, ts, sz, rd, fn = hit
+                lim -= 1
+                if lim <= 0:
+                    break
+
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
+                rp = os.path.join(vtop, rd, fn).replace("\\", "/")
+                sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
+
+            for hit in sret:
+                w = hit["w"]
+                del hit["w"]
+                tags = {}
+                q2 = "select k, v from mt where w = ? and k != 'x'"
+                for k, v2 in cur.execute(q2, (w,)):
+                    taglist[k] = True
+                    tags[k] = v2
+
+                hit["tags"] = tags
+
+            ret.extend(sret)
+
+        done_flag.append(True)
+        self.active_id = None
+
+        # undupe hits from multiple metadata keys
+        if len(ret) > 1:
+            ret = [ret[0]] + [
+                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+            ]
+
+        return ret, list(taglist.keys())
+
+    def terminator(self, identifier, done_flag):
+        for _ in range(self.timeout):
+            time.sleep(1)
+            if done_flag:
+                return
+
+        if identifier == self.active_id:
+            self.active_cur.connection.interrupt()
+
+
+def _open(ptop):
+    db_path = os.path.join(ptop, ".hist", "up2k.db")
+    if os.path.exists(db_path):
+        return sqlite3.connect(db_path).cursor()
+
+
+def _conv_sz(q, body, k, sql):
+    if k in body:
+        q[sql] = int(float(body[k]) * 1024 * 1024)
+
+
+def _conv_dt(q, body, k, sql):
+    if k not in body:
+        return
+
+    v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ")
+    while "  " in v:
+        v = v.replace("  ", " ")
+
+    for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]:
+        try:
+            ts = datetime.strptime(v, fmt).timestamp()
+            break
+        except:
+            ts = None
+
+    if ts:
+        q[sql] = ts
+
+
+def _conv_txt(q, body, k, sql):
+    for v in body[k].split(" "):
+        inv = ""
+        if v.startswith("-"):
+            inv = "not"
+            v = v[1:]
+
+        if not v:
+            continue
+
+        head = "'%'||"
+        if v.startswith("^"):
+            head = ""
+            v = v[1:]
+
+        tail = "||'%'"
+        if v.endswith("$"):
+            tail = ""
+            v = v[:-1]
+
+        qk = "{} {} like {}?{}".format(sql, inv, head, tail)
+        q[qk + "\n" + v] = u8safe(v)
+
+
+def _conv_adv(q, body, k):
+    ptn = re.compile(r"^(\.?[a-z]+) *(==?|!=|<=?|>=?) *(.*)$")
+
+    parts = body[k].split(" ")
+    parts = [x.strip() for x in parts if x.strip()]
+
+    for part in parts:
+        m = ptn.match(part)
+        if not m:
+            p = html_escape(part)
+            raise Pebkac(400, "invalid argument [" + p + "]")
+
+        k, op, v = m.groups()
+        qk = "mt.k = '{}' and mt.v {} ?".format(k, op)
+        q[qk + "\n" + v] = u8safe(v)
+
+
+def _sqlize(qobj):
+    keys = []
+    values = []
+    for k, v in sorted(qobj.items()):
+        keys.append(k.split("\n")[0])
+        values.append(v)
+
+    return " and ".join(keys), values

copyparty/util.py

@@ -2,15 +2,21 @@
 from __future__ import print_function, unicode_literals
 
 import re
+import os
 import sys
+import time
 import base64
+import select
 import struct
 import hashlib
 import platform
+import traceback
 import threading
+import mimetypes
+import contextlib
 import subprocess as sp  # nosec
 
-from .__init__ import PY2
+from .__init__ import PY2, WINDOWS
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -27,16 +33,21 @@ except ImportError:
 if not PY2:
     from urllib.parse import unquote_to_bytes as unquote
     from urllib.parse import quote_from_bytes as quote
+    from queue import Queue
 else:
     from urllib import unquote  # pylint: disable=no-name-in-module
     from urllib import quote  # pylint: disable=no-name-in-module
+    from Queue import Queue  # pylint: disable=import-error,no-name-in-module
 
 surrogateescape.register_surrogateescape()
 FS_ENCODING = sys.getfilesystemencoding()
+if WINDOWS and PY2:
+    FS_ENCODING = "utf-8"
 
 
 HTTPCODE = {
     200: "OK",
+    204: "No Content",
     206: "Partial Content",
     304: "Not Modified",
     400: "Bad Request",
@@ -44,12 +55,60 @@ HTTPCODE = {
     404: "Not Found",
     405: "Method Not Allowed",
     413: "Payload Too Large",
+    416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
 }
 
 
+IMPLICATIONS = [
+    ["e2dsa", "e2ds"],
+    ["e2ds", "e2d"],
+    ["e2tsr", "e2ts"],
+    ["e2ts", "e2t"],
+    ["e2t", "e2d"],
+]
+
+
+REKOBO_KEY = {
+    v: ln.split(" ", 1)[0]
+    for ln in """
+1B 6d B
+2B 7d Gb F#
+3B 8d Db C#
+4B 9d Ab G#
+5B 10d Eb D#
+6B 11d Bb A#
+7B 12d F
+8B 1d C
+9B 2d G
+10B 3d D
+11B 4d A
+12B 5d E
+1A 6m Abm G#m
+2A 7m Ebm D#m
+3A 8m Bbm A#m
+4A 9m Fm
+5A 10m Cm
+6A 11m Gm
+7A 12m Dm
+8A 1m Am
+9A 2m Em
+10A 3m Bm
+11A 4m Gbm F#m
+12A 5m Dbm C#m
+""".strip().split(
+        "\n"
+    )
+    for v in ln.strip().split(" ")[1:]
+    if v
+}
+
+REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
+
+
 class Counter(object):
     def __init__(self, v=0):
         self.v = v
@@ -88,6 +147,145 @@ class Unrecv(object):
         self.buf = buf + self.buf
 
 
+class ProgressPrinter(threading.Thread):
+    """
+    periodically print progress info without linefeeds
+    """
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.msg = None
+        self.end = False
+        self.start()
+
+    def run(self):
+        msg = None
+        while not self.end:
+            time.sleep(0.1)
+            if msg == self.msg or self.end:
+                continue
+
+            msg = self.msg
+            uprint(" {}\033[K\r".format(msg))
+
+        print("\033[K", end="")
+        sys.stdout.flush()  # necessary on win10 even w/ stderr btw
+
+
+def uprint(msg):
+    try:
+        print(msg, end="")
+    except UnicodeEncodeError:
+        try:
+            print(msg.encode("utf-8", "replace").decode(), end="")
+        except:
+            print(msg.encode("ascii", "replace").decode(), end="")
+
+
+def nuprint(msg):
+    uprint("{}\n".format(msg))
+
+
+def rice_tid():
+    tid = threading.current_thread().ident
+    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
+
+
+def trace(*args, **kwargs):
+    t = time.time()
+    stack = "".join(
+        "\033[36m{}\033[33m{}".format(x[0].split(os.sep)[-1][:-3], x[1])
+        for x in traceback.extract_stack()[3:-1]
+    )
+    parts = ["{:.6f}".format(t), rice_tid(), stack]
+
+    if args:
+        parts.append(repr(args))
+
+    if kwargs:
+        parts.append(repr(kwargs))
+
+    msg = "\033[0m ".join(parts)
+    # _tracebuf.append(msg)
+    nuprint(msg)
+
+
+@contextlib.contextmanager
+def ren_open(fname, *args, **kwargs):
+    fdir = kwargs.pop("fdir", None)
+    suffix = kwargs.pop("suffix", None)
+
+    if fname == os.devnull:
+        with open(fname, *args, **kwargs) as f:
+            yield {"orz": [f, fname]}
+            return
+
+    orig_name = fname
+    bname = fname
+    ext = ""
+    while True:
+        ofs = bname.rfind(".")
+        if ofs < 0 or ofs < len(bname) - 7:
+            # doesn't look like an extension anymore
+            break
+
+        ext = bname[ofs:] + ext
+        bname = bname[:ofs]
+
+    b64 = ""
+    while True:
+        try:
+            if fdir:
+                fpath = os.path.join(fdir, fname)
+            else:
+                fpath = fname
+
+            if suffix and os.path.exists(fpath):
+                fpath += suffix
+                fname += suffix
+                ext += suffix
+
+            with open(fsenc(fpath), *args, **kwargs) as f:
+                if b64:
+                    fp2 = "fn-trunc.{}.txt".format(b64)
+                    fp2 = os.path.join(fdir, fp2)
+                    with open(fsenc(fp2), "wb") as f2:
+                        f2.write(orig_name.encode("utf-8"))
+
+                yield {"orz": [f, fname]}
+                return
+
+        except OSError as ex_:
+            ex = ex_
+            if ex.errno not in [36, 63] and (not WINDOWS or ex.errno != 22):
+                raise
+
+        if not b64:
+            b64 = (bname + ext).encode("utf-8", "replace")
+            b64 = hashlib.sha512(b64).digest()[:12]
+            b64 = base64.urlsafe_b64encode(b64).decode("utf-8").rstrip("=")
+
+        badlen = len(fname)
+        while len(fname) >= badlen:
+            if len(bname) < 8:
+                raise ex
+
+            if len(bname) > len(ext):
+                # drop the last letter of the filename
+                bname = bname[:-1]
+            else:
+                try:
+                    # drop the leftmost sub-extension
+                    _, ext = ext.split(".", 1)
+                except:
+                    # okay do the first letter then
+                    ext = "." + ext[2:]
+
+            fname = "{}~{}{}".format(bname, b64, ext)
+
+
 class MultipartParser(object):
     def __init__(self, log_func, sr, http_headers):
         self.sr = sr
@@ -186,7 +384,7 @@ class MultipartParser(object):
             buf = self.sr.recv(bufsz)
             if not buf:
                 # abort: client disconnected
-                raise Pebkac(400, "client disconnected during multipart post")
+                raise Pebkac(400, "client d/c during multipart post")
 
             while True:
                 ofs = buf.find(self.boundary)
@@ -220,7 +418,7 @@ class MultipartParser(object):
                 buf2 = self.sr.recv(bufsz)
                 if not buf2:
                     # abort: client disconnected
-                    raise Pebkac(400, "client disconnected during multipart post")
+                    raise Pebkac(400, "client d/c during multipart post")
 
                 buf += buf2
 
@@ -304,18 +502,7 @@ def get_boundary(headers):
 def read_header(sr):
     ret = b""
     while True:
-        if ret.endswith(b"\r\n\r\n"):
-            break
-        elif ret.endswith(b"\r\n\r"):
-            n = 1
-        elif ret.endswith(b"\r\n"):
-            n = 2
-        elif ret.endswith(b"\r"):
-            n = 3
-        else:
-            n = 4
-
-        buf = sr.recv(n)
+        buf = sr.recv(1024)
         if not buf:
             if not ret:
                 return None
@@ -327,11 +514,50 @@ def read_header(sr):
             )
 
         ret += buf
+        ofs = ret.find(b"\r\n\r\n")
+        if ofs < 0:
+            if len(ret) > 1024 * 64:
+                raise Pebkac(400, "header 2big")
+            else:
+                continue
 
-        if len(ret) > 1024 * 64:
-            raise Pebkac(400, "header 2big")
+        sr.unrecv(ret[ofs + 4 :])
+        return ret[:ofs].decode("utf-8", "surrogateescape").split("\r\n")
 
-    return ret[:-4].decode("utf-8", "surrogateescape").split("\r\n")
+
+def humansize(sz, terse=False):
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
+        if sz < 1024:
+            break
+
+        sz /= 1024.0
+
+    ret = " ".join([str(sz)[:4].rstrip("."), unit])
+
+    if not terse:
+        return ret
+
+    return ret.replace("iB", "").replace(" ", "")
+
+
+def get_spd(nbyte, t0, t=None):
+    if t is None:
+        t = time.time()
+
+    bps = nbyte / ((t - t0) + 0.001)
+    s1 = humansize(nbyte).replace(" ", "\033[33m").replace("iB", "")
+    s2 = humansize(bps).replace(" ", "\033[35m").replace("iB", "")
+    return "{} \033[0m{}/s\033[0m".format(s1, s2)
+
+
+def s2hms(s, optional_h=False):
+    s = int(s)
+    h, s = divmod(s, 3600)
+    m, s = divmod(s, 60)
+    if not h and optional_h:
+        return "{}:{:02}".format(m, s)
+
+    return "{}:{:02}:{:02}".format(h, m, s)
 
 
 def undot(path):
@@ -350,36 +576,80 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
-    return fn.replace("\\", "/").split("/")[-1].strip()
+def sanitize_fn(fn, ok=""):
+    if "/" not in ok:
+        fn = fn.replace("\\", "/").split("/")[-1]
+
+    if WINDOWS:
+        for bad, good in [x for x in [
+            ["<", "＜"],
+            [">", "＞"],
+            [":", "："],
+            ['"', "＂"],
+            ["/", "／"],
+            ["\\", "＼"],
+            ["|", "｜"],
+            ["?", "？"],
+            ["*", "＊"],
+        ] if x[0] not in ok]:
+            fn = fn.replace(bad, good)
+
+        bad = ["con", "prn", "aux", "nul"]
+        for n in range(1, 10):
+            bad += "com{0} lpt{0}".format(n).split(" ")
+
+        if fn.lower() in bad:
+            fn = "_" + fn
+
+    return fn.strip()
+
+
+def u8safe(txt):
+    try:
+        return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
+    except:
+        return txt.encode("utf-8", "replace").decode("utf-8", "replace")
 
 
 def exclude_dotfiles(filepaths):
-    for fpath in filepaths:
-        if not fpath.split("/")[-1].startswith("."):
-            yield fpath
+    return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
+
+
+def html_escape(s, quote=False):
+    """html.escape but also newlines"""
+    s = (
+        s.replace("&", "&amp;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace("\r", "&#13;")
+        .replace("\n", "&#10;")
+    )
+    if quote:
+        s = s.replace('"', "&quot;").replace("'", "&#x27;")
+
+    return s
 
 
 def quotep(txt):
     """url quoter which deals with bytes correctly"""
-    btxt = fsenc(txt)
+    btxt = w8enc(txt)
     quot1 = quote(btxt, safe=b"/")
     if not PY2:
         quot1 = quot1.encode("ascii")
 
     quot2 = quot1.replace(b" ", b"+")
-    return fsdec(quot2)
+    return w8dec(quot2)
 
 
 def unquotep(txt):
     """url unquoter which deals with bytes correctly"""
-    btxt = fsenc(txt)
-    unq1 = btxt.replace(b"+", b" ")
-    unq2 = unquote(unq1)
-    return fsdec(unq2)
+    btxt = w8enc(txt)
+    # btxt = btxt.replace(b"+", b" ")
+    unq2 = unquote(btxt)
+    return w8dec(unq2)
 
 
-def fsdec(txt):
+def w8dec(txt):
     """decodes filesystem-bytes to wtf8"""
     if PY2:
         return surrogateescape.decodefilename(txt)
@@ -387,7 +657,7 @@ def fsdec(txt):
     return txt.decode(FS_ENCODING, "surrogateescape")
 
 
-def fsenc(txt):
+def w8enc(txt):
     """encodes wtf8 to filesystem-bytes"""
     if PY2:
         return surrogateescape.encodefilename(txt)
@@ -395,6 +665,64 @@ def fsenc(txt):
     return txt.encode(FS_ENCODING, "surrogateescape")
 
 
+def w8b64dec(txt):
+    """decodes base64(filesystem-bytes) to wtf8"""
+    return w8dec(base64.urlsafe_b64decode(txt.encode("ascii")))
+
+
+def w8b64enc(txt):
+    """encodes wtf8 to base64(filesystem-bytes)"""
+    return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii")
+
+
+if PY2 and WINDOWS:
+    # moonrunes become \x3f with bytestrings,
+    # losing mojibake support is worth
+    def _not_actually_mbcs(txt):
+        return txt
+
+    fsenc = _not_actually_mbcs
+    fsdec = _not_actually_mbcs
+else:
+    fsenc = w8enc
+    fsdec = w8dec
+
+
+def s3enc(mem_cur, rd, fn):
+    ret = []
+    for v in [rd, fn]:
+        try:
+            mem_cur.execute("select * from a where b = ?", (v,))
+            ret.append(v)
+        except:
+            ret.append("//" + w8b64enc(v))
+            # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+    return tuple(ret)
+
+
+def s3dec(rd, fn):
+    ret = []
+    for k, v in [["d", rd], ["f", fn]]:
+        if v.startswith("//"):
+            ret.append(w8b64dec(v[2:]))
+            # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+        else:
+            ret.append(v)
+
+    return tuple(ret)
+
+
+def atomic_move(src, dst):
+    if not PY2:
+        os.replace(src, dst)
+    else:
+        if os.path.exists(dst):
+            os.unlink(dst)
+
+        os.rename(src, dst)
+
+
 def read_socket(sr, total_size):
     remains = total_size
     while remains > 0:
@@ -404,12 +732,65 @@ def read_socket(sr, total_size):
 
         buf = sr.recv(bufsz)
         if not buf:
-            raise Pebkac(400, "client disconnected during binary post")
+            raise Pebkac(400, "client d/c during binary post")
 
         remains -= len(buf)
         yield buf
 
 
+def read_socket_unbounded(sr):
+    while True:
+        buf = sr.recv(32 * 1024)
+        if not buf:
+            return
+
+        yield buf
+
+
+def read_socket_chunked(sr, log=None):
+    err = "expected chunk length, got [{}] |{}| instead"
+    while True:
+        buf = b""
+        while b"\r" not in buf:
+            rbuf = sr.recv(2)
+            if not rbuf or len(buf) > 16:
+                err = err.format(buf.decode("utf-8", "replace"), len(buf))
+                raise Pebkac(400, err)
+
+            buf += rbuf
+
+        if not buf.endswith(b"\n"):
+            sr.recv(1)
+
+        try:
+            chunklen = int(buf.rstrip(b"\r\n"), 16)
+        except:
+            err = err.format(buf.decode("utf-8", "replace"), len(buf))
+            raise Pebkac(400, err)
+
+        if chunklen == 0:
+            sr.recv(2)  # \r\n after final chunk
+            return
+
+        if log:
+            log("receiving {} byte chunk".format(chunklen))
+
+        for chunk in read_socket(sr, chunklen):
+            yield chunk
+
+        sr.recv(2)  # \r\n after each chunk too
+
+
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()
@@ -429,6 +810,73 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
+def sendfile_py(lower, upper, f, s):
+    remains = upper - lower
+    f.seek(lower)
+    while remains > 0:
+        # time.sleep(0.01)
+        buf = f.read(min(4096, remains))
+        if not buf:
+            return remains
+
+        try:
+            s.sendall(buf)
+            remains -= len(buf)
+        except:
+            return remains
+
+    return 0
+
+
+def sendfile_kern(lower, upper, f, s):
+    out_fd = s.fileno()
+    in_fd = f.fileno()
+    ofs = lower
+    while ofs < upper:
+        try:
+            req = min(2 ** 30, upper - ofs)
+            select.select([], [out_fd], [], 10)
+            n = os.sendfile(out_fd, in_fd, ofs, req)
+        except Exception as ex:
+            # print("sendfile: " + repr(ex))
+            n = 0
+
+        if n <= 0:
+            return upper - ofs
+
+        ofs += n
+        # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
+
+    return 0
+
+
+def statdir(logger, scandir, lstat, top):
+    try:
+        btop = fsenc(top)
+        if scandir and hasattr(os, "scandir"):
+            src = "scandir"
+            with os.scandir(btop) as dh:
+                for fh in dh:
+                    try:
+                        yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
+                    except Exception as ex:
+                        msg = "scan-stat: \033[36m{} @ {}"
+                        logger(msg.format(repr(ex), fsdec(fh.path)))
+        else:
+            src = "listdir"
+            fun = os.lstat if lstat else os.stat
+            for name in os.listdir(btop):
+                abspath = os.path.join(btop, name)
+                try:
+                    yield [fsdec(name), fun(abspath)]
+                except Exception as ex:
+                    msg = "list-stat: \033[36m{} @ {}"
+                    logger(msg.format(repr(ex), fsdec(abspath)))
+
+    except Exception as ex:
+        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
+
+
 def unescape_cookie(orig):
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""
@@ -457,6 +905,13 @@ def unescape_cookie(orig):
     return ret
 
 
+def guess_mime(url):
+    if url.endswith(".md"):
+        return ["text/plain; charset=UTF-8"]
+
+    return mimetypes.guess_type(url)
+
+
 def runcmd(*argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
@@ -476,22 +931,56 @@ def chkcmd(*argv):
 def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
-        return struct.unpack(b"I", f.read(4))[0]
+        rv = f.read(4)
+        try:
+            return struct.unpack(b"I", rv)[0]
+        except:
+            return struct.unpack("I", rv)[0]
 
 
 def py_desc():
+    interp = platform.python_implementation()
     py_ver = ".".join([str(x) for x in sys.version_info])
     ofs = py_ver.find(".final.")
     if ofs > 0:
         py_ver = py_ver[:ofs]
 
-    bitness = struct.calcsize(b"P") * 8
-    host_os = platform.system()
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
 
-    return "{0} on {1}{2}".format(py_ver, host_os, bitness)
+    host_os = platform.system()
+    compiler = platform.python_compiler()
+
+    os_ver = re.search(r"([0-9]+\.[0-9\.]+)", platform.version())
+    os_ver = os_ver.group(1) if os_ver else ""
+
+    return "{:>9} v{} on {}{} {} [{}]".format(
+        interp, py_ver, host_os, bitness, os_ver, compiler
+    )
+
+
+def align_tab(lines):
+    rows = []
+    ncols = 0
+    for ln in lines:
+        row = [x for x in ln.split(" ") if x]
+        ncols = max(ncols, len(row))
+        rows.append(row)
+
+    lens = [0] * ncols
+    for row in rows:
+        for n, col in enumerate(row):
+            lens[n] = max(lens[n], len(col))
+
+    return ["".join(x.ljust(y + 2) for x, y in zip(row, lens)) for row in rows]
 
 
 class Pebkac(Exception):
     def __init__(self, code, msg=None):
         super(Pebkac, self).__init__(msg or HTTPCODE[code])
         self.code = code
+
+    def __repr__(self):
+        return "Pebkac({}, {})".format(self.code, repr(self.args))

copyparty/web/Makefile

@@ -0,0 +1,12 @@
+# run me to zopfli all the static files
+# which should help on really slow connections
+# but then why are you using copyparty in the first place
+
+pk: $(addsuffix .gz, $(wildcard *.js *.css))
+un: $(addsuffix .un, $(wildcard *.gz))
+
+%.gz: %
+	pigz -11 -J 34 -I 5730 $<
+
+%.un: %
+	pigz -d $<

copyparty/web/browser.css

@@ -36,18 +36,25 @@ body {
 	padding: .35em .5em .2em .5em;
 	border-radius: 0 .3em .3em 0;
 	box-shadow: .1em .1em .4em #222;
-	margin: 2em 0 1em 0;
+	margin: 1.3em 0 0 0;
 	font-size: 1.4em;
 }
+#path #entree {
+	margin-left: -.7em;
+}
 #files {
-	border-collapse: collapse;
-	margin-top: 1em;
+	border-spacing: 0;
+	z-index: 1;
+	position: relative;
 }
 #files tbody a {
 	display: block;
 	padding: .3em 0;
 }
-a {
+#files tbody div a {
+	color: #f5a;
+}
+a, #files tbody div a:last-child {
 	color: #fc5;
 	padding: .2em;
 	text-decoration: none;
@@ -55,20 +62,22 @@ a {
 #files a:hover {
 	color: #fff;
 	background: #161616;
+	text-decoration: underline;
 }
 #files thead a {
 	color: #999;
 	font-weight: normal;
 }
-#files tr:hover {
+#files tr+tr:hover {
 	background: #1c1c1c;
 }
 #files thead th {
 	padding: .5em 1.3em .3em 1.3em;
+	cursor: pointer;
 }
 #files thead th:last-child {
 	background: #444;
-	border-radius: .7em 0 0 0;
+	border-radius: .7em .7em 0 0;
 }
 #files thead th:first-child {
 	background: #222;
@@ -82,6 +91,16 @@ a {
 	margin: 0;
 	padding: 0 .5em;
 }
+#files td {
+	border-bottom: 1px solid #111;
+}
+#files td+td+td {
+	max-width: 30em;
+	overflow: hidden;
+}
+#files tr+tr td {
+	border-top: 1px solid #383838;
+}
 #files tbody td:nth-child(3) {
 	font-family: monospace;
 	font-size: 1.3em;
@@ -100,6 +119,9 @@ a {
 	padding-bottom: 1.3em;
 	border-bottom: .5em solid #444;
 }
+#files tbody tr td:last-child {
+	white-space: nowrap;
+}
 #files thead th[style] {
 	width: auto !important;
 }
@@ -131,11 +153,39 @@ a {
 .logue {
 	padding: .2em 1.5em;
 }
-a.play {
-	color: #e70;
+.logue:empty {
+	display: none;
 }
-a.play.act {
-	color: #af0;
+#pro.logue {
+	margin-bottom: .8em;
+}
+#epi.logue {
+	margin: .8em 0;
+}
+#srv_info {
+	opacity: .5;
+	font-size: .8em;
+	color: #fc5;
+	position: absolute;
+	top: .5em;
+	left: 2em;
+}
+#srv_info span {
+	color: #fff;
+}
+#files tbody a.play {
+	color: #e70;
+	padding: .2em;
+	margin: -.2em;
+}
+#files tbody a.play.act {
+	color: #840;
+	text-shadow: 0 0 .3em #b80;
+}
+#files tbody tr.sel td {
+	background: #80b;
+	color: #fff;
+	border-color: #a3d;
 }
 #blocked {
 	position: fixed;
@@ -145,7 +195,7 @@ a.play.act {
 	height: 100%;
 	background: #333;
 	font-size: 2.5em;
-	z-index:99;
+	z-index: 99;
 }
 #blk_play,
 #blk_abrt {
@@ -179,6 +229,7 @@ a.play.act {
 	bottom: -6em;
 	height: 6em;
 	width: 100%;
+	z-index: 3;
 	transition: bottom 0.15s;
 }
 #widget.open {
@@ -203,6 +254,9 @@ a.play.act {
 	75% {cursor: url(/.cpr/dd/5.png), pointer}
 	85% {cursor: url(/.cpr/dd/1.png), pointer}
 }
+@keyframes spin {
+	100% {transform: rotate(360deg)}
+}
 #wtoggle {
 	position: absolute;
 	top: -1.2em;
@@ -219,6 +273,25 @@ a.play.act {
 	padding: .2em 0 0 .07em;
 	color: #fff;
 }
+#wtoggle>span {
+	display: none;
+}
+#wtoggle.sel {
+	width: 4.27em;
+}
+#wtoggle.sel>span {
+	display: inline-block;
+	line-height: 0;
+}
+#wtoggle.sel>span a {
+	font-size: .4em;
+	margin: -.3em 0;
+	position: relative;
+	display: inline-block;
+}
+#wtoggle.sel>span #selzip {
+	top: -.6em;
+}
 #barpos,
 #barbuf {
 	position: absolute;
@@ -262,3 +335,344 @@ a.play.act {
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
+@media (min-width: 90em) {
+	#barpos,
+	#barbuf {
+		width: calc(100% - 24em);
+		left: 9.8em;
+		top: .7em;
+		height: 1.6em;
+		bottom: auto;
+	}
+	#widget {
+		bottom: -3.2em;
+		height: 3.2em;
+	}
+}
+
+
+
+.opview {
+	display: none;
+}
+.opview.act {
+	display: block;
+}
+#ops a {
+	color: #fc5;
+	font-size: 1.5em;
+	padding: .25em .3em;
+	margin: 0;
+	outline: none;
+}
+#ops a.act {
+	background: #281838;
+	border-radius: 0 0 .2em .2em;
+	border-bottom: .3em solid #d90;
+	box-shadow: 0 -.15em .2em #000 inset;
+	padding-bottom: .3em;
+}
+#ops i {
+	font-size: 1.5em;
+}
+#ops i:before {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #01a7e1;
+	position: relative;
+}
+#ops i:after {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #ff3f1a;
+	margin-left: -.35em;
+	font-size: 1.05em;
+}
+#ops,
+.opbox {
+	border: 1px solid #3a3a3a;
+	box-shadow: 0 0 1em #222 inset;
+}
+#ops {
+	background: #333;
+	margin: 1.7em 1.5em 0 1.5em;
+	padding: .3em .6em;
+	border-radius: .3em;
+	border-width: .15em 0;
+}
+.opbox {
+	background: #2d2d2d;
+	margin: 1.5em 0 0 0;
+	padding: .5em;
+	border-radius: 0 1em 1em 0;
+	border-width: .15em .3em .3em 0;
+	max-width: 40em;
+}
+.opbox input {
+	margin: .5em;
+}
+.opview input[type=text] {
+	color: #fff;
+	background: #383838;
+	border: none;
+	box-shadow: 0 0 .3em #222;
+	border-bottom: 1px solid #fc5;
+	border-radius: .2em;
+	padding: .2em .3em;
+}
+input[type="checkbox"]+label {
+	color: #f5a;
+}
+input[type="checkbox"]:checked+label {
+	color: #fc5;
+}
+
+
+
+#srch_form {
+	border: 1px solid #3a3a3a;
+	box-shadow: 0 0 1em #222 inset;
+	background: #2d2d2d;
+	border-radius: .4em;
+	margin: 1.4em;
+	margin-bottom: 0;
+	padding: 0 .5em .5em 0;
+}
+#srch_form table {
+	display: inline-block;
+}
+#srch_form td {
+	padding: .6em .6em;
+}
+#srch_form td:first-child {
+	width: 3em;
+	padding-right: .2em;
+	text-align: right;
+}
+#op_search input {
+	margin: 0;
+}
+#srch_q {
+	white-space: pre;
+	color: #f80;
+	height: 1em;
+	margin: .2em 0 -1em 1.6em;
+}
+#files td div span {
+	color: #fff;
+	padding: 0 .4em;
+	font-weight: bold;
+	font-style: italic;
+}
+#files td div a:hover {
+	background: #444;
+	color: #fff;
+}
+#files td div a {
+	display: inline-block;
+	white-space: nowrap;
+}
+#files td div a:last-child {
+	width: 100%;
+}
+#files td div {
+	border-collapse: collapse;
+	width: 100%;
+}
+#files td div a:last-child {
+	width: 100%;
+}
+#wrap {
+	margin-top: 2em;
+}
+#tree {
+	display: none;
+	position: fixed;
+	left: 0;
+	bottom: 0;
+	top: 7em;
+	padding-top: .2em;
+	overflow-y: auto;
+	-ms-scroll-chaining: none;
+	overscroll-behavior-y: none;
+	scrollbar-color: #eb0 #333;
+}
+#thx_ff {
+	padding: 5em 0;
+}
+#tree::-webkit-scrollbar-track {
+	background: #333;
+}
+#tree::-webkit-scrollbar {
+	background: #333;
+}
+#tree::-webkit-scrollbar-thumb {
+	background: #eb0;
+}
+#tree:hover {
+	z-index: 2;
+}
+#treeul {
+	position: relative;
+	left: -1.7em;
+	width: calc(100% + 1.3em);
+}
+.tglbtn,
+#tree>a+a {
+	padding: .2em .4em;
+	font-size: 1.2em;
+	background: #2a2a2a;
+	box-shadow: 0 .1em .2em #222 inset;
+	border-radius: .3em;
+	margin: .2em;
+	position: relative;
+	top: -.2em;
+}
+.tglbtn:hover,
+#tree>a+a:hover {
+	background: #805;
+}
+.tglbtn.on,
+#tree>a+a.on {
+	background: #fc4;
+	color: #400;
+	text-shadow: none;
+}
+#detree {
+	padding: .3em .5em;
+	font-size: 1.5em;
+}
+#tree ul,
+#tree li {
+	padding: 0;
+	margin: 0;
+}
+#tree ul {
+	border-left: .2em solid #555;
+}
+#tree li {
+	margin-left: 1em;
+	list-style: none;
+	border-top: 1px solid #4c4c4c;
+	border-bottom: 1px solid #222;
+}
+#tree li:last-child {
+	border-bottom: none;
+}
+#treeul a.hl {
+	color: #400;
+	background: #fc4;
+	border-radius: .3em;
+	text-shadow: none;
+}
+#treeul a {
+	display: inline-block;
+}
+#treeul a+a {
+	width: calc(100% - 2em);
+	background: #333;
+	line-height: 1em;
+}
+#treeul a+a:hover {
+	background: #222;
+	color: #fff;
+}
+#treeul a:first-child {
+	font-family: monospace, monospace;
+}
+.dumb_loader_thing {
+	display: inline-block;
+	margin: 1em .3em 1em 1em;
+	padding: 0 1.2em 0 0;
+	font-size: 4em;
+	animation: spin 1s linear infinite;
+	position: absolute;
+	z-index: 9;
+}
+#files .cfg {
+	display: none;
+	font-size: 2em;
+	white-space: nowrap;
+}
+#files th:hover .cfg,
+#files th.min .cfg {
+	display: block;
+	width: 1em;
+	border-radius: .2em;
+	margin: -1.3em auto 0 auto;
+	background: #444;
+}
+#files th.min .cfg {
+	margin: -.6em;
+}
+#files>thead>tr>th.min span {
+	position: absolute;
+	transform: rotate(270deg);
+	background: linear-gradient(90deg, rgba(68,68,68,0), rgba(68,68,68,0.5) 70%, #444);
+	margin-left: -4.6em;
+	padding: .4em;
+	top: 5.4em;
+	width: 8em;
+	text-align: right;
+	letter-spacing: .04em;
+}
+#files td:nth-child(2n) {
+	color: #f5a;
+}
+#files td.min a {
+	display: none;
+}
+#files tr.play td {
+	background: #fc4;
+	border-color: transparent;
+	color: #400;
+	text-shadow: none;
+}
+#files tr.play a {
+	color: inherit;
+}
+#files tr.play a:hover {
+	color: #300;
+	background: #fea;
+}
+#op_cfg {
+	max-width: none;
+	margin-right: 1.5em;
+}
+#op_cfg>div>a {
+	line-height: 2em;
+}
+#op_cfg>div>span {
+	display: inline-block;
+	padding: .2em .4em;
+}
+#op_cfg h3 {
+	margin: .8em 0 0 .6em;
+	padding: 0;
+	border-bottom: 1px solid #555;
+}
+#opdesc {
+	display: none;
+}
+#ops:hover #opdesc {
+	display: block;
+	background: linear-gradient(0deg,#555, #4c4c4c 80%, #444);
+	box-shadow: 0 .3em 1em #222;
+	padding: 1em;
+	border-radius: .3em;
+	position: absolute;
+	z-index: 3;
+	top: 6em;
+	right: 1.5em;
+}
+#ops:hover #opdesc.off {
+	display: none;
+}
+#opdesc code {
+	background: #3c3c3c;
+	padding: .2em .3em;
+	border-top: 1px solid #777;
+	border-radius: .3em;
+	font-family: monospace, monospace;
+	line-height: 2em;
+}

copyparty/web/browser.html

@@ -3,58 +3,123 @@
 
 <head>
     <meta charset="utf-8">
-    <title>copyparty</title>
+    <title>⇆🎉 {{ title }}</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
-    {%- if can_upload %}
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
-    {%- endif %}
 </head>
 
 <body>
+    <div id="ops">
+        <a href="#" data-dest="" data-desc="close submenu">---</a>
+        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
+        {%- if have_up2k_idx %}
+        <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
+        {%- else %}
+        <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
+        {%- endif %}
+        <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
+        <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
+        <a href="#" data-perm="write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
+        <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
+        <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
+        <div id="opdesc"></div>
+    </div>
+
+    <div id="op_search" class="opview">
+        {%- if have_tags_idx %}
+        <div id="srch_form" class="tags"></div>
+        {%- else %}
+        <div id="srch_form"></div>
+        {%- endif %}
+        <div id="srch_q"></div>
+    </div>
+
+    {%- include 'upload.html' %}
+
+    <div id="op_cfg" class="opview opbox">
+        <h3>key notation</h3>
+        <div id="key_notation"></div>
+        {%- if have_zip %}
+        <h3>folder download</h3>
+        <div id="arc_fmt"></div>
+        {%- endif %}
+        <h3>tooltips</h3>
+        <div><a id="tooltips" class="tglbtn" href="#">enable</a></div>
+    </div>
+    
     <h1 id="path">
+        <a href="#" id="entree">🌲</a>
         {%- for n in vpnodes %}
         <a href="/{{ n[0] }}">{{ n[1] }}</a>
         {%- endfor %}
     </h1>
     
-    {%- if can_upload %}
-    {%- include 'upload.html' %}
-    {%- endif %}
-    
-    {%- if can_read %}
-    {%- if prologue %}
-    <div id="pro" class="logue">{{ prologue }}</div>
-    {%- endif %}
+    <div id="tree">
+        <a href="#" id="detree">🍞...</a>
+        <a href="#" step="2" id="twobytwo">+</a>
+        <a href="#" step="-2" id="twig">&ndash;</a>
+        <a href="#" class="tglbtn" id="dyntree">a</a>
+        <ul id="treeul"></ul>
+        <div id="thx_ff">&nbsp;</div>
+    </div>
+
+<div id="wrap">
+
+    <div id="pro" class="logue">{{ logues[0] }}</div>
 
     <table id="files">
         <thead>
             <tr>
-                <th></th>
-                <th>File Name</th>
-                <th>File Size</th>
-                <th>Date</th>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
+                {%- for k in taglist %}
+                    {%- if k.startswith('.') %}
+                        <th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
+                    {%- else %}
+                        <th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
+                    {%- endif %}
+                {%- endfor %}
+                <th name="ext"><span>T</span></th>
+                <th name="ts"><span>Date</span></th>
             </tr>
         </thead>
         <tbody>
 
 {%- for f in files %}
-<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td></tr>
+    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td>
+    {%- if f.tags is defined %}
+        {%- for k in taglist %}
+            <td>{{ f.tags[k] }}</td>
+        {%- endfor %}
+    {%- endif %}
+    <td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
         </tbody>
     </table>
     
-    {%- if epilogue %}
-    <div id="epi" class="logue">{{ epilogue }}</div>
-    {%- endif %}
-    {%- endif %}
+    <div id="epi" class="logue">{{ logues[1] }}</div>
 
     <h2><a href="?h">control-panel</a></h2>
 
+</div>
+
+    {%- if srv_info %}
+    <div id="srv_info"><span>{{ srv_info }}</span></div>
+    {%- endif %}
+
     <div id="widget">
-        <div id="wtoggle">♫</div>
+        <div id="wtoggle">
+            <span>
+                <a href="#" id="selall">sel.<br />all</a>
+                <a href="#" id="selinv">sel.<br />inv.</a>
+                <a href="#" id="selzip">zip</a>
+            </span>
+            ♫
+        </div>
         <div id="widgeti">
             <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
             <canvas id="pvol" width="288" height="38"></canvas>
@@ -62,14 +127,16 @@
             <canvas id="barbuf"></canvas>
         </div>
     </div>
-    
-    {%- if can_read %}
+
+    <script>
+        var tag_order_cfg = {{ tag_order }};
+    </script>
+    <script src="/.cpr/util.js{{ ts }}"></script>
     <script src="/.cpr/browser.js{{ ts }}"></script>
-    {%- endif %}
-    
-    {%- if can_upload %}
     <script src="/.cpr/up2k.js{{ ts }}"></script>
-    {%- endif %}
+    <script>
+        apply_perms({{ perms }});
+    </script>
 </body>
 
 </html>

copyparty/web/md.css

@@ -0,0 +1,602 @@
+@font-face {
+	font-family: 'scp';
+	src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(/.cpr/deps/scp.woff2) format('woff2');
+}
+html, body {
+	color: #333;
+	background: #eee;
+	font-family: sans-serif;
+	line-height: 1.5em;
+}
+#mtw {
+	display: none;
+}
+#mw {
+	margin: 0 auto;
+	padding: 0 1.5em;
+}
+pre, code, a {
+	color: #480;
+	background: #f7f7f7;
+	border: .07em solid #ddd;
+	border-radius: .2em;
+	padding: .1em .3em;
+	margin: 0 .1em;
+}
+code {
+	font-size: .96em;
+}
+pre, code {
+	font-family: 'scp', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-all;
+}
+pre {
+	counter-reset: precode;
+}
+pre code {
+	counter-increment: precode;
+	display: inline-block;
+	margin: 0 -.3em;
+	padding: .4em .5em;
+	border: none;
+	border-bottom: 1px solid #cdc;
+	min-width: calc(100% - .6em);
+	line-height: 1.1em;
+}
+pre code:last-child {
+	border-bottom: none;
+}
+pre code::before {
+	content: counter(precode);
+	-webkit-user-select: none;
+	display: inline-block;
+	text-align: right;
+	font-size: .75em;
+	color: #48a;
+	width: 4em;
+	padding-right: 1.5em;
+	margin-left: -5.5em;
+}
+pre code:hover {
+	background: #fec;
+	color: #360;
+}
+h1, h2 {
+	line-height: 1.5em;
+}
+h1 {
+	font-size: 1.7em;
+	text-align: center;
+	border: 1em solid #777;
+	border-width: .05em 0;
+	margin: 3em 0;
+}
+h2 {
+	font-size: 1.5em;
+	font-weight: normal;
+	background: #f7f7f7;
+	border-top: .07em solid #fff;
+	border-bottom: .07em solid #bbb;
+	border-radius: .5em .5em 0 0;
+	padding-left: .4em;
+	margin-top: 3em;
+}
+h3 {
+	border-bottom: .1em solid #999;
+}
+h1 a, h3 a, h5 a,
+h2 a, h4 a, h6 a {
+	color: inherit;
+	display: block;
+	background: none;
+	border: none;
+	padding: 0;
+	margin: 0;
+}
+#mp ul,
+#mp ol {
+	border-left: .3em solid #ddd;
+}
+#m>ul,
+#m>ol {
+	border-color: #bbb;
+}
+#mp ul>li {
+	list-style-type: disc;
+}
+#mp ul>li,
+#mp ol>li {
+	margin: .7em 0;
+}
+strong {
+	color: #000;
+}
+p>em,
+li>em,
+td>em {
+	color: #c50;
+	padding: .1em;
+	border-bottom: .1em solid #bbb;
+}
+blockquote {
+	font-family: serif;
+	background: #f7f7f7;
+	border: .07em dashed #ccc;
+	padding: 0 2em;
+	margin: 1em 0;
+}
+small {
+	opacity: .8;
+}
+#toc {
+	margin: 0 1em;
+	-ms-scroll-chaining: none;
+	overscroll-behavior-y: none;
+}
+#toc ul {
+	padding-left: 1em;
+}
+#toc>ul {
+	text-align: left;
+	padding-left: .5em;
+}
+#toc li {
+	list-style-type: none;
+	line-height: 1.2em;
+	margin: .5em 0;
+}
+#toc a {
+	color: #057;
+	border: none;
+	background: none;
+	display: block;
+	margin-left: -.3em;
+	padding: .2em .3em;
+}
+#toc a.act {
+	color: #fff;
+	background: #07a;
+}
+.todo_pend,
+.todo_done {
+	z-index: 99;
+	position: relative;
+	display: inline-block;
+	font-family: monospace, monospace;
+	font-weight: bold;
+	font-size: 1.3em;
+	line-height: .1em;
+	margin: -.5em 0 -.5em -.85em;
+	top: .1em;
+	color: #b29;
+}
+.todo_done {
+	color: #6b3;
+	text-shadow: .02em 0 0 #6b3;
+}
+table {
+	border-collapse: collapse;
+	margin: 1em 0;
+}
+th, td {
+	padding: .2em .5em;
+	border: .12em solid #aaa;
+}
+blink {
+	animation: blinker .7s cubic-bezier(.9, 0, .1, 1) infinite;
+}
+@keyframes blinker {
+	10% {
+		opacity: 0;
+	}
+	60% {
+		opacity: 1;
+	}
+}
+
+@media screen {
+	html, body {
+		margin: 0;
+		padding: 0;
+		outline: 0;
+		border: none;
+		width: 100%;
+		height: 100%;
+	}
+	#mw {
+		margin: 0 auto;
+		right: 0;
+	}
+	#mp {
+		max-width: 52em;
+		margin-bottom: 6em;
+		word-break: break-word;
+		overflow-wrap: break-word;
+		word-wrap: break-word; /*ie*/
+	}
+	a {
+		color: #fff;
+		background: #39b;
+		text-decoration: none;
+		padding: 0 .3em;
+		border: none;
+		border-bottom: .07em solid #079;
+	}
+	h2 {
+		color: #fff;
+		background: #555;
+		margin-top: 2em;
+		border-bottom: .22em solid #999;
+		border-top: none;
+	}
+	h1 {
+		color: #fff;
+		background: #444;
+		font-weight: normal;
+		border-top: .4em solid #fb0;
+		border-bottom: .4em solid #777;
+		border-radius: 0 1em 0 1em;
+		margin: 3em 0 1em 0;
+		padding: .5em 0;
+	}
+	#mn {
+		padding: 1.3em 0 .7em 1em;
+		border-bottom: 1px solid #ccc;
+		background: #eee;
+		z-index: 10;
+		width: calc(100% - 1em);
+	}
+	#mn a {
+		color: #444;
+		background: none;
+		margin: 0 0 0 -.2em;
+		padding: .3em 0 .3em .4em;
+		text-decoration: none;
+		border: none;
+		/* ie: */
+		border-bottom: .1em solid #777\9;
+		margin-right: 1em\9;
+	}
+	#mn a:first-child {
+		padding-left: .5em;
+	}
+	#mn a:last-child {
+		padding-right: .5em;
+	}
+	#mn a:not(:last-child)::after {
+		content: '';
+		width: 1.05em;
+		height: 1.05em;
+		margin: -.2em .3em -.2em -.4em;
+		display: inline-block;
+		border: 1px solid rgba(0,0,0,0.2);
+		border-width: .2em .2em 0 0;
+		transform: rotate(45deg);
+	}
+	#mn a:hover {
+		color: #000;
+		text-decoration: underline;
+	}
+	#mh {
+		padding: .4em 1em;
+		position: relative;
+		width: 100%;
+		width: calc(100% - 3em);
+		background: #eee;
+		z-index: 9;
+		top: 0;
+	}
+	#mh a {
+		color: #444;
+		background: none;
+		text-decoration: underline;
+		border: none;
+	}
+	#mh a:hover {
+		color: #000;
+		background: #ddd;
+	}
+	#toolsbox {
+		overflow: hidden;
+		display: inline-block;
+		background: #eee;
+		height: 1.5em;
+		padding: 0 .2em;
+		margin: 0 .2em;
+		position: absolute;
+	}
+	#toolsbox.open {
+		height: auto;
+		overflow: visible;
+		background: #eee;
+		box-shadow: 0 .2em .2em #ccc;
+		padding-bottom: .2em;
+	}
+	#toolsbox a {
+		display: block;
+	}
+	#toolsbox a+a {
+		text-decoration: none;
+	}
+
+
+
+	html.dark,
+	html.dark body {
+		background: #222;
+		color: #ccc;
+	}
+	html.dark #toc a {
+		color: #ccc;
+		border-left: .4em solid #444;
+		border-bottom: .1em solid #333;
+	}
+	html.dark #toc a.act {
+		color: #fff;
+		border-left: .4em solid #3ad;
+	}
+	html.dark #toc li {
+		border-width: 0;
+	}
+	html.dark #mp a {
+		background: #057;
+	}
+	html.dark #mp h1 a, html.dark #mp h4 a,
+	html.dark #mp h2 a, html.dark #mp h5 a,
+	html.dark #mp h3 a, html.dark #mp h6 a {
+		color: inherit;
+		background: none;
+	}
+	html.dark pre,
+	html.dark code {
+		color: #8c0;
+		background: #1a1a1a;
+		border: .07em solid #333;
+	}
+	html.dark #mp ul,
+	html.dark #mp ol {
+		border-color: #444;
+	}
+	html.dark #m>ul,
+	html.dark #m>ol {
+		border-color: #555;
+	}
+	html.dark strong {
+		color: #fff;
+	}
+	html.dark p>em,
+	html.dark li>em,
+	html.dark td>em {
+		color: #f94;
+		border-color: #666;
+	}
+	html.dark h1 {
+		background: #383838;
+		border-top: .4em solid #b80;
+		border-bottom: .4em solid #4c4c4c;
+	}
+	html.dark h2 {
+		background: #444;
+		border-bottom: .22em solid #555;
+	}
+	html.dark td,
+	html.dark th {
+		border-color: #444;
+	}
+	html.dark blockquote {
+		background: #282828;
+		border: .07em dashed #444;
+	}
+	html.dark #mn a:not(:last-child)::after {
+		border-color: rgba(255,255,255,0.3);
+	}
+	html.dark #mn a {
+		color: #ccc;
+	}
+	html.dark #mn {
+		border-bottom: 1px solid #333;
+	}
+	html.dark #mn,
+	html.dark #mh {
+		background: #222;
+	}
+	html.dark #mh a {
+		color: #ccc;
+		background: none;
+	}
+	html.dark #mh a:hover {
+		background: #333;
+		color: #fff;
+	}
+	html.dark #toolsbox {
+		background: #222;
+	}
+	html.dark #toolsbox.open {
+		box-shadow: 0 .2em .2em #069;
+		border-radius: 0 0 .4em .4em;
+	}
+}
+
+@media screen and (min-width: 66em) {
+	#mw {
+		position: fixed;
+		overflow-y: auto;
+		left: 14em;
+		left: calc(100% - 55em);
+		max-width: none;
+		bottom: 0;
+		scrollbar-color: #eb0 #f7f7f7;
+	}
+	#toc {
+		width: 13em;
+		width: calc(100% - 55.3em);
+		max-width: 30em;
+		background: #eee;
+		position: fixed;
+		overflow-y: auto;
+		top: 0;
+		left: 0;
+		bottom: 0;
+		padding: 0;
+		margin: 0;
+		scrollbar-color: #eb0 #f7f7f7;
+		box-shadow: 0 0 1em rgba(0,0,0,0.1);
+		border-top: 1px solid #d7d7d7;
+	}
+	#toc li {
+		border-left: .3em solid #ccc;
+	}
+	#toc::-webkit-scrollbar-track {
+		background: #f7f7f7;
+	}
+	#toc::-webkit-scrollbar {
+		background: #f7f7f7;
+		width: .8em;
+	}
+	#toc::-webkit-scrollbar-thumb {
+		background: #eb0;
+	}
+
+
+	
+	html.dark #toc {
+		background: #282828;
+		border-top: 1px solid #2c2c2c;
+		box-shadow: 0 0 1em #181818;
+	}
+	html.dark #toc,
+	html.dark #mw {
+		scrollbar-color: #b80 #282828;
+	}
+	html.dark #toc::-webkit-scrollbar-track {
+		background: #282828;
+	}
+	html.dark #toc::-webkit-scrollbar {
+		background: #282828;
+		width: .8em;
+	}
+	html.dark #toc::-webkit-scrollbar-thumb {
+		background: #b80;
+	}
+}
+@media screen and (min-width: 85.5em) {
+	#toc { width: 30em }
+	#mw { left: 30.5em }
+}
+@media print {
+	@page {
+		size: A4;
+		padding: 0;
+		margin: .5in .6in;
+		mso-header-margin: .6in;
+		mso-footer-margin: .6in;
+		mso-paper-source: 0;
+	}
+	a {
+		color: #079;
+		text-decoration: none;
+		border-bottom: .07em solid #4ac;
+		padding: 0 .3em;
+	}
+	#toc>ul {
+		border-left: .1em solid #84c4dd;
+	}
+	#mn, #mh {
+		display: none;
+	}
+	html, body, #toc, #mw {
+		margin: 0 !important;
+		word-break: break-word;
+		width: 52em;
+	}
+	#toc {
+		margin-left: 1em !important;
+	}
+	#toc a {
+		color: #000 !important;
+	}
+	#toc a::after {
+		/* hopefully supported by browsers eventually */
+		content: leader('.') target-counter(attr(href), page);
+	}
+	a[ctr]::before {
+		content: attr(ctr) '. ';
+	}
+	h1 {
+		margin: 2em 0;
+	}
+	h2 {
+		margin: 2em 0 0 0;
+	}
+	h1, h2, h3 {
+		page-break-inside: avoid;
+	}
+	h1::after,
+	h2::after,
+	h3::after {
+		content: 'orz';
+		color: transparent;
+		display: block;
+		line-height: 1em;
+		padding: 4em 0 0 0;
+		margin: 0 0 -5em 0;
+	}
+	p {
+		page-break-inside: avoid;
+	}
+	table {
+		page-break-inside: auto;
+	}
+	tr {
+		page-break-inside: avoid;
+		page-break-after: auto;
+	}
+	thead {
+		display: table-header-group;
+	}
+	tfoot {
+		display: table-footer-group;
+	}
+	#mp a.vis::after {
+		content: ' (' attr(href) ')';
+		border-bottom: 1px solid #bbb;
+		color: #444;
+	}
+	blockquote {
+		border-color: #555;
+	}
+	code {
+		border-color: #bbb;
+	}
+	pre, pre code {
+		border-color: #999;
+	}
+	pre code::before {
+		color: #058;
+	}
+
+
+	
+	html.dark a {
+		color: #000;
+	}
+	html.dark pre,
+	html.dark code {
+		color: #240;
+	}
+	html.dark p>em,
+	html.dark li>em,
+	html.dark td>em {
+		color: #940;
+	}
+}
+
+/*
+*[data-ln]:before {
+	content: attr(data-ln);
+	font-size: .8em;
+	margin: 0 .4em;
+	color: #f0c;
+}
+*/

copyparty/web/md.html

@@ -0,0 +1,155 @@
+<!DOCTYPE html><html><head>
+	<meta charset="utf-8">
+	<title>📝🎉 {{ title }}</title> <!-- 📜 -->
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.7">
+	<link href="/.cpr/md.css" rel="stylesheet">
+	{%- if edit %}
+	<link href="/.cpr/md2.css" rel="stylesheet">
+	{%- endif %}
+</head>
+<body>
+	<div id="mn">navbar</div>
+	<div id="mh">
+		<a id="lightswitch" href="#">go dark</a>
+		<a id="navtoggle" href="#">hide nav</a>
+		{%- if edit %}
+			<a id="save" href="?edit">save</a>
+			<a id="sbs" href="#">sbs</a>
+			<a id="nsbs" href="#">editor</a>
+			<div id="toolsbox">
+				<a id="tools" href="#">tools</a>
+				<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
+				<a id="iter_uni" href="#">non-ascii: iterate (ctrl-u)</a>
+				<a id="mark_uni" href="#">non-ascii: markup</a>
+				<a id="cfg_uni" href="#">non-ascii: whitelist</a>
+				<a id="help" href="#">help</a>
+			</div>
+		{%- else %}
+			<a href="?edit">edit (basic)</a>
+			<a href="?edit2">edit (fancy)</a>
+			<a href="?raw">view raw</a>
+		{%- endif %}
+	</div>
+	<div id="toc"></div>
+	<div id="mtw">
+		<textarea id="mt" autocomplete="off">{{ md }}</textarea>
+	</div>
+	<div id="mw">
+		<div id="ml">
+			<div style="text-align:center;margin:5em 0">
+				<div style="font-size:2em;margin:1em 0">Loading</div>
+				if you're still reading this, check that javascript is allowed
+			</div>
+		</div>
+		<div id="mp"></div>
+	</div>
+	
+	{%- if edit %}
+	<div id="helpbox">
+		<textarea autocomplete="off">
+
+write markdown (most html is 🙆 too)
+
+## hotkey list
+* `Ctrl-S` to save
+* `Ctrl-E` to toggle mode
+* `Ctrl-K` to prettyprint a table
+* `Ctrl-U` to iterate non-ascii chars
+* `Ctrl-H` / `Ctrl-Shift-H` to create a header
+* `TAB` / `Shift-TAB` to indent/dedent a selection
+
+## toolbar
+1. toggle dark mode
+2. show/hide navigation bar
+3. save changes on server
+4. side-by-side editing
+5. toggle editor/preview
+6. this thing :^)
+
+## markdown
+|||
+|--|--|
+|`**bold**`|**bold**|
+|`_italic_`|_italic_|
+|`~~strike~~`|~~strike~~|
+|`` `code` ``|`code`|
+|`[](#hotkey-list)`|[](#hotkey-list)|
+|`[](/foo/bar.md#header)`|[](/foo/bar.md#header)|
+|`<blink>💯</blink>`|<blink>💯</blink>|
+
+## tables
+    |left-aligned|centered|right-aligned
+    | ---------- | :----: | ----------:
+    |one         |two     |three
+
+|left-aligned|centered|right-aligned
+| ---------- | :----: | ----------:
+|one         |two     |three
+
+## lists
+	* one
+	* two
+	1. one
+	1. two
+* one
+* two
+1. one
+1. two
+
+## headers
+	# level 1
+	## level 2
+	### level 3
+
+## quote
+	> hello
+> hello
+
+## codeblock
+		four spaces (no tab pls)
+
+## code in lists
+	* foo
+	  bar
+          six spaces total
+* foo
+  bar
+      six spaces total
+.
+		</textarea>
+	</div>
+	{%- endif %}
+	
+	<script>
+
+var last_modified = {{ lastmod }};
+var md_opt = {
+	link_md_as_html: false,
+	allow_plugins: {{ md_plug }},
+	modpoll_freq: {{ md_chk_rate }}
+};
+
+(function () {
+    var btn = document.getElementById("lightswitch");
+    var toggle = function (e) {
+		if (e) e.preventDefault();
+        var dark = !document.documentElement.getAttribute("class");
+        document.documentElement.setAttribute("class", dark ? "dark" : "");
+        btn.innerHTML = "go " + (dark ? "light" : "dark");
+        if (window.localStorage)
+            localStorage.setItem('darkmode', dark ? 1 : 0);
+    };
+    btn.onclick = toggle;
+    if (window.localStorage && localStorage.getItem('darkmode') == 1)
+		toggle();
+})();
+
+	</script>
+    <script src="/.cpr/util.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
+	<script src="/.cpr/md.js"></script>
+	{%- if edit %}
+	<script src="/.cpr/md2.js"></script>
+	{%- endif %}
+</body></html>

copyparty/web/md.js

@@ -0,0 +1,532 @@
+"use strict";
+
+var dom_toc = ebi('toc');
+var dom_wrap = ebi('mw');
+var dom_hbar = ebi('mh');
+var dom_nav = ebi('mn');
+var dom_pre = ebi('mp');
+var dom_src = ebi('mt');
+var dom_navtgl = ebi('navtoggle');
+
+
+// chrome 49 needs this
+var chromedbg = function () { console.log(arguments); }
+
+// null-logger
+var dbg = function () { };
+
+// replace dbg with the real deal here or in the console:
+// dbg = chromedbg
+// dbg = console.log
+
+
+// plugins
+var md_plug = {};
+
+
+function hesc(txt) {
+    return txt.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+
+function cls(dom, name, add) {
+    var re = new RegExp('(^| )' + name + '( |$)');
+    var lst = (dom.getAttribute('class') + '').replace(re, "$1$2").replace(/  /, "");
+    dom.setAttribute('class', lst + (add ? ' ' + name : ''));
+}
+
+
+function statify(obj) {
+    return JSON.parse(JSON.stringify(obj));
+}
+
+
+// dodge browser issues
+(function () {
+    var ua = navigator.userAgent;
+    if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
+        // necessary on ff-68.7 at least
+        var s = document.createElement('style');
+        s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
+        console.log(s.innerHTML);
+        document.head.appendChild(s);
+    }
+})();
+
+
+// add navbar
+(function () {
+    var n = document.location + '';
+    n = n.substr(n.indexOf('//') + 2).split('?')[0].split('/');
+    n[0] = 'top';
+    var loc = [];
+    var nav = [];
+    for (var a = 0; a < n.length; a++) {
+        if (a > 0)
+            loc.push(n[a]);
+
+        var dec = hesc(uricom_dec(n[a])[0]);
+
+        nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
+    }
+    dom_nav.innerHTML = nav.join('');
+})();
+
+
+// faster than replacing the entire html (chrome 1.8x, firefox 1.6x)
+function copydom(src, dst, lv) {
+    var sc = src.childNodes,
+        dc = dst.childNodes;
+
+    if (sc.length !== dc.length) {
+        dbg("replace L%d (%d/%d) |%d|",
+            lv, sc.length, dc.length, src.innerHTML.length);
+
+        dst.innerHTML = src.innerHTML;
+        return;
+    }
+
+    var rpl = [];
+    for (var a = sc.length - 1; a >= 0; a--) {
+        var st = sc[a].tagName,
+            dt = dc[a].tagName;
+
+        if (st !== dt) {
+            dbg("replace L%d (%d/%d) type %s/%s", lv, a, sc.length, st, dt);
+            rpl.push(a);
+            continue;
+        }
+
+        var sa = sc[a].attributes || [],
+            da = dc[a].attributes || [];
+
+        if (sa.length !== da.length) {
+            dbg("replace L%d (%d/%d) attr# %d/%d",
+                lv, a, sc.length, sa.length, da.length);
+
+            rpl.push(a);
+            continue;
+        }
+
+        var dirty = false;
+        for (var b = sa.length - 1; b >= 0; b--) {
+            var name = sa[b].name,
+                sv = sa[b].value,
+                dv = dc[a].getAttribute(name);
+
+            if (name == "data-ln" && sv !== dv) {
+                dc[a].setAttribute(name, sv);
+                continue;
+            }
+
+            if (sv !== dv) {
+                dbg("replace L%d (%d/%d) attr %s [%s] [%s]",
+                    lv, a, sc.length, name, sv, dv);
+
+                dirty = true;
+                break;
+            }
+        }
+        if (dirty)
+            rpl.push(a);
+    }
+
+    // TODO pure guessing
+    if (rpl.length > sc.length / 3) {
+        dbg("replace L%d fully, %s (%d/%d) |%d|",
+            lv, rpl.length, sc.length, src.innerHTML.length);
+
+        dst.innerHTML = src.innerHTML;
+        return;
+    }
+
+    // repl is reversed; build top-down
+    var nbytes = 0;
+    for (var a = rpl.length - 1; a >= 0; a--) {
+        var html = sc[rpl[a]].outerHTML;
+        dc[rpl[a]].outerHTML = html;
+        nbytes += html.length;
+    }
+    if (nbytes > 0)
+        dbg("replaced %d bytes L%d", nbytes, lv);
+
+    for (var a = 0; a < sc.length; a++)
+        copydom(sc[a], dc[a], lv + 1);
+
+    if (src.innerHTML !== dst.innerHTML) {
+        dbg("setting %d bytes L%d", src.innerHTML.length, lv);
+        dst.innerHTML = src.innerHTML;
+    }
+}
+
+
+function md_plug_err(ex, js) {
+    var errbox = ebi('md_errbox');
+    if (errbox)
+        errbox.parentNode.removeChild(errbox);
+
+    if (!ex)
+        return;
+
+    var msg = (ex + '').split('\n')[0];
+    var ln = ex.lineNumber;
+    var o = null;
+    if (ln) {
+        msg = "Line " + ln + ", " + msg;
+        var lns = js.split('\n');
+        if (ln < lns.length) {
+            o = document.createElement('span');
+            o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
+            o.textContent = lns[ln - 1];
+        }
+    }
+    errbox = document.createElement('div');
+    errbox.setAttribute('id', 'md_errbox');
+    errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
+    errbox.textContent = msg;
+    errbox.onclick = function () {
+        alert('' + ex.stack);
+    };
+    if (o) {
+        errbox.appendChild(o);
+        errbox.style.padding = '.25em .5em';
+    }
+    dom_nav.appendChild(errbox);
+
+    try {
+        console.trace();
+    }
+    catch (ex2) { }
+}
+
+
+function load_plug(md_text, plug_type) {
+    if (!md_opt.allow_plugins)
+        return md_text;
+
+    var find = '\n```copyparty_' + plug_type + '\n';
+    var ofs = md_text.indexOf(find);
+    if (ofs === -1)
+        return md_text;
+
+    var ofs2 = md_text.indexOf('\n```', ofs + 1);
+    if (ofs2 == -1)
+        return md_text;
+
+    var js = md_text.slice(ofs + find.length, ofs2 + 1);
+    var md = md_text.slice(0, ofs + 1) + md_text.slice(ofs2 + 4);
+
+    var old_plug = md_plug[plug_type];
+    if (!old_plug || old_plug[1] != js) {
+        js = 'const x = { ' + js + ' }; x;';
+        try {
+            var x = eval(js);
+        }
+        catch (ex) {
+            md_plug[plug_type] = null;
+            md_plug_err(ex, js);
+            return md;
+        }
+        if (x['ctor']) {
+            x['ctor']();
+            delete x['ctor'];
+        }
+        md_plug[plug_type] = [x, js];
+    }
+
+    return md;
+}
+
+
+function convert_markdown(md_text, dest_dom) {
+    md_text = md_text.replace(/\r/g, '');
+
+    md_plug_err(null);
+    md_text = load_plug(md_text, 'pre');
+    md_text = load_plug(md_text, 'post');
+
+    var marked_opts = {
+        //headerPrefix: 'h-',
+        breaks: true,
+        gfm: true
+    };
+
+    var ext = md_plug['pre'];
+    if (ext)
+        Object.assign(marked_opts, ext[0]);
+
+    try {
+        var md_html = marked(md_text, marked_opts);
+    }
+    catch (ex) {
+        if (ext)
+            md_plug_err(ex, ext[1]);
+
+        throw ex;
+    }
+    var md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
+
+    var nodes = md_dom.getElementsByTagName('a');
+    for (var a = nodes.length - 1; a >= 0; a--) {
+        var href = nodes[a].getAttribute('href');
+        var txt = nodes[a].textContent;
+
+        if (!txt)
+            nodes[a].textContent = href;
+        else if (href !== txt)
+            nodes[a].setAttribute('class', 'vis');
+    }
+
+    // todo-lists (should probably be a marked extension)
+    nodes = md_dom.getElementsByTagName('input');
+    for (var a = nodes.length - 1; a >= 0; a--) {
+        var dom_box = nodes[a];
+        if (dom_box.getAttribute('type') !== 'checkbox')
+            continue;
+
+        var dom_li = dom_box.parentNode;
+        var done = dom_box.getAttribute('checked');
+        done = done !== null;
+        var clas = done ? 'done' : 'pend';
+        var char = done ? 'Y' : 'N';
+
+        dom_li.setAttribute('class', 'task-list-item');
+        dom_li.style.listStyleType = 'none';
+        var html = dom_li.innerHTML;
+        dom_li.innerHTML =
+            '<span class="todo_' + clas + '">' + char + '</span>' +
+            html.substr(html.indexOf('>') + 1);
+    }
+
+    // separate <code> for each line in <pre>
+    nodes = md_dom.getElementsByTagName('pre');
+    for (var a = nodes.length - 1; a >= 0; a--) {
+        var el = nodes[a];
+
+        var is_precode =
+            el.tagName == 'PRE' &&
+            el.childNodes.length === 1 &&
+            el.childNodes[0].tagName == 'CODE';
+
+        if (!is_precode)
+            continue;
+
+        var nline = parseInt(el.getAttribute('data-ln')) + 1;
+        var lines = el.innerHTML.replace(/\n<\/code>$/i, '</code>').split(/\n/g);
+        for (var b = 0; b < lines.length - 1; b++)
+            lines[b] += '</code>\n<code data-ln="' + (nline + b) + '">';
+
+        el.innerHTML = lines.join('');
+    }
+
+    // self-link headers
+    var id_seen = {},
+        dyn = md_dom.getElementsByTagName('*');
+
+    nodes = [];
+    for (var a = 0, aa = dyn.length; a < aa; a++)
+        if (/^[Hh]([1-6])/.exec(dyn[a].tagName) !== null)
+            nodes.push(dyn[a]);
+
+    for (var a = 0; a < nodes.length; a++) {
+        el = nodes[a];
+        var id = el.getAttribute('id'),
+            orig_id = id;
+
+        if (id_seen[id]) {
+            for (var n = 1; n < 4096; n++) {
+                id = orig_id + '-' + n;
+                if (!id_seen[id])
+                    break;
+            }
+            el.setAttribute('id', id);
+        }
+        id_seen[id] = 1;
+        el.innerHTML = '<a href="#' + id + '">' + el.innerHTML + '</a>';
+    }
+
+    ext = md_plug['post'];
+    if (ext && ext[0].render)
+        try {
+            ext[0].render(md_dom);
+        }
+        catch (ex) {
+            md_plug_err(ex, ext[1]);
+        }
+
+    copydom(md_dom, dest_dom, 0);
+
+    if (ext && ext[0].render2)
+        try {
+            ext[0].render2(dest_dom);
+        }
+        catch (ex) {
+            md_plug_err(ex, ext[1]);
+        }
+}
+
+
+function init_toc() {
+    var loader = ebi('ml');
+    loader.parentNode.removeChild(loader);
+
+    var anchors = [];  // list of toc entries, complex objects
+    var anchor = null; // current toc node
+    var html = [];     // generated toc html
+    var lv = 0;        // current indentation level in the toc html
+    var ctr = [0, 0, 0, 0, 0, 0];
+
+    var manip_nodes_dyn = dom_pre.getElementsByTagName('*');
+    var manip_nodes = [];
+    for (var a = 0, aa = manip_nodes_dyn.length; a < aa; a++)
+        manip_nodes.push(manip_nodes_dyn[a]);
+
+    for (var a = 0, aa = manip_nodes.length; a < aa; a++) {
+        var elm = manip_nodes[a];
+        var m = /^[Hh]([1-6])/.exec(elm.tagName);
+        var is_header = m !== null;
+        if (is_header) {
+            var nlv = m[1];
+            while (lv < nlv) {
+                html.push('<ul>');
+                lv++;
+            }
+            while (lv > nlv) {
+                html.push('</ul>');
+                lv--;
+            }
+            ctr[lv - 1]++;
+            for (var b = lv; b < 6; b++)
+                ctr[b] = 0;
+
+            elm.childNodes[0].setAttribute('ctr', ctr.slice(0, lv).join('.'));
+
+            var elm2 = elm.cloneNode(true);
+            elm2.childNodes[0].textContent = elm.textContent;
+            while (elm2.childNodes.length > 1)
+                elm2.removeChild(elm2.childNodes[1]);
+
+            html.push('<li>' + elm2.innerHTML + '</li>');
+
+            if (anchor != null)
+                anchors.push(anchor);
+
+            anchor = {
+                elm: elm,
+                kids: [],
+                y: null
+            };
+        }
+        if (!is_header && anchor)
+            anchor.kids.push(elm);
+    }
+    dom_toc.innerHTML = html.join('\n');
+    if (anchor != null)
+        anchors.push(anchor);
+
+    // copy toc links into the toc list
+    var atoc = dom_toc.getElementsByTagName('a');
+    for (var a = 0, aa = anchors.length; a < aa; a++)
+        anchors[a].lnk = atoc[a];
+
+    // collect vertical position of all toc items (headers in document)
+    function freshen_offsets() {
+        var top = window.pageYOffset || document.documentElement.scrollTop;
+        for (var a = anchors.length - 1; a >= 0; a--) {
+            var y = top + anchors[a].elm.getBoundingClientRect().top;
+            y = Math.round(y * 10.0) / 10;
+            if (anchors[a].y === y)
+                break;
+
+            anchors[a].y = y;
+        }
+    }
+
+    // hilight the correct toc items + scroll into view
+    function freshen_toclist() {
+        if (anchors.length == 0)
+            return;
+
+        var ptop = window.pageYOffset || document.documentElement.scrollTop;
+        var hit = anchors.length - 1;
+        for (var a = 0; a < anchors.length; a++) {
+            if (anchors[a].y >= ptop - 8) {  //???
+                hit = a;
+                break;
+            }
+        }
+
+        var links = dom_toc.getElementsByTagName('a');
+        if (!anchors[hit].active) {
+            for (var a = 0; a < anchors.length; a++) {
+                if (anchors[a].active) {
+                    anchors[a].active = false;
+                    links[a].setAttribute('class', '');
+                }
+            }
+            anchors[hit].active = true;
+            links[hit].setAttribute('class', 'act');
+        }
+
+        var pane_height = parseInt(getComputedStyle(dom_toc).height);
+        var link_bounds = links[hit].getBoundingClientRect();
+        var top = link_bounds.top - (pane_height / 6);
+        var btm = link_bounds.bottom + (pane_height / 6);
+        if (top < 0)
+            dom_toc.scrollTop -= -top;
+        else if (btm > pane_height)
+            dom_toc.scrollTop += btm - pane_height;
+    }
+
+    function refresh() {
+        freshen_offsets();
+        freshen_toclist();
+    }
+
+    return { "refresh": refresh }
+}
+
+
+// "main" :p
+convert_markdown(dom_src.value, dom_pre);
+var toc = init_toc();
+
+
+// scroll handler
+var redraw = (function () {
+    var sbs = false;
+    function onresize() {
+        sbs = window.matchMedia('(min-width: 64em)').matches;
+        var y = (dom_hbar.offsetTop + dom_hbar.offsetHeight) + 'px';
+        if (sbs) {
+            dom_toc.style.top = y;
+            dom_wrap.style.top = y;
+            dom_toc.style.marginTop = '0';
+        }
+        onscroll();
+    }
+
+    function onscroll() {
+        toc.refresh();
+    }
+
+    window.onresize = onresize;
+    window.onscroll = onscroll;
+    dom_wrap.onscroll = onscroll;
+
+    onresize();
+    return onresize;
+})();
+
+
+dom_navtgl.onclick = function () {
+    var hidden = dom_navtgl.innerHTML == 'hide nav';
+    dom_navtgl.innerHTML = hidden ? 'show nav' : 'hide nav';
+    dom_nav.style.display = hidden ? 'none' : 'block';
+
+    swrite('hidenav', hidden ? 1 : 0);
+    redraw();
+};
+
+if (sread('hidenav') == 1)
+    dom_navtgl.onclick();

copyparty/web/md2.css

@@ -0,0 +1,126 @@
+#toc {
+    display: none;
+}
+#mtw {
+    display: block;
+    position: fixed;
+    left: .5em;
+    bottom: 0;
+    width: calc(100% - 56em);
+}
+#mw {
+    left: calc(100% - 55em);
+    overflow-y: auto;
+    position: fixed;
+    bottom: 0;
+}
+
+
+/* single-screen */
+#mtw.preview,
+#mw.editor {
+    opacity: 0;
+    z-index: 1;
+}
+#mw.preview,
+#mtw.editor {
+    z-index: 5;
+}
+#mtw.single,
+#mw.single {
+    margin: 0;
+    left: 1em;
+    left: max(1em, calc((100% - 56em) / 2));
+}
+#mtw.single {
+    width: 55em;
+    width: min(55em, calc(100% - 2em));
+}
+
+
+#mp {
+    position: relative;
+}
+#mt, #mtr {
+    width: 100%;
+    height: calc(100% - 1px);
+    color: #444;
+    background: #f7f7f7;
+    border: 1px solid #999;
+    outline: none;
+    padding: 0;
+    margin: 0;
+    font-family: 'consolas', monospace, monospace;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    word-wrap: break-word; /*ie*/
+    overflow-y: scroll;
+    line-height: 1.3em;
+    font-size: .9em;
+    position: relative;
+    scrollbar-color: #eb0 #f7f7f7;
+}
+html.dark #mt {
+    color: #eee;
+    background: #222;
+    border: 1px solid #777;
+    scrollbar-color: #b80 #282828;
+}
+#mtr {
+    position: absolute;
+    top: 0;
+    left: 0;
+}
+#save.force-save {
+    color: #400;
+    background: #f97;
+    border-radius: .15em;
+}
+html.dark #save.force-save {
+    color: #fca;
+    background: #720;
+}
+#save.disabled {
+    opacity: .4;
+}
+#helpbox,
+#toast {
+    background: #f7f7f7;
+    border-radius: .4em;
+    z-index: 9001;
+}
+#helpbox {
+    display: none;
+    position: fixed;
+    padding: 2em;
+    top: 4em;
+    overflow-y: auto;
+    box-shadow: 0 .5em 2em #777;
+    height: calc(100% - 12em);
+    left: calc(50% - 15em);
+    right: 0;
+    width: 30em;
+}
+#helpclose {
+    display: block;
+}
+html.dark #helpbox {
+    box-shadow: 0 .5em 2em #444;
+}
+html.dark #helpbox,
+html.dark #toast {
+    background: #222;
+    border: 1px solid #079;
+    border-width: 1px 0;
+}
+#toast {
+    font-weight: bold;
+    text-align: center;
+    padding: .6em 0;
+    position: fixed;
+    z-index: 9001;
+    top: 30%;
+    transition: opacity 0.2s ease-in-out;
+    opacity: 1;
+}

copyparty/web/mde.css

@@ -0,0 +1,321 @@
+html .editor-toolbar>button { margin-left: -1px; border: 1px solid rgba(0,0,0,0.1) }
+html .editor-toolbar>button+button { border-left: 1px solid rgba(0,0,0,0) }
+html .editor-toolbar>button:hover,
+html .editor-toolbar>button:active { box-shadow: 0 .1em .3em #999; z-index: 9 }
+html .editor-toolbar>button:active,
+html .editor-toolbar>button.active { border-color: rgba(0,0,0,0.4); background: #fc0 }
+html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
+html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
+
+html {
+    line-height: 1.5em;
+}
+html, body {
+    margin: 0;
+    padding: 0;
+    min-height: 100%;
+    font-family: sans-serif;
+    background: #f7f7f7;
+    color: #333;
+}
+#mn {
+    font-weight: normal;
+    margin: 1.3em 0 .7em 1em;
+}
+#mn a {
+    color: #444;
+    margin: 0 0 0 -.2em;
+    padding: 0 0 0 .4em;
+    text-decoration: none;
+    /* ie: */
+    border-bottom: .1em solid #777\9;
+    margin-right: 1em\9;
+}
+#mn a:first-child {
+    padding-left: .5em;
+}
+#mn a:last-child {
+    padding-right: .5em;
+}
+#mn a:not(:last-child):after {
+    content: '';
+    width: 1.05em;
+    height: 1.05em;
+    margin: -.2em .3em -.2em -.4em;
+    display: inline-block;
+    border: 1px solid rgba(0,0,0,0.2);
+    border-width: .2em .2em 0 0;
+    transform: rotate(45deg);
+}
+#mn a:hover {
+    color: #000;
+    text-decoration: underline;
+}
+
+html .editor-toolbar>button.disabled {
+    opacity: .35;
+    pointer-events: none;
+}
+html .editor-toolbar>button.save.force-save {
+    background: #f97;
+}
+
+/*
+*[data-ln]:before {
+	content: attr(data-ln);
+	font-size: .8em;
+	margin: 0 .4em;
+	color: #f0c;
+}
+.cm-header { font-size: .4em !important }
+*/
+
+
+
+
+
+/* copied from md.css for now */
+.mdo pre,
+.mdo code,
+.mdo a {
+	color: #480;
+	background: #f7f7f7;
+	border: .07em solid #ddd;
+	border-radius: .2em;
+	padding: .1em .3em;
+	margin: 0 .1em;
+}
+.mdo code {
+	font-size: .96em;
+}
+.mdo pre,
+.mdo code {
+	font-family: monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-all;
+}
+.mdo pre code {
+	display: block;
+	margin: 0 -.3em;
+	padding: .4em .5em;
+	line-height: 1.1em;
+}
+.mdo a {
+    color: #fff;
+    background: #39b;
+    text-decoration: none;
+    padding: 0 .3em;
+    border: none;
+    border-bottom: .07em solid #079;
+}
+.mdo h2 {
+    color: #fff;
+    background: #555;
+    margin-top: 2em;
+    border-bottom: .22em solid #999;
+    border-top: none;
+}
+.mdo h1 {
+    color: #fff;
+    background: #444;
+    font-weight: normal;
+    border-top: .4em solid #fb0;
+    border-bottom: .4em solid #777;
+    border-radius: 0 1em 0 1em;
+    margin: 3em 0 1em 0;
+    padding: .5em 0;
+}
+h1, h2 {
+	line-height: 1.5em;
+}
+h1 {
+	font-size: 1.7em;
+	text-align: center;
+	border: 1em solid #777;
+	border-width: .05em 0;
+	margin: 3em 0;
+}
+h2 {
+	font-size: 1.5em;
+	font-weight: normal;
+	background: #f7f7f7;
+	border-top: .07em solid #fff;
+	border-bottom: .07em solid #bbb;
+	border-radius: .5em .5em 0 0;
+	padding-left: .4em;
+	margin-top: 3em;
+}
+.mdo ul,
+.mdo ol {
+	border-left: .3em solid #ddd;
+}
+.mdo>ul,
+.mdo>ol {
+	border-color: #bbb;
+}
+.mdo ul>li {
+	list-style-type: disc;
+}
+.mdo ul>li,
+.mdo ol>li {
+	margin: .7em 0;
+}
+strong {
+	color: #000;
+}
+p>em,
+li>em,
+td>em {
+	color: #c50;
+	padding: .1em;
+	border-bottom: .1em solid #bbb;
+}
+blockquote {
+	font-family: serif;
+	background: #f7f7f7;
+	border: .07em dashed #ccc;
+	padding: 0 2em;
+	margin: 1em 0;
+}
+small {
+	opacity: .8;
+}
+table {
+	border-collapse: collapse;
+}
+td {
+	padding: .2em .5em;
+	border: .12em solid #aaa;
+}
+th {
+	border: .12em solid #aaa;
+}
+
+
+
+
+
+/* mde support */
+.mdo {
+    padding: 1em;
+    background: #f7f7f7;
+}
+html.dark .mdo {
+    background: #1c1c1c;
+}
+.CodeMirror {
+    background: #f7f7f7;
+}
+
+
+
+
+
+/* darkmode */
+html.dark .mdo,
+html.dark .CodeMirror {
+    border-color: #222;
+}
+html.dark,
+html.dark body,
+html.dark .CodeMirror {
+    background: #222;
+    color: #ccc;
+}
+html.dark .CodeMirror-cursor {
+    border-color: #fff;
+}
+html.dark .CodeMirror-selected {
+    box-shadow: 0 0 1px #0cf inset;
+}
+html.dark .CodeMirror-selected,
+html.dark .CodeMirror-selectedtext {
+    border-radius: .1em;
+    background: #246;
+    color: #fff;
+}
+html.dark .mdo a {
+    background: #057;
+}
+html.dark .mdo h1 a, html.dark .mdo h4 a,
+html.dark .mdo h2 a, html.dark .mdo h5 a,
+html.dark .mdo h3 a, html.dark .mdo h6 a {
+    color: inherit;
+    background: none;
+}
+html.dark pre,
+html.dark code {
+    color: #8c0;
+    background: #1a1a1a;
+    border: .07em solid #333;
+}
+html.dark .mdo ul,
+html.dark .mdo ol {
+    border-color: #444;
+}
+html.dark .mdo>ul,
+html.dark .mdo>ol {
+    border-color: #555;
+}
+html.dark strong {
+    color: #fff;
+}
+html.dark p>em,
+html.dark li>em,
+html.dark td>em {
+    color: #f94;
+    border-color: #666;
+}
+html.dark h1 {
+    background: #383838;
+    border-top: .4em solid #b80;
+    border-bottom: .4em solid #4c4c4c;
+}
+html.dark h2 {
+    background: #444;
+    border-bottom: .22em solid #555;
+}
+html.dark td,
+html.dark th {
+    border-color: #444;
+}
+html.dark blockquote {
+    background: #282828;
+    border: .07em dashed #444;
+}
+
+
+
+html.dark #mn a {
+    color: #ccc;
+}
+html.dark #mn a:not(:last-child):after {
+    border-color: rgba(255,255,255,0.3);
+}
+html.dark .editor-toolbar {
+    border-color: #2c2c2c;
+    background: #1c1c1c;
+}
+html.dark .editor-toolbar>i.separator {
+    border-left: 1px solid #444;
+    border-right: 1px solid #111;
+}
+html.dark .editor-toolbar>button {
+    margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
+    color: #aaa;
+}
+
+
+
+html.dark .editor-toolbar>button:hover {
+    color: #333;
+}
+html.dark .editor-toolbar>button.active {
+    color: #333;
+    border-color: #ec1;
+    background: #c90;
+}
+html.dark .editor-toolbar::after,
+html.dark .editor-toolbar::before {
+    background: none;
+}

copyparty/web/mde.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html><html><head>
+	<meta charset="utf-8">
+	<title>📝🎉 {{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.7">
+	<link href="/.cpr/mde.css" rel="stylesheet">
+	<link href="/.cpr/deps/mini-fa.css" rel="stylesheet">
+	<link href="/.cpr/deps/easymde.css" rel="stylesheet">
+</head>
+<body>
+	<div id="mw">
+		<div id="mn"></div>
+		<div id="ml">
+			<div style="text-align:center;margin:5em 0">
+				<div style="font-size:2em;margin:1em 0">Loading</div>
+				if you're still reading this, check that javascript is allowed
+			</div>
+		</div>
+		<div id="m">
+			<textarea id="mt" style="display:none" autocomplete="off">{{ md }}</textarea>
+		</div>
+	</div>
+	<script>
+
+var last_modified = {{ lastmod }};
+var md_opt = {
+	link_md_as_html: false,
+	allow_plugins: {{ md_plug }},
+	modpoll_freq: {{ md_chk_rate }}
+};
+
+var lightswitch = (function () {
+	var fun = function () {
+		var dark = !!!document.documentElement.getAttribute("class");
+		document.documentElement.setAttribute("class", dark ? "dark" : "");
+		if (window.localStorage)
+			localStorage.setItem('darkmode', dark ? 1 : 0);
+	};
+	if (window.localStorage && localStorage.getItem('darkmode') == 1)
+		fun();
+	
+	return fun;
+})();
+
+	</script>
+    <script src="/.cpr/util.js"></script>
+	<script src="/.cpr/deps/easymde.js"></script>
+	<script src="/.cpr/mde.js"></script>
+</body></html>

copyparty/web/mde.js

@@ -0,0 +1,226 @@
+"use strict";
+
+var dom_wrap = ebi('mw');
+var dom_nav = ebi('mn');
+var dom_doc = ebi('m');
+var dom_md = ebi('mt');
+
+(function () {
+    var n = document.location + '';
+    n = n.substr(n.indexOf('//') + 2).split('?')[0].split('/');
+    n[0] = 'top';
+    var loc = [];
+    var nav = [];
+    for (var a = 0; a < n.length; a++) {
+        if (a > 0)
+            loc.push(n[a]);
+
+        var dec = uricom_dec(n[a])[0].replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+
+        nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
+    }
+    dom_nav.innerHTML = nav.join('');
+})();
+
+var mde = (function () {
+    var tbar = [
+        {
+            name: "light",
+            title: "light",
+            className: "fa fa-lightbulb",
+            action: lightswitch
+        }, {
+            name: "save",
+            title: "save",
+            className: "fa fa-save",
+            action: save
+        }, '|',
+        'bold', 'italic', 'strikethrough', 'heading', '|',
+        'code', 'quote', 'unordered-list', 'ordered-list', 'clean-block', '|',
+        'link', 'image', 'table', 'horizontal-rule', '|',
+        'preview', 'side-by-side', 'fullscreen', '|',
+        'undo', 'redo'];
+
+    var mde = new EasyMDE({
+        autoDownloadFontAwesome: false,
+        autofocus: true,
+        spellChecker: false,
+        renderingConfig: {
+            markedOptions: {
+                breaks: true,
+                gfm: true
+            }
+        },
+        shortcuts: {
+            "save": "Ctrl-S"
+        },
+        insertTexts: ["[](", ")"],
+        indentWithTabs: false,
+        tabSize: 2,
+        toolbar: tbar,
+        previewClass: 'mdo',
+        onToggleFullScreen: set_jumpto,
+    });
+    md_changed(mde, true);
+    mde.codemirror.on("change", function () {
+        md_changed(mde);
+    });
+    var loader = ebi('ml');
+    loader.parentNode.removeChild(loader);
+    return mde;
+})();
+
+function set_jumpto() {
+    document.querySelector('.editor-preview-side').onclick = jumpto;
+}
+
+function jumpto(ev) {
+    var tgt = ev.target || ev.srcElement;
+    var ln = null;
+    while (tgt && !ln) {
+        ln = tgt.getAttribute('data-ln');
+        tgt = tgt.parentElement;
+    }
+    var ln = parseInt(ln);
+    console.log(ln);
+    var cm = mde.codemirror;
+    var y = cm.heightAtLine(ln - 1, 'local');
+    var y2 = cm.heightAtLine(ln, 'local');
+    cm.scrollTo(null, y + (y2 - y) - cm.getScrollInfo().clientHeight / 2);
+}
+
+function md_changed(mde, on_srv) {
+    if (on_srv)
+        window.md_saved = mde.value();
+
+    var md_now = mde.value();
+    var save_btn = document.querySelector('.editor-toolbar button.save');
+
+    if (md_now == window.md_saved)
+        save_btn.classList.add('disabled');
+    else
+        save_btn.classList.remove('disabled');
+
+    set_jumpto();
+}
+
+function save(mde) {
+    var save_btn = document.querySelector('.editor-toolbar button.save');
+    if (save_btn.classList.contains('disabled')) {
+        alert('there is nothing to save');
+        return;
+    }
+    var force = save_btn.classList.contains('force-save');
+    if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
+        alert('ok, aborted');
+        return;
+    }
+
+    var txt = mde.value();
+
+    var fd = new FormData();
+    fd.append("act", "tput");
+    fd.append("lastmod", (force ? -1 : last_modified));
+    fd.append("body", txt);
+
+    var url = (document.location + '').split('?')[0];
+    var xhr = new XMLHttpRequest();
+    xhr.open('POST', url, true);
+    xhr.responseType = 'text';
+    xhr.onreadystatechange = save_cb;
+    xhr.btn = save_btn;
+    xhr.mde = mde;
+    xhr.txt = txt;
+    xhr.send(fd);
+}
+
+function save_cb() {
+    if (this.readyState != XMLHttpRequest.DONE)
+        return;
+
+    if (this.status !== 200) {
+        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
+        return;
+    }
+
+    var r;
+    try {
+        r = JSON.parse(this.responseText);
+    }
+    catch (ex) {
+        alert('Failed to parse reply from server:\n\n' + this.responseText);
+        return;
+    }
+
+    if (!r.ok) {
+        if (!this.btn.classList.contains('force-save')) {
+            this.btn.classList.add('force-save');
+            var msg = [
+                'This file has been modified since you started editing it!\n',
+                'if you really want to overwrite, press save again.\n',
+                'modified ' + ((r.now - r.lastmod) / 1000) + ' seconds ago,',
+                ((r.lastmod - last_modified) / 1000) + ' sec after you opened it\n',
+                last_modified + ' lastmod when you opened it,',
+                r.lastmod + ' lastmod on the server now,',
+                r.now + ' server time now,\n',
+            ];
+            alert(msg.join('\n'));
+        }
+        else {
+            alert('Error! Save failed.  Maybe this JSON explains why:\n\n' + this.responseText);
+        }
+        return;
+    }
+
+    this.btn.classList.remove('force-save');
+    //alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
+
+    // download the saved doc from the server and compare
+    var url = (document.location + '').split('?')[0] + '?raw';
+    var xhr = new XMLHttpRequest();
+    xhr.open('GET', url, true);
+    xhr.responseType = 'text';
+    xhr.onreadystatechange = save_chk;
+    xhr.btn = this.save_btn;
+    xhr.mde = this.mde;
+    xhr.txt = this.txt;
+    xhr.lastmod = r.lastmod;
+    xhr.send();
+}
+
+function save_chk() {
+    if (this.readyState != XMLHttpRequest.DONE)
+        return;
+
+    if (this.status !== 200) {
+        alert('Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
+        return;
+    }
+
+    var doc1 = this.txt.replace(/\r\n/g, "\n");
+    var doc2 = this.responseText.replace(/\r\n/g, "\n");
+    if (doc1 != doc2) {
+        alert(
+            'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
+            'Length: yours=' + doc1.length + ', server=' + doc2.length
+        );
+        alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
+        alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
+        return;
+    }
+
+    last_modified = this.lastmod;
+    md_changed(this.mde, true);
+
+    var ok = document.createElement('div');
+    ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
+    ok.innerHTML = 'OK✔️';
+    var parent = ebi('m');
+    document.documentElement.appendChild(ok);
+    setTimeout(function () {
+        ok.style.opacity = 0;
+    }, 500);
+    setTimeout(function () {
+        ok.parentNode.removeChild(ok);
+    }, 750);
+}

copyparty/web/msg.html

@@ -31,6 +31,10 @@
         {%- if html %}
         {{ html }}
         {%- endif %}
+
+        {%- if click %}
+        <script>document.getElementsByTagName("a")[0].click()</script>
+        {%- endif %}
     </div>
 
     {%- if redir %}

copyparty/web/splash.css

@@ -13,6 +13,7 @@ h1 {
 	border-bottom: 1px solid #ccc;
 	margin: 2em 0 .4em 0;
 	padding: 0 0 .2em 0;
+	font-weight: normal;
 }
 li {
 	margin: 1em 0;
@@ -24,4 +25,29 @@ a {
 	border-bottom: 1px solid #aaa;
 	border-radius: .2em;
 	padding: .2em .8em;
+}
+
+
+html.dark,
+html.dark body,
+html.dark #wrap {
+	background: #222;
+	color: #ccc;
+}
+html.dark h1 {
+	border-color: #777;
+}
+html.dark a {
+	color: #fff;
+	background: #057;
+	border-color: #37a;
+}
+html.dark input {
+	color: #fff;
+	background: #624;
+	border: 1px solid #c27;
+	border-width: 1px 0 0 0;
+	border-radius: .5em;
+	padding: .5em .7em;
+	margin: 0 .5em 0 0;
 }

copyparty/web/splash.html

@@ -36,7 +36,11 @@
             </form>
         </ul>
     </div>
-    <!-- script src="/.cpr/splash.js"></script -->
-</body>
+    <script>
 
+if (window.localStorage && localStorage.getItem('darkmode') == 1)
+    document.documentElement.setAttribute("class", "dark");
+
+</script>
+</body>
 </html>

copyparty/web/upload.css

@@ -1,24 +1,14 @@
-#bup {
-	padding: .5em .5em .5em .3em;
-	margin: 1em 0 2em 0;
-	background: #2d2d2d;
-	border-radius: 0 1em 1em 0;
-	border: 1px solid #3a3a3a;
-	border-width: 0 .3em .3em 0;
-	box-shadow: 0 0 1em #222 inset;
-	max-width: 40em;
-}
-#bup input {
-	margin: .5em;
-}
-#up2k {
-	display: none;
-	padding: 0 1em;
+
+#op_up2k {
+	padding: 0 1em 1em 1em;
 }
 #u2form {
 	position: absolute;
 	top: 0;
 	left: 0;
+	width: 2px;
+	height: 2px;
+	overflow: hidden;
 }
 #u2form input {
 	background: #444;
@@ -29,21 +19,6 @@
 	color: #f87;
 	padding: .5em;
 }
-#u2tgl {
-	color: #fc5;
-	font-size: 1.5em;
-	margin: .5em 0 1em 0;
-	display: block;
-}
-#u2body {
-	display: none;
-	padding-bottom: 1em;
-}
-#u2form {
-	width: 2px;
-	height: 2px;
-	overflow: hidden;
-}
 #u2btn {
 	color: #eee;
 	background: #555;
@@ -52,17 +27,27 @@
 	background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%);
 	filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0);
 	text-decoration: none;
-	line-height: 1.5em;
+	line-height: 1.3em;
 	border: 1px solid #222;
 	border-radius: .4em;
 	text-align: center;
-	font-size: 2em;
-	margin: 1em auto;
-	padding: 1em 0;
-	width: 12em;
+	font-size: 1.5em;
+	margin: .5em auto;
+	padding: .8em 0;
+	width: 16em;
 	cursor: pointer;
 	box-shadow: .4em .4em 0 #111;
 }
+#op_up2k.srch #u2btn {
+	background: linear-gradient(to bottom, #ca3 0%, #fd8 50%, #fc6 51%, #b92 100%);
+	text-shadow: 1px 1px 1px #fc6;
+	color: #333;
+}
+#u2conf #u2btn {
+	margin: -1.5em 0;
+	padding: .8em 0;
+	width: 100%;
+}
 #u2notbtn {
 	display: none;
 	text-align: center;
@@ -77,6 +62,9 @@
 	width: calc(100% - 2em);
 	max-width: 100em;
 }
+#op_up2k.srch #u2tab {
+	max-width: none;
+}
 #u2tab td {
 	border: 1px solid #ccc;
 	border-width: 0 0px 1px 0;
@@ -88,12 +76,19 @@
 #u2tab td:nth-child(3) {
 	width: 40%;
 }
+#op_up2k.srch #u2tab td:nth-child(3) {
+	font-family: sans-serif;
+	width: auto;
+}
 #u2tab tr+tr:hover td {
 	background: #222;
 }
 #u2conf {
 	margin: 1em auto;
-	width: 26em;
+	width: 30em;
+}
+#u2conf.has_btn {
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;
@@ -129,10 +124,72 @@
 #u2conf input+a {
 	background: #d80;
 }
+#u2conf label {
+	font-size: 1.6em;
+	width: 2em;
+	height: 1em;
+	padding: .4em 0;
+	display: block;
+	user-select: none;
+	border-radius: .25em;
+}
+#u2conf input[type="checkbox"] {
+	position: relative;
+	opacity: .02;
+	top: 2em;
+}
+#u2conf input[type="checkbox"]+label {
+	position: relative;
+	background: #603;
+	border-bottom: .2em solid #a16;
+	box-shadow: 0 .1em .3em #a00 inset;
+}
+#u2conf input[type="checkbox"]:checked+label {
+	background: #6a1;
+	border-bottom: .2em solid #efa;
+	box-shadow: 0 .1em .5em #0c0;
+}
+#u2conf input[type="checkbox"]+label:hover {
+	box-shadow: 0 .1em .3em #fb0;
+	border-color: #fb0;
+}
+#op_up2k.srch #u2conf td:nth-child(1)>*,
+#op_up2k.srch #u2conf td:nth-child(2)>*,
+#op_up2k.srch #u2conf td:nth-child(3)>* {
+	background: #777;
+	border-color: #ccc;
+	box-shadow: none;
+	opacity: .2;
+}
+#u2cdesc {
+	position: absolute;
+	width: 34em;
+	left: calc(50% - 15em);
+	background: #222;
+	border: 0 solid #555;
+	text-align: center;
+	overflow: hidden;
+	margin: 0 -2em;
+	height: 0;
+	padding: 0 1em;
+	opacity: .1;
+    transition: all 0.14s ease-in-out;
+	border-radius: .4em;
+	box-shadow: 0 .2em .5em #222;
+}
+#u2cdesc.show {
+	padding: 1em;
+	height: auto;
+	border-width: .2em 0;
+	opacity: 1;
+}
 #u2foot {
 	color: #fff;
 	font-style: italic;
 }
+#u2footfoot {
+	margin-bottom: -1em;
+}
 .prog {
 	font-family: monospace;
 }
@@ -154,3 +211,13 @@
 	bottom: 0;
 	background: #0a0;
 }
+#u2tab a>span {
+	font-weight: bold;
+	font-style: italic;
+	color: #fff;
+	padding-left: .2em;
+}
+#u2cleanup {
+	float: right;
+	margin-bottom: -.3em;
+}

copyparty/web/upload.html

@@ -1,20 +1,61 @@
-    <div id="bup">
+
+    <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
         </form>
     </div>
 
-    <div id="up2k">
-        <a href="#" id="u2tgl">you can upload here</a>
-        <form id="u2form" method="POST" enctype="multipart/form-data" onsubmit="return false;"></form>
-        <div id="u2body">
+    <div id="op_mkdir" class="opview opbox act">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+            <input type="hidden" name="act" value="mkdir" />
+            <input type="text" name="name" size="30">
+            <input type="submit" value="mkdir">
+        </form>
+    </div>
+
+    <div id="op_new_md" class="opview opbox">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+            <input type="hidden" name="act" value="new_md" />
+            <input type="text" name="name" size="30">
+            <input type="submit" value="create doc">
+        </form>
+    </div>
+
+    <div id="op_msg" class="opview opbox">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
+            <input type="text" name="msg" size="30">
+            <input type="submit" value="send msg">
+        </form>
+    </div>
+
+    <div id="op_up2k" class="opview">
+        <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
 
             <table id="u2conf">
                 <tr>
                     <td>parallel uploads</td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="multitask" />
+                        <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="ask_up" />
+                        <label for="ask_up" alt="ask for confirmation befofre upload starts">💭</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="flag_en" />
+                        <label for="flag_en" alt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>
+                    </td>
+                {%- if have_up2k_idx %}
+                    <td data-perm="read" rowspan="2">
+                        <input type="checkbox" id="fsearch" />
+                        <label for="fsearch" alt="don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)">🔎</label>
+                    </td>
+                {%- endif %}
+                    <td data-perm="read" rowspan="2" id="u2btn_cw"></td>
                 </tr>
                 <tr>
                     <td>
@@ -22,29 +63,30 @@
                         <input class="txtbox" id="nthread" value="2" />
                         <a href="#" id="nthread_add">+</a>
                     </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="multitask" />
-                        <label for="multitask">hash while<br />uploading</label>
-                    </td>
                 </tr>
             </table>
 
+            <div id="u2cdesc"></div>
+
             <div id="u2notbtn"></div>
 
-            <div id="u2btn">
-                drop files here<br />
-                (or click me)
+            <div id="u2btn_ct">
+                <div id="u2btn">
+                    <span id="u2bm"></span><br />
+                    drag/drop files<br />
+                    and folders here<br />
+                    (or click me)
+                </div>
             </div>
 
             <table id="u2tab">
                 <tr>
                     <td>filename</td>
                     <td>status</td>
-                    <td>progress</td>
+                    <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
                 </tr>
             </table>
 
             <p id="u2foot"></p>
-            <p>( if you don't need resumable uploads and progress bars just use the <a href="#" id="u2nope" onclick="javascript:un2k();">basic uploader</a>)</p>
-        </div>
+            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
     </div>

copyparty/web/util.js

@@ -0,0 +1,425 @@
+"use strict";
+
+// error handler for mobile devices
+function hcroak(msg) {
+    document.body.innerHTML = msg;
+    window.onerror = undefined;
+    throw 'fatal_err';
+}
+function croak(msg) {
+    document.body.textContent = msg;
+    window.onerror = undefined;
+    throw msg;
+}
+function esc(txt) {
+    return txt.replace(/[&"<>]/g, function (c) {
+        return {
+            '&': '&amp;',
+            '"': '&quot;',
+            '<': '&lt;',
+            '>': '&gt;'
+        }[c];
+    });
+}
+function vis_exh(msg, url, lineNo, columnNo, error) {
+    window.onerror = undefined;
+    window['vis_exh'] = null;
+    var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
+        esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
+
+    if (error) {
+        var find = ['desc', 'stack', 'trace'];
+        for (var a = 0; a < find.length; a++)
+            if (String(error[find[a]]) !== 'undefined')
+                html.push('<h2>' + find[a] + '</h2>' +
+                    esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
+    }
+    document.body.style.fontSize = '0.8em';
+    document.body.style.padding = '0 1em 1em 1em';
+    hcroak(html.join('\n'));
+}
+
+
+function ebi(id) {
+    return document.getElementById(id);
+}
+
+function ev(e) {
+    e = e || window.event;
+    if (!e)
+        return;
+
+    if (e.preventDefault)
+        e.preventDefault()
+
+    if (e.stopPropagation)
+        e.stopPropagation();
+
+    e.returnValue = false;
+    return e;
+}
+
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
+if (!String.prototype.endsWith) {
+    String.prototype.endsWith = function (search, this_len) {
+        if (this_len === undefined || this_len > this.length) {
+            this_len = this.length;
+        }
+        return this.substring(this_len - search.length, this_len) === search;
+    };
+}
+if (!String.startsWith) {
+    String.prototype.startsWith = function (s, i) {
+        i = i > 0 ? i | 0 : 0;
+        return this.substring(i, i + s.length) === s;
+    };
+}
+
+
+// https://stackoverflow.com/a/950146
+function import_js(url, cb) {
+    var head = document.head || document.getElementsByTagName('head')[0];
+    var script = document.createElement('script');
+    script.type = 'text/javascript';
+    script.src = url;
+
+    script.onreadystatechange = cb;
+    script.onload = cb;
+
+    head.appendChild(script);
+}
+
+
+var crctab = (function () {
+    var c, tab = [];
+    for (var n = 0; n < 256; n++) {
+        c = n;
+        for (var k = 0; k < 8; k++) {
+            c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+        }
+        tab[n] = c;
+    }
+    return tab;
+})();
+
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+        crc = (crc >>> 8) ^ crctab[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return ((crc ^ (-1)) >>> 0).toString(16);
+};
+
+
+function sortTable(table, col, cb) {
+    var tb = table.tBodies[0],
+        th = table.tHead.rows[0].cells,
+        tr = Array.prototype.slice.call(tb.rows, 0),
+        i, reverse = th[col].className.indexOf('sort1') !== -1 ? -1 : 1;
+    for (var a = 0, thl = th.length; a < thl; a++)
+        th[a].className = th[a].className.replace(/ *sort-?1 */, " ");
+    th[col].className += ' sort' + reverse;
+    var stype = th[col].getAttribute('sort');
+    try {
+        var nrules = [], rules = jread("fsort", []);
+        rules.unshift([th[col].getAttribute('name'), reverse, stype || '']);
+        for (var a = 0; a < rules.length; a++) {
+            var add = true;
+            for (var b = 0; b < a; b++)
+                if (rules[a][0] == rules[b][0])
+                    add = false;
+
+            if (add)
+                nrules.push(rules[a]);
+
+            if (nrules.length >= 10)
+                break;
+        }
+        jwrite("fsort", nrules);
+    }
+    catch (ex) {
+        console.log("failed to persist sort rules, resetting: " + ex);
+        jwrite("fsort", null);
+    }
+    var vl = [];
+    for (var a = 0; a < tr.length; a++) {
+        var cell = tr[a].cells[col];
+        if (!cell) {
+            vl.push([null, a]);
+            continue;
+        }
+        var v = cell.getAttribute('sortv') || cell.textContent.trim();
+        if (stype == 'int') {
+            v = parseInt(v.replace(/[, ]/g, '')) || 0;
+        }
+        vl.push([v, a]);
+    }
+    vl.sort(function (a, b) {
+        a = a[0];
+        b = b[0];
+        if (a === null)
+            return -1;
+        if (b === null)
+            return 1;
+
+        if (stype == 'int') {
+            return reverse * (a - b);
+        }
+        return reverse * (a.localeCompare(b));
+    });
+    for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
+    if (cb) cb();
+}
+function makeSortable(table, cb) {
+    var th = table.tHead, i;
+    th && (th = th.rows[0]) && (th = th.cells);
+    if (th) i = th.length;
+    else return; // if no `<thead>` then do nothing
+    while (--i >= 0) (function (i) {
+        th[i].onclick = function (e) {
+            ev(e);
+            sortTable(table, i, cb);
+        };
+    }(i));
+}
+
+
+
+(function () {
+    var ops = document.querySelectorAll('#ops>a');
+    for (var a = 0; a < ops.length; a++) {
+        ops[a].onclick = opclick;
+    }
+})();
+
+
+function opclick(e) {
+    ev(e);
+
+    var dest = this.getAttribute('data-dest');
+    goto(dest);
+
+    swrite('opmode', dest || null);
+
+    var input = document.querySelector('.opview.act input:not([type="hidden"])')
+    if (input)
+        input.focus();
+}
+
+
+function goto(dest) {
+    var obj = document.querySelectorAll('.opview.act');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].classList.remove('act');
+
+    obj = document.querySelectorAll('#ops>a');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].classList.remove('act');
+
+    if (dest) {
+        var ui = ebi('op_' + dest);
+        ui.classList.add('act');
+        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+
+        var fn = window['goto_' + dest];
+        if (fn)
+            fn();
+    }
+
+    if (window['treectl'])
+        treectl.onscroll();
+}
+
+
+(function () {
+    goto();
+    var op = sread('opmode');
+    if (op !== null && op !== '.')
+        try {
+            goto(op);
+        }
+        catch (ex) { }
+})();
+
+
+function linksplit(rp) {
+    var ret = [];
+    var apath = '/';
+    if (rp && rp.charAt(0) == '/')
+        rp = rp.slice(1);
+
+    while (rp) {
+        var link = rp;
+        var ofs = rp.indexOf('/');
+        if (ofs === -1) {
+            rp = null;
+        }
+        else {
+            link = rp.slice(0, ofs + 1);
+            rp = rp.slice(ofs + 1);
+        }
+        var vlink = link;
+        if (link.indexOf('/') !== -1)
+            vlink = link.slice(0, -1) + '<span>/</span>';
+
+        ret.push('<a href="' + apath + link + '">' + vlink + '</a>');
+        apath += link;
+    }
+    return ret;
+}
+
+
+function uricom_enc(txt, do_fb_enc) {
+    try {
+        return encodeURIComponent(txt);
+    }
+    catch (ex) {
+        console.log("uce-err [" + txt + "]");
+        if (do_fb_enc)
+            return esc(txt);
+
+        return txt;
+    }
+}
+
+
+function uricom_dec(txt) {
+    try {
+        return [decodeURIComponent(txt), true];
+    }
+    catch (ex) {
+        console.log("ucd-err [" + txt + "]");
+        return [txt, false];
+    }
+}
+
+
+function get_evpath() {
+    var ret = document.location.pathname;
+
+    if (ret.indexOf('/') !== 0)
+        ret = '/' + ret;
+
+    if (ret.lastIndexOf('/') !== ret.length - 1)
+        ret += '/';
+
+    return ret;
+}
+
+
+function get_vpath() {
+    return uricom_dec(get_evpath())[0];
+}
+
+
+function unix2iso(ts) {
+    return new Date(ts * 1000).toISOString().replace("T", " ").slice(0, -5);
+}
+
+
+function s2ms(s) {
+    s = Math.floor(s);
+    var m = Math.floor(s / 60);
+    return m + ":" + ("0" + (s - m * 60)).slice(-2);
+}
+
+
+function has(haystack, needle) {
+    for (var a = 0; a < haystack.length; a++)
+        if (haystack[a] == needle)
+            return true;
+
+    return false;
+}
+
+
+function sread(key) {
+    if (window.localStorage)
+        return localStorage.getItem(key);
+
+    return null;
+}
+
+function swrite(key, val) {
+    if (window.localStorage) {
+        if (val === undefined || val === null)
+            localStorage.removeItem(key);
+        else
+            localStorage.setItem(key, val);
+    }
+}
+
+function jread(key, fb) {
+    var str = sread(key);
+    if (!str)
+        return fb;
+
+    return JSON.parse(str);
+}
+
+function jwrite(key, val) {
+    if (!val)
+        swrite(key);
+    else
+        swrite(key, JSON.stringify(val));
+}
+
+function icfg_get(name, defval) {
+    var o = ebi(name);
+
+    var val = parseInt(sread(name));
+    if (isNaN(val))
+        return parseInt(o ? o.value : defval);
+
+    if (o)
+        o.value = val;
+
+    return val;
+}
+
+function bcfg_get(name, defval) {
+    var o = ebi(name);
+    if (!o)
+        return defval;
+
+    var val = sread(name);
+    if (val === null)
+        val = defval;
+    else
+        val = (val == '1');
+
+    bcfg_upd_ui(name, val);
+    return val;
+}
+
+function bcfg_set(name, val) {
+    swrite(name, val ? '1' : '0');
+    bcfg_upd_ui(name, val);
+    return val;
+}
+
+function bcfg_upd_ui(name, val) {
+    var o = ebi(name);
+    if (!o)
+        return;
+
+    if (o.getAttribute('type') == 'checkbox')
+        o.checked = val;
+    else if (o) {
+        var fun = val ? 'add' : 'remove';
+        o.classList[fun]('on');
+    }
+}
+
+
+function hist_push(url) {
+    console.log("h-push " + url);
+    history.pushState(url, url, url);
+}
+
+function hist_replace(url) {
+    console.log("h-repl " + url);
+    history.replaceState(url, url, url);
+}

docs/music-analysis.sh

@@ -0,0 +1,242 @@
+#!/bin/bash
+echo please dont actually run this as a scriopt
+exit 1
+
+
+# dependency-heavy, not particularly good fit
+pacman -S llvm10
+python3 -m pip install --user librosa
+git clone https://github.com/librosa/librosa.git
+
+
+# correct bpm for tracks with bad tags
+br='
+/Trip Trip Trip\(Hardcore Edit\).mp3/ {v=176}
+/World!!.BIG_SOS/ {v=175}
+/\/08\..*\(BIG_SOS Bootleg\)\.mp3/ {v=175}
+/もってけ！セーラ服.Asterisk DnB/ {v=175}
+/Rondo\(Asterisk DnB Re.mp3/ {v=175}
+/Ray Nautica 175 Edit/ {v=175;x="thunk"}
+/TOKIMEKI Language.Jauz/ {v=174}
+/YUPPUN Hardcore Remix\).mp3/ {v=174;x="keeps drifting"}
+/(èâAâï.î╧ûδ|バーチャリアル.狐耶)J-Core Remix\).mp3/ {v=172;x="hard"}
+/lucky train..Freezer/ {v=170}
+/Alf zero Bootleg ReMix/ {v=170}
+/Prisoner of Love.Kacky/ {v=170}
+/火炎 .Qota/ {v=170}
+/\(hu-zin Bootleg\)\.mp3/ {v=170}
+/15. STRAIGHT BET\(Milynn Bootleg\)\.mp3/ {v=170}
+/\/13.*\(Milynn Bootleg\)\.mp3/ {v=167;x="way hard"}
+/COLOR PLANET .10SAI . nijikon Remix\)\.mp3/ {v=165}
+/11\. (朝はご飯派|Æ⌐é═é▓ö╤öh)\.mp3/ {v=162}
+/09\. Where.s the core/ {v=160}
+/PLANET\(Koushif Jersey Club Bootleg\)remaster.mp3/ {v=160;x="starts ez turns bs"}
+/kened Soul - Madeon x Angel Beats!.mp3/ {v=160}
+/Dear Moments\(Mother Harlot Bootleg\)\.mp3/ {v=150}
+/POWER.Ringos UKG/ {v=140}
+/ブルー・フィールド\(Ringos UKG Remix\).mp3/ {v=135}
+/プラチナジェット.Ringo Remix..mp3/ {v=131.2}
+/Mirrorball Love \(TKM Bootleg Mix\).mp3/ {v=130}
+/Photon Melodies \(TKM Bootleg Mix\).mp3/ {v=128}
+/Trap of Love \(TKM Bootleg Mix\).mp3/ {v=128}
+/One Step \(TKM Bootleg Mix\)\.mp3/ {v=126}
+/04 (トリカムイ岩|âgâèâJâÇâCèΓ).mp3/ {v=125}
+/Get your Wish \(NAWN REMIX\)\.mp3/ {v=95}
+/Flicker .Nitro Fun/ {v=92}
+/\/14\..*suicat Remix/ {v=85.5;x="tricky"}
+/Yanagi Nagi - Harumodoki \(EO Remix\)\.mp3/ {v=150}
+/Azure - Nicology\.mp3/ {v=128;x="off by 5 how"}
+'
+
+
+# afun host, collects/grades the results
+runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | awk '{v=$1;sub(/[^ ]+ /,"")} '"$br"' {printf "%s %s\n",v,$0}' | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk 'function prt(c) {printf "\033[3%sm%s\033[0m\n",c,$0} $8!="div,"{next} $5!~/^[0-9\.]+/{next} {meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;prt(6);next} idiff==1{n1++;prt(3);next} idiff>10{nx++;prt(1);next} {n10++;prt(5)} END {printf "ok: %d   1off: %2s   (%3s)   10off: %2s   (%3s)   fail: %2s\n",n0,n1,n0+n1,n10,n0+n1+n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; }
+
+
+# ok:   8   1off: 62   ( 70)   10off: 86   (156)   fail: 25   # 105 sec,  librosa @ 8c archvm on 3700x w10
+# ok:   4   1off: 59   ( 63)   10off: 65   (128)   fail: 53   # using original tags (bad)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } runfun
+
+
+# ok: 119   1off:  5   (124)   10off:  8   (132)   fail: 49   # 51 sec,  vamp-example-fixedtempo
+# ok: 109   1off:  4   (113)   10off:  9   (122)   fail: 59   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; }; runfun
+
+
+# ok: 102   1off: 61   (163)   10off: 12   (175)   fail:  6   # 61 sec,  vamp-qm-tempotracker
+# ok:  80   1off: 48   (128)   10off: 11   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; }; runfun
+
+
+# ok: 133   1off: 32   (165)   10off: 12   (177)   fail:  3   # 51 sec,  vamp-beatroot
+# ok: 101   1off: 22   (123)   10off: 16   (139)   fail: 39   # bad-tags
+# note: some tracks fully fail to analyze (unlike the others which always provide a guess)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; }; runfun
+
+
+# ok: 124   1off:  9   (133)   10off: 40   (173)   fail:  8   # 231 sec,  essentia/full
+# ok: 109   1off:  8   (117)   10off: 22   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'import essentia; import essentia.standard as es; fe, fef = es.MusicExtractor(lowlevelStats=["mean", "stdev"], rhythmStats=["mean", "stdev"], tonalStats=["mean", "stdev"])("/dev/shm/'$core'.wav"); print("{:.2f}".format(fe["rhythm.bpm"]))')"; }; runfun
+
+
+# ok: 113   1off: 18   (131)   10off: 46   (177)   fail:  4   # 134 sec,  essentia/re2013
+# ok: 101   1off: 15   (116)   10off: 26   (142)   fail: 39   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'from essentia.standard import *; a=MonoLoader(filename="/dev/shm/'$core'.wav")(); bpm,beats,confidence,_,intervals=RhythmExtractor2013(method="multifeature")(a); print("{:.2f}".format(bpm))')"; }; runfun
+
+
+
+########################################################################
+##
+##  key detectyion
+##
+########################################################################
+
+
+
+# console scriptlet reusing keytabs from browser.js
+var m=''; for (var a=0; a<24; a++) m += 's/\\|(' + maps["traktor_sharps"][a].trim() + "|" + maps["rekobo_classic"][a].trim() + "|" + maps["traktor_musical"][a].trim() + "|" + maps["traktor_open"][a].trim() + ')$/|' + maps["rekobo_alnum"][a].trim() + '/;'; console.log(m);
+
+
+# translate to camelot
+re='s/\|(B|B|B|6d)$/|1B/;s/\|(F#|F#|Gb|7d)$/|2B/;s/\|(C#|Db|Db|8d)$/|3B/;s/\|(G#|Ab|Ab|9d)$/|4B/;s/\|(D#|Eb|Eb|10d)$/|5B/;s/\|(A#|Bb|Bb|11d)$/|6B/;s/\|(F|F|F|12d)$/|7B/;s/\|(C|C|C|1d)$/|8B/;s/\|(G|G|G|2d)$/|9B/;s/\|(D|D|D|3d)$/|10B/;s/\|(A|A|A|4d)$/|11B/;s/\|(E|E|E|5d)$/|12B/;s/\|(G#m|Abm|Abm|6m)$/|1A/;s/\|(D#m|Ebm|Ebm|7m)$/|2A/;s/\|(A#m|Bbm|Bbm|8m)$/|3A/;s/\|(Fm|Fm|Fm|9m)$/|4A/;s/\|(Cm|Cm|Cm|10m)$/|5A/;s/\|(Gm|Gm|Gm|11m)$/|6A/;s/\|(Dm|Dm|Dm|12m)$/|7A/;s/\|(Am|Am|Am|1m)$/|8A/;s/\|(Em|Em|Em|2m)$/|9A/;s/\|(Bm|Bm|Bm|3m)$/|10A/;s/\|(F#m|F#m|Gbm|4m)$/|11A/;s/\|(C#m|Dbm|Dbm|5m)$/|12A/;'
+
+
+# runner/wrapper
+runfun() { cores=8; touch run; tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, key.v from mt key join mt dur on key.w = dur.w where key.k = "key" and dur.k = ".dur" order by dur.w' | uniq -w16 | grep -vE '(Off-Key|None)$' | sed -r "s/ //g;$re" | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done| grep mir/cr | tr \| / | while read key fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); [ "$key" = "$py" ] && c=2 || c=5; printf '%4s sec, %4s orig, \033[3%dm%4s py,\033[0m %s\n' $td "$key" $c "$py" "$fn") || break; done & done; time wait 2>/dev/null; }
+
+
+# ok: 26   1off: 10   2off: 1   fail: 3   #  15 sec, keyfinder
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 -c 'import sys; import keyfinder; print(keyfinder.key(sys.argv[1]).camelot())' "/dev/shm/$core.wav")"; }; runfun
+
+
+# https://github.com/MTG/essentia/raw/master/src/examples/tutorial/example_key_by_steps_streaming.py
+# https://essentia.upf.edu/reference/std_Key.html  # edma edmm braw bgate
+sed -ri 's/^(key = Key\().*/\1profileType="bgate")/' example_key_by_steps_streaming.py
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 example_key_by_steps_streaming.py /dev/shm/$core.{wav,yml} 2>/dev/null | sed -r "s/ major//;s/ minor/m/;s/^/|/;$re;s/.//")"; }; runfun
+
+
+
+########################################################################
+##
+##  misc
+##
+########################################################################
+
+
+
+python3 -m pip install --user vamp
+
+import librosa
+d, r = librosa.load('/dev/shm/0.wav')
+d.dtype
+# dtype('float32')
+d.shape
+# (1323000,)
+d
+# array([-1.9614939e-08,  1.8037968e-08, -1.4106059e-08, ...,
+#         1.2024145e-01,  2.7462116e-01,  1.6202132e-01], dtype=float32)
+
+
+
+import vamp
+c = vamp.collect(d, r, "vamp-example-plugins:fixedtempo")
+c
+# {'list': [{'timestamp':  0.005804988, 'duration':  9.999092971, 'label': '110.0 bpm', 'values': array([109.98116], dtype=float32)}]}
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+d
+array([-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+       -0.0267825 , -0.03564296], dtype=float32)
+
+d = np.reshape(d, [1, -1])
+d
+array([[-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+        -0.0267825 , -0.03564296]], dtype=float32)
+
+
+
+import vampyhost
+print("\n".join(vampyhost.list_plugins()))
+
+mvamp:marsyas_bextract_centroid
+mvamp:marsyas_bextract_lpcc
+mvamp:marsyas_bextract_lsp
+mvamp:marsyas_bextract_mfcc
+mvamp:marsyas_bextract_rolloff
+mvamp:marsyas_bextract_scf
+mvamp:marsyas_bextract_sfm
+mvamp:marsyas_bextract_zero_crossings
+mvamp:marsyas_ibt
+mvamp:zerocrossing
+qm-vamp-plugins:qm-adaptivespectrogram
+qm-vamp-plugins:qm-barbeattracker
+qm-vamp-plugins:qm-chromagram
+qm-vamp-plugins:qm-constantq
+qm-vamp-plugins:qm-dwt
+qm-vamp-plugins:qm-keydetector
+qm-vamp-plugins:qm-mfcc
+qm-vamp-plugins:qm-onsetdetector
+qm-vamp-plugins:qm-segmenter
+qm-vamp-plugins:qm-similarity
+qm-vamp-plugins:qm-tempotracker
+qm-vamp-plugins:qm-tonalchange
+qm-vamp-plugins:qm-transcription
+vamp-aubio:aubiomelenergy
+vamp-aubio:aubiomfcc
+vamp-aubio:aubionotes
+vamp-aubio:aubioonset
+vamp-aubio:aubiopitch
+vamp-aubio:aubiosilence
+vamp-aubio:aubiospecdesc
+vamp-aubio:aubiotempo
+vamp-example-plugins:amplitudefollower
+vamp-example-plugins:fixedtempo
+vamp-example-plugins:percussiononsets
+vamp-example-plugins:powerspectrum
+vamp-example-plugins:spectralcentroid
+vamp-example-plugins:zerocrossing
+vamp-rubberband:rubberband
+
+
+
+plug = vampyhost.load_plugin("vamp-example-plugins:fixedtempo", 22050, 0)
+plug.info
+{'apiVersion': 2, 'pluginVersion': 1, 'identifier': 'fixedtempo', 'name': 'Simple Fixed Tempo Estimator', 'description': 'Study a short section of audio and estimate its tempo, assuming the tempo is constant', 'maker': 'Vamp SDK Example Plugins', 'copyright': 'Code copyright 2008 Queen Mary, University of London.  Freely redistributable (BSD license)'}
+plug = vampyhost.load_plugin("qm-vamp-plugins:qm-tempotracker", 22050, 0)
+from pprint import pprint; pprint(plug.parameters)
+
+
+
+for c in plug.parameters: print("{} \033[36m{}  [\033[33m{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], "\033[36m, \033[33m".join(c["valueNames"]), c["valueNames"][int(c["defaultValue"])])) if "valueNames" in c else print("{} \033[36m{}  [\033[33m{}..{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], c["minValue"], c["maxValue"], c["defaultValue"]))
+
+
+
+beatroot-vamp:beatroot
+cl=c["list"]; 60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"])))
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+# 128 bpm, key 5A Cm
+
+import vamp
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40})
+c["list"][0]["label"]
+# 127.6 bpm
+
+c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150})
+print("\n".join([v["label"] for v in c["list"] if v["label"]]))
+v = [float(x["label"].split(' ')[0]) for x in c["list"] if x["label"]]
+v = list(sorted(v))[len(v)//4:-len(v)//4]
+v = sum(v) / len(v)
+# 128.1 bpm
+

docs/notes.sh

@@ -3,6 +3,21 @@ echo not a script
 exit 1
 
 
+##
+## delete all partial uploads
+##  (supports linux/macos, probably windows+msys2)
+
+gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f -- "$f"; done
+gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done
+
+
+##
+## detect partial uploads based on file contents
+##  (in case of context loss or old copyparties)
+
+echo; find -type f | while IFS= read -r x; do printf '\033[A\033[36m%s\033[K\033[0m\n' "$x"; tail -c$((1024*1024)) <"$x" | xxd -a | awk 'NR==1&&/^[0: ]+.{16}$/{next} NR==2&&/^\*$/{next} NR==3&&/^[0f]+: [0 ]+65 +.{16}$/{next} {e=1} END {exit e}' || continue; printf '\033[A\033[31msus:\033[33m %s \033[0m\n\n' "$x"; done
+
+
 ##
 ## create a test payload
 
@@ -13,7 +28,7 @@ head -c $((2*1024*1024*1024)) /dev/zero | openssl enc -aes-256-ctr -pass pass:hu
 ## testing multiple parallel uploads
 ## usage:  para | tee log
 
-para() { for s in 1 2 3 4 5 6 7 8 12 16 24 32 48 64; do echo $s; for r in {1..4}; do for ((n=0;n<s;n++)); do curl -sF "act=bput" -F "f=@garbage.file" http://127.0.0.1:1234/ 2>&1 & done; wait; echo; done; done; }
+para() { for s in 1 2 3 4 5 6 7 8 12 16 24 32 48 64; do echo $s; for r in {1..4}; do for ((n=0;n<s;n++)); do curl -sF "act=bput" -F "f=@garbage.file" http://127.0.0.1:3923/ 2>&1 & done; wait; echo; done; done; }
 
 
 ##
@@ -36,13 +51,13 @@ for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qwe,rty;asd
 
 fn=$(printf '\xba\xdc\xab.cab')
 echo asdf > "$fn"
-curl --cookie cppwd=wark -sF "act=bput" -F "f=@$fn" http://127.0.0.1:1234/moji/%ED%91/
+curl --cookie cppwd=wark -sF "act=bput" -F "f=@$fn" http://127.0.0.1:3923/moji/%ED%91/
 
 
 ##
 ## test compression
 
-wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:1234/.cpr/deps/ogv.js -O- | md5sum; p=~ed/dev/copyparty/copyparty/web/deps/ogv.js.gz; md5sum $p; gzip -d < $p | md5sum
+wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:3923/.cpr/deps/ogv.js -O- | md5sum; p=~ed/dev/copyparty/copyparty/web/deps/ogv.js.gz; md5sum $p; gzip -d < $p | md5sum
 
 
 ##
@@ -52,6 +67,36 @@ wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:123
 shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*1024*1024)); printf $(tail -c +$((v+1)) "$f" | head -c $((w-v)) | sha512sum | cut -c-64 | sed -r 's/ .*//;s/(..)/\\x\1/g') | base64 -w0 | cut -c-43 | tr '+/' '-_'; v=$w; [ $v -lt $sz ] || break; done; }
 
 
+##
+## poll url for performance issues
+
+command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
+
+
+##
+## sqlite3 stuff
+
+# find dupe metadata keys
+sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = mt2.k and mt1.rowid != mt2.rowid'
+
+# partial reindex by deleting all tags for a list of files
+time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
+cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
+
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
+
+##
+## media
+
+# split track into test files
+e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
+
+-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
+sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
+
+
 ##
 ## vscode
 
@@ -80,3 +125,57 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 # py2 on osx
 brew install python@2
 pip install virtualenv
+
+# readme toc
+cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
+
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
+##
+## http 206
+
+# az = abcdefghijklmnopqrstuvwxyz
+
+printf '%s\r\n' 'GET /az HTTP/1.1' 'Host: ocv.me' 'Range: bytes=5-10' '' | ncat ocv.me 80 
+# Content-Range: bytes 5-10/26
+# Content-Length: 6
+# fghijk
+
+Range: bytes=0-1    "ab" Content-Range: bytes 0-1/26
+Range: bytes=24-24  "y"  Content-Range: bytes 24-24/26
+Range: bytes=24-25  "yz" Content-Range: bytes 24-25/26
+Range: bytes=24-    "yz" Content-Range: bytes 24-25/26
+Range: bytes=25-29  "z"  Content-Range: bytes 25-25/26
+Range: bytes=26-         Content-Range: bytes */26
+  HTTP/1.1 416 Requested Range Not Satisfiable
+
+
+##
+## md perf
+
+var tsh = [];
+function convert_markdown(md_text, dest_dom) {
+    tsh.push(new Date().getTime());
+    while (tsh.length > 10)
+        tsh.shift();
+    if (tsh.length > 1) {
+        var end = tsh.slice(-2);
+        console.log("render", end.pop() - end.pop(), (tsh[tsh.length - 1] - tsh[0]) / (tsh.length - 1));
+    }
+
+
+##
+## tmpfiles.d meme
+
+mk() { rm -rf /tmp/foo; sudo -u ed bash -c 'mkdir /tmp/foo; echo hi > /tmp/foo/bar'; }
+mk && t0="$(date)" && while true; do date -s "$(date '+ 1 hour')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
+mk && sudo -u ed flock /tmp/foo sleep 40 & sleep 1; ps aux | grep -E 'sleep 40$' && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
+mk && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; tar -cf/dev/null /tmp/foo; done; echo "$t0"

docs/pretend-youre-qnap.patch

@@ -0,0 +1,35 @@
+diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py
+index 2d3c1ad..e1e85a0 100644
+--- a/copyparty/httpcli.py
++++ b/copyparty/httpcli.py
+@@ -864,6 +864,30 @@ class HttpCli(object):
+         #
+         # send reply
+ 
++        try:
++            fakefn = self.conn.hsrv.fakefn
++            fakectr = self.conn.hsrv.fakectr
++            fakedata = self.conn.hsrv.fakedata
++        except:
++            fakefn = b''
++            fakectr = 0
++            fakedata = b''
++        
++        self.log('\n{} {}\n{}'.format(fakefn, fakectr, open_args[0]))
++        if fakefn == open_args[0] and fakectr > 0:
++            self.reply(fakedata, mime=guess_mime(req_path)[0])
++            self.conn.hsrv.fakectr = fakectr - 1
++        else:
++            with open_func(*open_args) as f:
++                fakedata = f.read()
++            
++            self.conn.hsrv.fakefn = open_args[0]
++            self.conn.hsrv.fakedata = fakedata
++            self.conn.hsrv.fakectr = 15
++            self.reply(fakedata, mime=guess_mime(req_path)[0])
++        
++        return True
++
+         self.out_headers["Accept-Ranges"] = "bytes"
+         self.send_headers(
+             length=upper - lower,

docs/rclone.md

@@ -0,0 +1,62 @@
+# using rclone to mount a remote copyparty server as a local filesystem
+
+speed estimates with server and client on the same win10 machine:
+* `1070 MiB/s` with rclone as both server and client
+* `570 MiB/s` with rclone-client and `copyparty -ed -j16` as server
+* `220 MiB/s` with rclone-client and `copyparty -ed` as server
+* `100 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+
+when server is on another machine (1gbit LAN),
+* `75 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+* `92 MiB/s` with rclone-client and `copyparty -ed` as server
+* `103 MiB/s` (connection max) with `copyparty -ed -j16` and all the others
+
+
+# creating the config file
+
+if you want to use password auth, add `headers = Cookie,cppwd=fgsfds` below
+
+
+### on windows clients:
+```
+(
+echo [cpp]
+echo type = http
+echo url = http://127.0.0.1:3923/
+) > %userprofile%\.config\rclone\rclone.conf
+```
+
+also install the windows dependencies: [winfsp](https://github.com/billziss-gh/winfsp/releases/latest)
+
+
+### on unix clients:
+```
+cat > ~/.config/rclone/rclone.conf <<'EOF'
+[cpp]
+type = http
+url = http://127.0.0.1:3923/
+EOF
+```
+
+
+# mounting the copyparty server locally
+```
+rclone.exe mount --vfs-cache-max-age 5s --attr-timeout 5s --dir-cache-time 5s cpp: Z:
+```
+
+
+# use rclone as server too, replacing copyparty
+
+feels out of place but is too good not to mention
+
+```
+rclone.exe serve http --read-only .
+```
+
+* `webdav` gives write-access but `http` is twice as fast
+* `ftp` is buggy, avoid
+
+
+# bugs
+
+* rclone-client throws an exception if you try to read an empty file (should return zero bytes)

docs/unirange.py

@@ -0,0 +1,10 @@
+v = "U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD"
+for v in v.split(","):
+    if "+" in v:
+        v = v.split("+")[1]
+    if "-" in v:
+        lo, hi = v.split("-")
+    else:
+        lo = hi = v
+    for v in range(int(lo, 16), int(hi, 16) + 1):
+        print("{:4x} [{}]".format(v, chr(v)))

scripts/copyparty-android.sh

@@ -5,8 +5,6 @@ _msg() { printf "$2"'\033[1;30m>\033[0;33m>\033[1m>\033[0m %s\n' "$1" >&2; }
 imsg() { _msg "$1" ''; }
 msg() { _msg "$1" \\n; }
 
-mkdir -p ~/src
-
 ##
 ## helper which installs termux packages
 
@@ -24,22 +22,6 @@ addpkg() {
 	apt install -y $1
 }
 
-##
-## ensure git and copyparty is available
-
-[ -e ~/src/copyparty/.ok ] || {
-	command -v git >/dev/null ||
-		addpkg git
-	
-	msg "downloading copyparty from github"
-	(
-		cd ~/src
-		rm -rf copyparty
-		git clone https://github.com/9001/copyparty
-		touch copyparty/.ok
-	)
-}
-
 ##
 ## ensure python is available
 
@@ -49,20 +31,21 @@ command -v python3 >/dev/null ||
 ##
 ## ensure virtualenv and dependencies are available
 
-[ -e ~/src/copyparty/.env/.ok ] || { (
-	cd ~/src/copyparty
-	rm -rf .env
+ve=$HOME/ve.copyparty
+
+[ -e $ve/.ok ] || (
+	rm -rf $ve
 
 	msg "creating python3 virtualenv"
-	python3 -m venv .env
+	python3 -m venv $ve
 
-	msg "installing python dependencies"
-	. .env/bin/activate
-	pip install jinja2
+	msg "installing copyparty"
+	. $ve/bin/activate
+	pip install copyparty
 
 	deactivate
-	touch .env/.ok
-) }
+	touch $ve/.ok
+)
 
 ##
 ## add copyparty alias to bashrc
@@ -75,9 +58,5 @@ grep -qE '^alias copyparty=' ~/.bashrc 2>/dev/null || {
 ##
 ## start copyparty
 
-imsg "activating virtualenv"
-. ~/src/copyparty/.env/bin/activate
-
 imsg "starting copyparty"
-PYTHONPATH=~/src/copyparty python3 -m copyparty "$@"
-
+$ve/bin/python -m copyparty "$@"

scripts/copyparty-repack.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+repacker=1
+set -e
+
+# -- download latest copyparty (source.tgz and sfx),
+# -- build minimal sfx versions,
+# -- create a .tar.gz bundle
+#
+# convenient for deploying updates to inconvenient locations
+#  (and those are usually linux so bash is good inaff)
+#   (but that said this even has macos support)
+#
+# bundle will look like:
+# -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty
+# -rw-r--r--  0 ed ed  491318 Nov 19 00:40 copyparty-extras/copyparty-0.5.4.tar.gz
+# -rwxr-xr-x  0 ed ed   30254 Nov 17 23:58 copyparty-extras/copyparty-fuse.py
+# -rwxr-xr-x  0 ed ed  481403 Nov 19 00:40 copyparty-extras/sfx-full/copyparty-sfx.sh
+# -rwxr-xr-x  0 ed ed  506043 Nov 19 00:40 copyparty-extras/sfx-full/copyparty-sfx.py
+# -rwxr-xr-x  0 ed ed  167699 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.sh
+# -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+td="$(mktemp -d)"
+od="$(pwd)"
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	command -v curl >/dev/null && exec curl -L --remote-name-all "$@"
+	exec wget "$@"
+}
+export -f dl_files
+
+
+# if cache exists, use that instead of bothering github
+cache="$od/.copyparty-repack.cache"
+[ -e "$cache" ] &&
+	tar -xf "$cache" ||
+{
+	# get download links from github
+	dl_text https://api.github.com/repos/9001/copyparty/releases/latest |
+	(
+		# prefer jq if available
+		jq -r '.assets[]|select(.name|test("-sfx|tar.gz")).browser_download_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"browser_download_url".*(\.tar\.gz|-sfx\.)/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+
+	tar -czf "$cache" *
+}
+
+
+# move src into copyparty-extras/,
+# move sfx into copyparty-extras/sfx-full/
+mkdir -p copyparty-extras/sfx-{full,lite}
+mv copyparty-sfx.* copyparty-extras/sfx-full/
+mv copyparty-*.tar.gz copyparty-extras/
+
+
+# unpack the source code
+( cd copyparty-extras/
+tar -xf *.tar.gz
+)
+
+
+# use repacker from release if that is newer
+p_other=copyparty-extras/copyparty-*/scripts/copyparty-repack.sh
+other=$(awk -F= 'BEGIN{v=-1} NR<10&&/^repacker=/{v=$NF} END{print v}' <$p_other) 
+[ $repacker -lt $other ] &&
+  cat $p_other >"$od/$0" && cd "$od" && rm -rf "$td" && exec "$0" "$@"
+
+
+# now drop the cache
+rm -f "$cache"
+
+
+# fix permissions
+chmod 755 \
+  copyparty-extras/sfx-full/* \
+  copyparty-extras/copyparty-*/{scripts,bin}/*
+
+
+# extract and repack the sfx with less features enabled
+( cd copyparty-extras/sfx-full/
+./copyparty-sfx.py -h
+cd ../copyparty-*/
+./scripts/make-sfx.sh re no-ogv no-cm
+)
+
+
+# put new sfx into copyparty-extras/sfx-lite/,
+# fuse client into copyparty-extras/,
+# copy lite-sfx.py to ./copyparty,
+# delete extracted source code
+( cd copyparty-extras/
+mv copyparty-*/dist/* sfx-lite/
+mv copyparty-*/bin/copyparty-fuse.py .
+cp -pv sfx-lite/copyparty-sfx.py ../copyparty
+rm -rf copyparty-{0..9}*.*.*{0..9}
+)
+
+
+# and include the repacker itself too
+cp -av "$od/$0" copyparty-extras/ ||
+cp -av "$0" copyparty-extras/ ||
+true
+
+
+# create the bundle
+fn=copyparty-$(date +%Y-%m%d-%H%M%S).tgz
+tar -czvf "$od/$fn" *
+cd "$od"
+rm -rf "$td"
+
+
+echo
+echo "done, here's your bundle:"
+ls -al "$fn"

scripts/deps-docker/Dockerfile

@@ -1,17 +1,58 @@
-FROM    alpine:3.10
+FROM    alpine:3.13
 WORKDIR /z
-ENV     ver_asmcrypto=2821dd1dedd1196c378f5854037dda5c869313f3 \
-        ver_ogvjs=1.6.1
+ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
+        ver_marked=1.1.0 \
+        ver_ogvjs=1.8.0 \
+        ver_mde=2.14.0 \
+        ver_codemirror=5.59.3 \
+        ver_fontawesome=5.13.0 \
+        ver_zopfli=1.0.3
+
+
+# download;
+# the scp url is latin from https://fonts.googleapis.com/css2?family=Source+Code+Pro&display=swap
+RUN     mkdir -p /z/dist/no-pk \
+        && wget https://fonts.gstatic.com/s/sourcecodepro/v11/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevW.woff2 -O scp.woff2 \
+        && apk add cmake make g++ git bash npm patch wget tar pigz brotli gzip unzip python3 python3-dev brotli py3-brotli \
+        && wget https://github.com/brion/ogv.js/releases/download/$ver_ogvjs/ogvjs-$ver_ogvjs.zip -O ogvjs.zip \
+        && wget https://github.com/openpgpjs/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
+        && wget https://github.com/markedjs/marked/archive/v$ver_marked.tar.gz -O marked.tgz \
+        && wget https://github.com/Ionaru/easy-markdown-editor/archive/$ver_mde.tar.gz -O mde.tgz \
+        && wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
+        && wget https://github.com/FortAwesome/Font-Awesome/releases/download/$ver_fontawesome/fontawesome-free-$ver_fontawesome-web.zip -O fontawesome.zip \
+        && wget https://github.com/google/zopfli/archive/zopfli-$ver_zopfli.tar.gz -O zopfli.tgz \
+        && unzip ogvjs.zip \
+        && (tar -xf asmcrypto.tgz \
+            && cd asmcrypto.js-$ver_asmcrypto \
+            && npm install ) \
+        && (tar -xf marked.tgz \
+            && cd marked-$ver_marked \
+            && npm install \
+            && npm i grunt uglify-js -g ) \
+        && (tar -xf codemirror.tgz \
+            && cd CodeMirror-$ver_codemirror \
+            && npm install ) \
+        && (tar -xf mde.tgz \
+            && cd easy-markdown-editor* \
+            && npm install \
+            && npm i gulp-cli -g ) \
+        && unzip fontawesome.zip \
+        && tar -xf zopfli.tgz
+
+
+# build fonttools (which needs zopfli)
+RUN     tar -xf zopfli.tgz \
+        && cd zopfli* \
+        && cmake \
+            -DCMAKE_INSTALL_PREFIX=/usr \
+            -DZOPFLI_BUILD_SHARED=ON \
+            -B build \
+            -S . \
+        && make -C build \
+        && make -C build install \
+        && python3 -m ensurepip \
+        && python3 -m pip install fonttools zopfli
 
-# download
-RUN     apk add make g++ git bash npm patch wget tar pigz gzip unzip \
-        && wget https://github.com/brion/ogv.js/releases/download/$ver_ogvjs/ogvjs-$ver_ogvjs.zip \
-        && wget https://github.com/asmcrypto/asmcrypto.js/archive/$ver_asmcrypto.tar.gz \
-        && unzip ogvjs-$ver_ogvjs.zip \
-        && tar -xf $ver_asmcrypto.tar.gz \
-        && cd asmcrypto.js-$ver_asmcrypto \
-        && npm install \
-        && mkdir /z/dist
 
 # build asmcrypto
 RUN     cd asmcrypto.js-$ver_asmcrypto \
@@ -19,28 +60,84 @@ RUN     cd asmcrypto.js-$ver_asmcrypto \
         && node -r esm build.js \
         && mv asmcrypto.all.es5.js /z/dist/sha512.js
 
+
 # build ogvjs
 RUN     cd ogvjs-$ver_ogvjs \
         && cp -pv \
             ogv.js \
             ogv-worker-audio.js \
-            ogv-demuxer-ogg.js \
             ogv-demuxer-ogg-wasm.js \
             ogv-demuxer-ogg-wasm.wasm \
-            ogv-demuxer-webm.js \
             ogv-demuxer-webm-wasm.js \
             ogv-demuxer-webm-wasm.wasm \
-            ogv-decoder-audio-opus.js \
             ogv-decoder-audio-opus-wasm.js \
             ogv-decoder-audio-opus-wasm.wasm \
-            ogv-decoder-audio-vorbis.js \
             ogv-decoder-audio-vorbis-wasm.js \
             ogv-decoder-audio-vorbis-wasm.wasm \
-            dynamicaudio.swf \
             /z/dist
 
+#            ogv-demuxer-ogg.js \
+#            ogv-demuxer-webm.js \
+#            ogv-decoder-audio-opus.js \
+#            ogv-decoder-audio-vorbis.js \
+#            dynamicaudio.swf \
+
+
+# build marked
+COPY    marked.patch /z/
+COPY    marked-ln.patch /z/
+RUN     cd marked-$ver_marked \
+        && patch -p1 < /z/marked-ln.patch \
+        && patch -p1 < /z/marked.patch \
+        && npm run build \
+        && cp -pv marked.min.js /z/dist/marked.js \
+        && cp -pv lib/marked.js /z/dist/marked.full.js \
+        && mkdir -p /z/nodepkgs \
+        && ln -s $(pwd) /z/nodepkgs/marked
+#        && npm run test \
+
+
+# build codemirror
+COPY    codemirror.patch /z/
+RUN     cd CodeMirror-$ver_codemirror \
+        && patch -p1 < /z/codemirror.patch \
+        && sed -ri '/^var urlRE = /d' mode/gfm/gfm.js \
+        && npm run build \
+        && ln -s $(pwd) /z/nodepkgs/codemirror
+
+
+# build easymde
+COPY    easymde.patch /z/
+RUN     cd easy-markdown-editor-$ver_mde \
+        && patch -p1 < /z/easymde.patch \
+        && sed -ri 's`https://registry.npmjs.org/marked/-/marked-0.8.2.tgz`file:/z/nodepkgs/marked`' package-lock.json \
+        && sed -ri 's`("marked": ")[^"]+`\1file:/z/nodepkgs/marked`' ./package.json \
+        && sed -ri 's`("codemirror": ")[^"]+`\1file:/z/nodepkgs/codemirror`' ./package.json \
+        && npm install
+
+COPY    easymde-ln.patch /z/
+RUN     cd easy-markdown-editor-$ver_mde \
+        && patch -p1 < /z/easymde-ln.patch \
+        && gulp \
+        && cp -pv dist/easymde.min.css /z/dist/easymde.css \
+        && cp -pv dist/easymde.min.js /z/dist/easymde.js
+
+
+# build fontawesome and scp
+COPY    mini-fa.sh /z
+COPY    mini-fa.css /z
+RUN     /bin/ash /z/mini-fa.sh
+
+
 # compress
 COPY    zopfli.makefile /z/dist/Makefile
 RUN     cd /z/dist \
         && make -j$(nproc) \
-        && rm Makefile
+        && rm Makefile \
+        && mv no-pk/* . \
+        && rmdir no-pk
+
+
+# git diff -U2 --no-index marked-1.1.0-orig/ marked-1.1.0-edit/ -U2 | sed -r '/^index /d;s`^(diff --git a/)[^/]+/(.* b/)[^/]+/`\1\2`; s`^(---|\+\+\+) ([ab]/)[^/]+/`\1 \2`' > ../dev/copyparty/scripts/deps-docker/marked-ln.patch
+# d=/home/ed/dev/copyparty/scripts/deps-docker/; tar -cf ../x . && ssh root@$bip "cd $d && tar -xv >&2 && make >&2 && tar -cC ../../copyparty/web deps" <../x | (cd ../../copyparty/web/; cat > the.tgz; tar -xvf the.tgz; rm the.tgz)
+# gzip -dkf ../dev/copyparty/copyparty/web/deps/deps/marked.full.js.gz && diff -NarU2 ../dev/copyparty/copyparty/web/deps/{,deps/}marked.full.js

scripts/deps-docker/codemirror.patch

@@ -0,0 +1,276 @@
+diff -NarU2 codemirror-5.59.3-orig/mode/gfm/gfm.js codemirror-5.59.3/mode/gfm/gfm.js
+--- codemirror-5.59.3-orig/mode/gfm/gfm.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/mode/gfm/gfm.js	2021-02-21 20:42:02.166174775 +0000
+@@ -97,5 +97,5 @@
+         }
+       }
+-      if (stream.match(urlRE) &&
++      /*if (stream.match(urlRE) &&
+           stream.string.slice(stream.start - 2, stream.start) != "](" &&
+           (stream.start == 0 || /\W/.test(stream.string.charAt(stream.start - 1)))) {
+@@ -106,5 +106,5 @@
+         state.combineTokens = true;
+         return "link";
+-      }
++      }*/
+       stream.next();
+       return null;
+diff -NarU2 codemirror-5.59.3-orig/mode/meta.js codemirror-5.59.3/mode/meta.js
+--- codemirror-5.59.3-orig/mode/meta.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/mode/meta.js	2021-02-21 20:42:54.798742821 +0000
+@@ -13,4 +13,5 @@
+ 
+   CodeMirror.modeInfo = [
++    /*
+     {name: "APL", mime: "text/apl", mode: "apl", ext: ["dyalog", "apl"]},
+     {name: "PGP", mimes: ["application/pgp", "application/pgp-encrypted", "application/pgp-keys", "application/pgp-signature"], mode: "asciiarmor", ext: ["asc", "pgp", "sig"]},
+@@ -56,5 +57,7 @@
+     {name: "Gas", mime: "text/x-gas", mode: "gas", ext: ["s"]},
+     {name: "Gherkin", mime: "text/x-feature", mode: "gherkin", ext: ["feature"]},
++    */
+     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history)\.md$/i},
++    /*
+     {name: "Go", mime: "text/x-go", mode: "go", ext: ["go"]},
+     {name: "Groovy", mime: "text/x-groovy", mode: "groovy", ext: ["groovy", "gradle"], file: /^Jenkinsfile$/},
+@@ -65,5 +68,7 @@
+     {name: "HXML", mime: "text/x-hxml", mode: "haxe", ext: ["hxml"]},
+     {name: "ASP.NET", mime: "application/x-aspx", mode: "htmlembedded", ext: ["aspx"], alias: ["asp", "aspx"]},
++    */
+     {name: "HTML", mime: "text/html", mode: "htmlmixed", ext: ["html", "htm", "handlebars", "hbs"], alias: ["xhtml"]},
++    /*
+     {name: "HTTP", mime: "message/http", mode: "http"},
+     {name: "IDL", mime: "text/x-idl", mode: "idl", ext: ["pro"]},
+@@ -82,5 +87,7 @@
+     {name: "LiveScript", mime: "text/x-livescript", mode: "livescript", ext: ["ls"], alias: ["ls"]},
+     {name: "Lua", mime: "text/x-lua", mode: "lua", ext: ["lua"]},
++    */
+     {name: "Markdown", mime: "text/x-markdown", mode: "markdown", ext: ["markdown", "md", "mkd"]},
++    /*
+     {name: "mIRC", mime: "text/mirc", mode: "mirc"},
+     {name: "MariaDB SQL", mime: "text/x-mariadb", mode: "sql"},
+@@ -163,5 +170,7 @@
+     {name: "VHDL", mime: "text/x-vhdl", mode: "vhdl", ext: ["vhd", "vhdl"]},
+     {name: "Vue.js Component", mimes: ["script/x-vue", "text/x-vue"], mode: "vue", ext: ["vue"]},
++    */
+     {name: "XML", mimes: ["application/xml", "text/xml"], mode: "xml", ext: ["xml", "xsl", "xsd", "svg"], alias: ["rss", "wsdl", "xsd"]},
++    /*
+     {name: "XQuery", mime: "application/xquery", mode: "xquery", ext: ["xy", "xquery"]},
+     {name: "Yacas", mime: "text/x-yacas", mode: "yacas", ext: ["ys"]},
+@@ -172,4 +181,5 @@
+     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]},
+     {name: "WebAssembly", mime: "text/webassembly", mode: "wast", ext: ["wat", "wast"]},
++    */
+   ];
+   // Ensure all modes have a mime property for backwards compatibility
+diff -NarU2 codemirror-5.59.3-orig/src/display/selection.js codemirror-5.59.3/src/display/selection.js
+--- codemirror-5.59.3-orig/src/display/selection.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/display/selection.js	2021-02-21 20:44:14.860894328 +0000
+@@ -84,29 +84,21 @@
+     let order = getOrder(lineObj, doc.direction)
+     iterateBidiSections(order, fromArg || 0, toArg == null ? lineLen : toArg, (from, to, dir, i) => {
+-      let ltr = dir == "ltr"
+-      let fromPos = coords(from, ltr ? "left" : "right")
+-      let toPos = coords(to - 1, ltr ? "right" : "left")
++      let fromPos = coords(from, "left")
++      let toPos = coords(to - 1, "right")
+ 
+       let openStart = fromArg == null && from == 0, openEnd = toArg == null && to == lineLen
+       let first = i == 0, last = !order || i == order.length - 1
+       if (toPos.top - fromPos.top <= 3) { // Single line
+-        let openLeft = (docLTR ? openStart : openEnd) && first
+-        let openRight = (docLTR ? openEnd : openStart) && last
+-        let left = openLeft ? leftSide : (ltr ? fromPos : toPos).left
+-        let right = openRight ? rightSide : (ltr ? toPos : fromPos).right
++        let openLeft = openStart && first
++        let openRight =  openEnd && last
++        let left = openLeft ? leftSide : fromPos.left
++        let right = openRight ? rightSide : toPos.right
+         add(left, fromPos.top, right - left, fromPos.bottom)
+       } else { // Multiple lines
+         let topLeft, topRight, botLeft, botRight
+-        if (ltr) {
+-          topLeft = docLTR && openStart && first ? leftSide : fromPos.left
+-          topRight = docLTR ? rightSide : wrapX(from, dir, "before")
+-          botLeft = docLTR ? leftSide : wrapX(to, dir, "after")
+-          botRight = docLTR && openEnd && last ? rightSide : toPos.right
+-        } else {
+-          topLeft = !docLTR ? leftSide : wrapX(from, dir, "before")
+-          topRight = !docLTR && openStart && first ? rightSide : fromPos.right
+-          botLeft = !docLTR && openEnd && last ? leftSide : toPos.left
+-          botRight = !docLTR ? rightSide : wrapX(to, dir, "after")
+-        }
++          topLeft = openStart && first ? leftSide : fromPos.left
++          topRight = rightSide
++          botLeft = leftSide
++          botRight = openEnd && last ? rightSide : toPos.right
+         add(topLeft, fromPos.top, topRight - topLeft, fromPos.bottom)
+         if (fromPos.bottom < toPos.top) add(leftSide, fromPos.bottom, null, toPos.top)
+diff -NarU2 codemirror-5.59.3-orig/src/input/ContentEditableInput.js codemirror-5.59.3/src/input/ContentEditableInput.js
+--- codemirror-5.59.3-orig/src/input/ContentEditableInput.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/input/ContentEditableInput.js	2021-02-21 20:44:33.273953867 +0000
+@@ -399,4 +399,5 @@
+   let info = mapFromLineView(view, line, pos.line)
+ 
++  /*
+   let order = getOrder(line, cm.doc.direction), side = "left"
+   if (order) {
+@@ -404,4 +405,5 @@
+     side = partPos % 2 ? "right" : "left"
+   }
++  */
+   let result = nodeAndOffsetInLineMap(info.map, pos.ch, side)
+   result.offset = result.collapse == "right" ? result.end : result.start
+diff -NarU2 codemirror-5.59.3-orig/src/input/movement.js codemirror-5.59.3/src/input/movement.js
+--- codemirror-5.59.3-orig/src/input/movement.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/input/movement.js	2021-02-21 20:45:12.763093671 +0000
+@@ -15,4 +15,5 @@
+ 
+ export function endOfLine(visually, cm, lineObj, lineNo, dir) {
++  /*
+   if (visually) {
+     if (cm.doc.direction == "rtl") dir = -dir
+@@ -39,8 +40,11 @@
+     }
+   }
++  */
+   return new Pos(lineNo, dir < 0 ? lineObj.text.length : 0, dir < 0 ? "before" : "after")
+ }
+ 
+ export function moveVisually(cm, line, start, dir) {
++  return moveLogically(line, start, dir)
++  /*
+   let bidi = getOrder(line, cm.doc.direction)
+   if (!bidi) return moveLogically(line, start, dir)
+@@ -109,3 +113,4 @@
+   // Case 4: Nowhere to move
+   return null
++  */
+ }
+diff -NarU2 codemirror-5.59.3-orig/src/line/line_data.js codemirror-5.59.3/src/line/line_data.js
+--- codemirror-5.59.3-orig/src/line/line_data.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/line/line_data.js	2021-02-21 20:45:36.472549599 +0000
+@@ -79,6 +79,6 @@
+     // Optionally wire in some hacks into the token-rendering
+     // algorithm, to deal with browser quirks.
+-    if (hasBadBidiRects(cm.display.measure) && (order = getOrder(line, cm.doc.direction)))
+-      builder.addToken = buildTokenBadBidi(builder.addToken, order)
++    //if (hasBadBidiRects(cm.display.measure) && (order = getOrder(line, cm.doc.direction)))
++    //  builder.addToken = buildTokenBadBidi(builder.addToken, order)
+     builder.map = []
+     let allowFrontierUpdate = lineView != cm.display.externalMeasured && lineNo(line)
+diff -NarU2 codemirror-5.59.3-orig/src/measurement/position_measurement.js codemirror-5.59.3/src/measurement/position_measurement.js
+--- codemirror-5.59.3-orig/src/measurement/position_measurement.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/measurement/position_measurement.js	2021-02-21 20:50:52.372945293 +0000
+@@ -380,5 +380,6 @@
+     sticky = "after"
+   }
+-  if (!order) return get(sticky == "before" ? ch - 1 : ch, sticky == "before")
++  /*if (!order)*/ return get(sticky == "before" ? ch - 1 : ch, sticky == "before")
++  /*
+ 
+   function getBidi(ch, partPos, invert) {
+@@ -391,4 +392,5 @@
+   if (other != null) val.other = getBidi(ch, other, sticky != "before")
+   return val
++  */
+ }
+ 
+@@ -468,4 +470,5 @@
+   let begin = 0, end = lineObj.text.length, ltr = true
+ 
++  /*
+   let order = getOrder(lineObj, cm.doc.direction)
+   // If the line isn't plain left-to-right text, first figure out
+@@ -482,4 +485,5 @@
+     end = ltr ? part.to : part.from - 1
+   }
++  */
+ 
+   // A binary search to find the first character whose bounding box
+@@ -526,4 +530,5 @@
+ }
+ 
++/*
+ function coordsBidiPart(cm, lineObj, lineNo, preparedMeasure, order, x, y) {
+   // Bidi parts are sorted left-to-right, and in a non-line-wrapping
+@@ -580,4 +585,5 @@
+   return part
+ }
++*/
+ 
+ let measureText
+diff -NarU2 codemirror-5.59.3-orig/src/util/bidi.js codemirror-5.59.3/src/util/bidi.js
+--- codemirror-5.59.3-orig/src/util/bidi.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/util/bidi.js	2021-02-21 20:52:18.168092225 +0000
+@@ -4,5 +4,5 @@
+ 
+ export function iterateBidiSections(order, from, to, f) {
+-  if (!order) return f(from, to, "ltr", 0)
++  /*if (!order)*/ return f(from, to, "ltr", 0) /*
+   let found = false
+   for (let i = 0; i < order.length; ++i) {
+@@ -14,4 +14,5 @@
+   }
+   if (!found) f(from, to, "ltr")
++*/
+ }
+ 
+@@ -32,5 +33,7 @@
+     }
+   }
+-  return found != null ? found : bidiOther
++  var ret = found != null ? found : bidiOther
++  console.log("getBidiPartAt(%s,%s,%s) => [%s]", order, ch, sticky, ret)
++  return ret
+ }
+ 
+@@ -55,4 +58,7 @@
+ // N (ON):  Other Neutrals
+ 
++let bidiOrdering = (function() { return function(str, direction) { return false; }})();
++/*
++
+ // Returns null if characters are ordered as they appear
+ // (left-to-right), or an array of sections ({from, to, level}
+@@ -81,5 +87,5 @@
+   }
+ 
+-  return function(str, direction) {
++  var fun = function(str, direction) {
+     let outerType = direction == "ltr" ? "L" : "R"
+ 
+@@ -204,5 +210,11 @@
+     return direction == "rtl" ? order.reverse() : order
+   }
++  return function(str, direction) {
++    var ret = fun(str, direction);
++    console.log("bidiOrdering inner ([%s], %s) => [%s]", str, direction, ret);
++    return ret;
++  }
+ })()
++*/
+ 
+ // Get the bidi ordering for the given line (and cache it). Returns
+@@ -210,6 +222,4 @@
+ // BidiSpan objects otherwise.
+ export function getOrder(line, direction) {
+-  let order = line.order
+-  if (order == null) order = line.order = bidiOrdering(line.text, direction)
+-  return order
++  return false;
+ }
+diff -NarU2 codemirror-5.59.3-orig/src/util/feature_detection.js codemirror-5.59.3/src/util/feature_detection.js
+--- codemirror-5.59.3-orig/src/util/feature_detection.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/util/feature_detection.js	2021-02-21 20:49:22.191269270 +0000
+@@ -25,4 +25,5 @@
+ }
+ 
++/*
+ // Feature-detect IE's crummy client rect reporting for bidi text
+ let badBidiRects
+@@ -36,4 +37,5 @@
+   return badBidiRects = (r1.right - r0.right < 3)
+ }
++*/
+ 
+ // See if "".split is the broken IE version, if so, provide an

scripts/deps-docker/easymde-ln.patch

@@ -0,0 +1,95 @@
+diff -NarU2 easymde-mod1/src/js/easymde.js easymde-edit/src/js/easymde.js
+--- easymde-mod1/src/js/easymde.js	2020-05-01 14:34:19.878774400 +0200
++++ easymde-edit/src/js/easymde.js	2020-05-01 21:24:44.142611200 +0200
+@@ -2189,4 +2189,5 @@
+ };
+ 
++
+ EasyMDE.prototype.createSideBySide = function () {
+     var cm = this.codemirror;
+@@ -2223,12 +2224,80 @@
+         }
+         pScroll = true;
+-        var height = v.getScrollInfo().height - v.getScrollInfo().clientHeight;
+-        var ratio = parseFloat(v.getScrollInfo().top) / height;
+-        var move = (preview.scrollHeight - preview.clientHeight) * ratio;
+-        preview.scrollTop = move;
++        var md_vp = v.getScrollInfo();
++        // viewport top: top
++        // viewport size: clientHeight
++        // document size: height
++        var md_scroll_y = md_vp.top + md_vp.clientHeight / 2;
++        var md_center_n = cm.lineAtHeight(md_scroll_y, 'local') + 1;
++        var md_next_n = md_center_n;
++        var md_top = cm.heightAtLine(md_center_n - 1, 'local');
++        while (md_next_n < cm.lineCount())
++            if (cm.getLine(md_next_n++).replace(/\s+/g, '').length > 0)
++                break;
++        var md_end = cm.heightAtLine(md_next_n - 1, 'local');
++        var md_frac = (md_scroll_y - md_top) * 1.0 / (md_end - md_top);
++        var get_pre_line = function(line_n, increase) {
++            var end = 0;
++            var step = -1;
++            if (increase) {
++                step = 1;
++                end = line_n + 1000;
++            }
++            // there might be multiple elements in the marked.js output,
++            // take the element with the biggest height
++            var biggest = -1;
++            var line_dom = null;
++            for (; line_n != end; line_n += step) {
++                var hits = document.querySelectorAll('.editor-preview-side *[data-ln=\'' + line_n + '\']');
++                for (var i = 0; i < hits.length; i++) {
++                    var hit_size = hits[i].offsetHeight;
++                    if (biggest < hit_size) {
++                        biggest = hit_size;
++                        line_dom = hits[i];
++                    }
++                }
++                if (line_dom) {
++                    var ret_y = 0;
++                    var el = line_dom;
++                    while (el && (el.getAttribute('class') + '').indexOf('editor-preview-side') < 0) {
++                        ret_y += el.offsetTop;
++                        el = el.offsetParent;
++                    }
++                    return [line_n, line_dom, ret_y];
++                }
++            }
++            return null;
++        };
++        var pre1 = get_pre_line(md_center_n, false);
++        var pre2 = get_pre_line(pre1[0] + 1, true) ||
++            [cm.lineCount(), null, preview.scrollHeight];
++        
++        //console.log('code-center %d, frac %.2f, pre [%d,%d] [%d,%d]',
++        //    md_center_n, md_frac, pre1[0], pre1[2], pre2[0], pre2[2]);
++        
++        // [0] is the markdown line which matches that preview y-pos
++        // and since not all preview lines are tagged with a line-number
++        // take the lineno diff and divide it by the distance
++        var pre_frac = md_frac / ((pre2[0] - pre1[0]) / (md_next_n - md_center_n));
++
++        // then use that fraction for the scroll offset
++        var pre_y = pre1[2] + (pre2[2] - pre1[2]) * pre_frac;
++
++        // unless we couldn't match the markdown line exactly to any preview line
++        if (md_center_n > pre1[0] && md_center_n < pre2[0])
++            pre_y = pre2[2];
++
++        // except jump to the top or bottom if we're close enough
++        if (md_vp.top < 32)
++            pre_y = 0;
++        else if (md_vp.top + 32 >= md_vp.height - md_vp.clientHeight)
++            pre_y = preview.scrollHeight;
++
++        preview.scrollTop = pre_y - preview.clientHeight / 2;
+     });
+ 
+     // Syncs scroll  preview -> editor
+-    preview.onscroll = function () {
++    // disabled since it should be possible to intentionally unsync
++    preview.onscroll_fgsfds = function () {
+         if (pScroll) {
+             pScroll = false;

scripts/deps-docker/easymde.patch

@@ -0,0 +1,88 @@
+diff -NarU2 easy-markdown-editor-2.14.0-orig/gulpfile.js easy-markdown-editor-2.14.0/gulpfile.js
+--- easy-markdown-editor-2.14.0-orig/gulpfile.js	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/gulpfile.js	2021-02-21 20:55:37.134701007 +0000
+@@ -25,5 +25,4 @@
+     './node_modules/codemirror/lib/codemirror.css',
+     './src/css/*.css',
+-    './node_modules/codemirror-spell-checker/src/css/spell-checker.css',
+ ];
+ 
+diff -NarU2 easy-markdown-editor-2.14.0-orig/package.json easy-markdown-editor-2.14.0/package.json
+--- easy-markdown-editor-2.14.0-orig/package.json	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/package.json	2021-02-21 20:55:47.761190082 +0000
+@@ -21,5 +21,4 @@
+     "dependencies": {
+         "codemirror": "^5.59.2",
+-        "codemirror-spell-checker": "1.1.2",
+         "marked": "^2.0.0"
+     },
+diff -NarU2 easy-markdown-editor-2.14.0-orig/src/js/easymde.js easy-markdown-editor-2.14.0/src/js/easymde.js
+--- easy-markdown-editor-2.14.0-orig/src/js/easymde.js	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/src/js/easymde.js	2021-02-21 20:57:09.143171536 +0000
+@@ -12,5 +12,4 @@
+ require('codemirror/mode/gfm/gfm.js');
+ require('codemirror/mode/xml/xml.js');
+-var CodeMirrorSpellChecker = require('codemirror-spell-checker');
+ var marked = require('marked/lib/marked');
+ 
+@@ -1762,9 +1761,4 @@
+         options.autosave.uniqueId = options.autosave.unique_id;
+ 
+-    // If overlay mode is specified and combine is not provided, default it to true
+-    if (options.overlayMode && options.overlayMode.combine === undefined) {
+-      options.overlayMode.combine = true;
+-    }
+-
+     // Update this options
+     this.options = options;
+@@ -2003,28 +1997,7 @@
+     var mode, backdrop;
+ 
+-    // CodeMirror overlay mode
+-    if (options.overlayMode) {
+-      CodeMirror.defineMode('overlay-mode', function(config) {
+-        return CodeMirror.overlayMode(CodeMirror.getMode(config, options.spellChecker !== false ? 'spell-checker' : 'gfm'), options.overlayMode.mode, options.overlayMode.combine);
+-      });
+-
+-      mode = 'overlay-mode';
+-      backdrop = options.parsingConfig;
+-      backdrop.gitHubSpice = false;
+-    } else {
+         mode = options.parsingConfig;
+         mode.name = 'gfm';
+         mode.gitHubSpice = false;
+-    }
+-    if (options.spellChecker !== false) {
+-        mode = 'spell-checker';
+-        backdrop = options.parsingConfig;
+-        backdrop.name = 'gfm';
+-        backdrop.gitHubSpice = false;
+-
+-        CodeMirrorSpellChecker({
+-            codeMirrorInstance: CodeMirror,
+-        });
+-    }
+ 
+     // eslint-disable-next-line no-unused-vars
+diff -NarU2 easy-markdown-editor-2.14.0-orig/types/easymde.d.ts easy-markdown-editor-2.14.0/types/easymde.d.ts
+--- easy-markdown-editor-2.14.0-orig/types/easymde.d.ts	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/types/easymde.d.ts	2021-02-21 20:57:42.492620979 +0000
+@@ -160,9 +160,4 @@
+     }
+ 
+-    interface OverlayModeOptions {
+-      mode: CodeMirror.Mode<any>
+-      combine?: boolean
+-    }
+-
+     interface Options {
+         autoDownloadFontAwesome?: boolean;
+@@ -214,7 +209,5 @@
+ 
+         promptTexts?: PromptTexts;
+-        syncSideBySidePreviewScroll?: boolean;
+-
+-        overlayMode?: OverlayModeOptions
++        syncSideBySidePreviewScroll?: boolean
+     }
+ }

scripts/deps-docker/markdown-it.patch

@@ -0,0 +1,10 @@
+diff -NarU1 markdown-it-10.0.0-orig/lib/common/entities.js markdown-it-10.0.0-edit/lib/common/entities.js
+--- markdown-it-10.0.0-orig/lib/common/entities.js	2019-09-10 21:39:58.000000000 +0000
++++ markdown-it-10.0.0-edit/lib/common/entities.js	2020-04-26 10:24:33.043023331 +0000
+@@ -5,2 +5,5 @@
+ /*eslint quotes:0*/
+-module.exports = require('entities/lib/maps/entities.json');
++//module.exports = require('entities/lib/maps/entities.json');
++module.exports = {
++    "amp": "&", "quot": "\"", "gt": ">", "lt": "<"
++}

scripts/deps-docker/marked-ln.patch

@@ -0,0 +1,298 @@
+diff --git a/src/Lexer.js b/src/Lexer.js
+adds linetracking to marked.js v1.0.0 +git;
+add data-ln="%d" to most tags, %d is the source markdown line
+--- a/src/Lexer.js
++++ b/src/Lexer.js
+@@ -49,4 +49,5 @@ function mangle(text) {
+ module.exports = class Lexer {
+   constructor(options) {
++    this.ln = 1;  // like most editors, start couting from 1
+     this.tokens = [];
+     this.tokens.links = Object.create(null);
+@@ -108,4 +109,15 @@ module.exports = class Lexer {
+   }
+ 
++  set_ln(token, ln = this.ln) {
++    // assigns ln (the current line numer) to the token,
++    // then bump this.ln by the number of newlines in the contents
++    //
++    // if ln is set, also assigns the line counter to a new value
++    // (usually a backup value from before a call into a subparser
++    //  which bumped the linecounter by a subset of the newlines)
++    token.ln = ln;
++    this.ln = ln + (token.raw.match(/\n/g) || []).length;
++  }
++
+   /**
+    * Lexing
+@@ -113,10 +125,15 @@ module.exports = class Lexer {
+   blockTokens(src, tokens = [], top = true) {
+     src = src.replace(/^ +$/gm, '');
+-    let token, i, l, lastToken;
++    let token, i, l, lastToken, ln;
+ 
+     while (src) {
++      // this.ln will be bumped by recursive calls into this func;
++      // reset the count and rely on the outermost token's raw only
++      ln = this.ln;
++
+       // newline
+       if (token = this.tokenizer.space(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token); // is \n if not type
+         if (token.type) {
+           tokens.push(token);
+@@ -128,4 +145,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.code(src, tokens)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         if (token.type) {
+           tokens.push(token);
+@@ -141,4 +159,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.fences(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -148,4 +167,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.heading(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -155,4 +175,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.nptable(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -162,4 +183,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.hr(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -170,4 +192,7 @@ module.exports = class Lexer {
+         src = src.substring(token.raw.length);
+         token.tokens = this.blockTokens(token.text, [], top);
++        // recursive call to blockTokens probably bumped this.ln,
++        // token.raw is more reliable so reset this.ln and use that
++        this.set_ln(token, ln);
+         tokens.push(token);
+         continue;
+@@ -180,5 +205,9 @@ module.exports = class Lexer {
+         for (i = 0; i < l; i++) {
+           token.items[i].tokens = this.blockTokens(token.items[i].text, [], false);
++          // list entries don't bump the linecounter, so let's
++          this.ln++;
+         }
++        // then reset like blockquote
++        this.set_ln(token, ln);
+         tokens.push(token);
+         continue;
+@@ -188,4 +217,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.html(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -195,4 +225,5 @@ module.exports = class Lexer {
+       if (top && (token = this.tokenizer.def(src))) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         if (!this.tokens.links[token.tag]) {
+           this.tokens.links[token.tag] = {
+@@ -207,4 +238,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.table(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -214,4 +246,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.lheading(src)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -221,4 +254,5 @@ module.exports = class Lexer {
+       if (top && (token = this.tokenizer.paragraph(src))) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         tokens.push(token);
+         continue;
+@@ -228,4 +262,5 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.text(src, tokens)) {
+         src = src.substring(token.raw.length);
++        this.set_ln(token);
+         if (token.type) {
+           tokens.push(token);
+@@ -263,4 +298,7 @@ module.exports = class Lexer {
+     for (i = 0; i < l; i++) {
+       token = tokens[i];
++      // this.ln is at EOF when inline() is invoked;
++      // all this affects <br> tags only so no biggie if it breaks
++      this.ln = token.ln || this.ln;
+       switch (token.type) {
+         case 'paragraph':
+@@ -386,4 +424,6 @@ module.exports = class Lexer {
+       if (token = this.tokenizer.br(src)) {
+         src = src.substring(token.raw.length);
++        // no need to reset (no more blockTokens anyways)
++        token.ln = this.ln++;
+         tokens.push(token);
+         continue;
+diff --git a/src/Parser.js b/src/Parser.js
+--- a/src/Parser.js
++++ b/src/Parser.js
+@@ -18,4 +18,5 @@ module.exports = class Parser {
+     this.textRenderer = new TextRenderer();
+     this.slugger = new Slugger();
++    this.ln = 0; // error indicator; should always be set >=1 from tokens
+   }
+ 
+@@ -55,4 +56,9 @@ module.exports = class Parser {
+     for (i = 0; i < l; i++) {
+       token = tokens[i];
++      // take line-numbers from tokens whenever possible
++      // and update the renderer's html attribute with the new value
++      this.ln = token.ln || this.ln;
++      this.renderer.tag_ln(this.ln);
++
+       switch (token.type) {
+         case 'space': {
+@@ -105,7 +111,10 @@ module.exports = class Parser {
+             }
+ 
+-            body += this.renderer.tablerow(cell);
++            // the +2 is to skip the table header
++            body += this.renderer.tag_ln(token.ln + j + 2).tablerow(cell);
+           }
+-          out += this.renderer.table(header, body);
++          // the html attribute is now at the end of the table,
++          // reset it before writing the <table> tag now
++          out += this.renderer.tag_ln(token.ln).table(header, body);
+           continue;
+         }
+@@ -148,8 +157,12 @@ module.exports = class Parser {
+ 
+             itemBody += this.parse(item.tokens, loose);
+-            body += this.renderer.listitem(itemBody, task, checked);
++            // similar to tables, writing contents before the <ul> tag
++            // so update the tag attribute as we go
++            // (assuming all list entries got tagged with a source-line, probably safe w)
++            body += this.renderer.tag_ln((item.tokens[0] || token).ln).listitem(itemBody, task, checked);
+           }
+ 
+-          out += this.renderer.list(body, ordered, start);
++          // then reset to the <ul>'s correct line number and write it
++          out += this.renderer.tag_ln(token.ln).list(body, ordered, start);
+           continue;
+         }
+@@ -160,5 +173,6 @@ module.exports = class Parser {
+         }
+         case 'paragraph': {
+-          out += this.renderer.paragraph(this.parseInline(token.tokens));
++          let t = this.parseInline(token.tokens);
++          out += this.renderer.tag_ln(token.ln).paragraph(t);
+           continue;
+         }
+@@ -199,4 +213,6 @@ module.exports = class Parser {
+     for (i = 0; i < l; i++) {
+       token = tokens[i];
++      // another thing that only affects <br/> and other inlines
++      this.ln = token.ln || this.ln;
+       switch (token.type) {
+         case 'escape': {
+@@ -229,5 +245,7 @@ module.exports = class Parser {
+         }
+         case 'br': {
+-          out += renderer.br();
++          // update the html attribute before writing each <br/>,
++          // don't care about the others
++          out += renderer.tag_ln(this.ln).br();
+           break;
+         }
+diff --git a/src/Renderer.js b/src/Renderer.js
+--- a/src/Renderer.js
++++ b/src/Renderer.js
+@@ -11,6 +11,12 @@ module.exports = class Renderer {
+   constructor(options) {
+     this.options = options || defaults;
++    this.ln = "";
+   }
+ 
++  tag_ln(n) {
++    this.ln = ' data-ln="' + n + '"';
++    return this;
++  };
++  
+   code(code, infostring, escaped) {
+     const lang = (infostring || '').match(/\S*/)[0];
+@@ -24,10 +30,10 @@ module.exports = class Renderer {
+ 
+     if (!lang) {
+-      return '<pre><code>'
++      return '<pre' + this.ln + '><code>'
+         + (escaped ? code : escape(code, true))
+         + '</code></pre>\n';
+     }
+ 
+-    return '<pre><code class="'
++    return '<pre' + this.ln + '><code class="'
+       + this.options.langPrefix
+       + escape(lang, true)
+@@ -38,5 +44,5 @@ module.exports = class Renderer {
+ 
+   blockquote(quote) {
+-    return '<blockquote>\n' + quote + '</blockquote>\n';
++    return '<blockquote' + this.ln + '>\n' + quote + '</blockquote>\n';
+   }
+ 
+@@ -49,4 +55,5 @@ module.exports = class Renderer {
+       return '<h'
+         + level
++        + this.ln
+         + ' id="'
+         + this.options.headerPrefix
+@@ -59,5 +66,5 @@ module.exports = class Renderer {
+     }
+     // ignore IDs
+-    return '<h' + level + '>' + text + '</h' + level + '>\n';
++    return '<h' + level + this.ln + '>' + text + '</h' + level + '>\n';
+   }
+ 
+@@ -73,5 +80,5 @@ module.exports = class Renderer {
+ 
+   listitem(text) {
+-    return '<li>' + text + '</li>\n';
++    return '<li' + this.ln + '>' + text + '</li>\n';
+   }
+ 
+@@ -85,5 +92,5 @@ module.exports = class Renderer {
+ 
+   paragraph(text) {
+-    return '<p>' + text + '</p>\n';
++    return '<p' + this.ln + '>' + text + '</p>\n';
+   }
+ 
+@@ -100,5 +107,5 @@ module.exports = class Renderer {
+ 
+   tablerow(content) {
+-    return '<tr>\n' + content + '</tr>\n';
++    return '<tr' + this.ln + '>\n' + content + '</tr>\n';
+   }
+ 
+@@ -125,5 +132,5 @@ module.exports = class Renderer {
+ 
+   br() {
+-    return this.options.xhtml ? '<br/>' : '<br>';
++    return this.options.xhtml ? '<br' + this.ln + '/>' : '<br' + this.ln + '>';
+   }
+ 
+@@ -151,5 +158,5 @@ module.exports = class Renderer {
+     }
+ 
+-    let out = '<img src="' + href + '" alt="' + text + '"';
++    let out = '<img' + this.ln + ' src="' + href + '" alt="' + text + '"';
+     if (title) {
+       out += ' title="' + title + '"';

scripts/deps-docker/marked.patch

@@ -0,0 +1,340 @@
+diff --git a/src/Lexer.js b/src/Lexer.js
+--- a/src/Lexer.js
++++ b/src/Lexer.js
+@@ -5,5 +5,5 @@ const { block, inline } = require('./rules.js');
+ /**
+  * smartypants text replacement
+- */
++ *
+ function smartypants(text) {
+   return text
+@@ -26,5 +26,5 @@ function smartypants(text) {
+ /**
+  * mangle email addresses
+- */
++ *
+ function mangle(text) {
+   let out = '',
+@@ -439,5 +439,5 @@ module.exports = class Lexer {
+ 
+       // autolink
+-      if (token = this.tokenizer.autolink(src, mangle)) {
++      if (token = this.tokenizer.autolink(src)) {
+         src = src.substring(token.raw.length);
+         tokens.push(token);
+@@ -446,5 +446,5 @@ module.exports = class Lexer {
+ 
+       // url (gfm)
+-      if (!inLink && (token = this.tokenizer.url(src, mangle))) {
++      if (!inLink && (token = this.tokenizer.url(src))) {
+         src = src.substring(token.raw.length);
+         tokens.push(token);
+@@ -453,5 +453,5 @@ module.exports = class Lexer {
+ 
+       // text
+-      if (token = this.tokenizer.inlineText(src, inRawBlock, smartypants)) {
++      if (token = this.tokenizer.inlineText(src, inRawBlock)) {
+         src = src.substring(token.raw.length);
+         tokens.push(token);
+diff --git a/src/Renderer.js b/src/Renderer.js
+--- a/src/Renderer.js
++++ b/src/Renderer.js
+@@ -140,5 +140,5 @@ module.exports = class Renderer {
+ 
+   link(href, title, text) {
+-    href = cleanUrl(this.options.sanitize, this.options.baseUrl, href);
++    href = cleanUrl(this.options.baseUrl, href);
+     if (href === null) {
+       return text;
+@@ -153,5 +153,5 @@ module.exports = class Renderer {
+ 
+   image(href, title, text) {
+-    href = cleanUrl(this.options.sanitize, this.options.baseUrl, href);
++    href = cleanUrl(this.options.baseUrl, href);
+     if (href === null) {
+       return text;
+diff --git a/src/Tokenizer.js b/src/Tokenizer.js
+--- a/src/Tokenizer.js
++++ b/src/Tokenizer.js
+@@ -287,11 +287,8 @@ module.exports = class Tokenizer {
+     if (cap) {
+       return {
+-        type: this.options.sanitize
+-          ? 'paragraph'
+-          : 'html',
++        type: 'html',
+         raw: cap[0],
+-        pre: !this.options.sanitizer
+-          && (cap[1] === 'pre' || cap[1] === 'script' || cap[1] === 'style'),
+-        text: this.options.sanitize ? (this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0])) : cap[0]
++        pre: cap[1] === 'pre' || cap[1] === 'script' || cap[1] === 'style',
++        text: cap[0]
+       };
+     }
+@@ -421,15 +418,9 @@ module.exports = class Tokenizer {
+ 
+       return {
+-        type: this.options.sanitize
+-          ? 'text'
+-          : 'html',
++        type: 'html',
+         raw: cap[0],
+         inLink,
+         inRawBlock,
+-        text: this.options.sanitize
+-          ? (this.options.sanitizer
+-            ? this.options.sanitizer(cap[0])
+-            : escape(cap[0]))
+-          : cap[0]
++        text: cap[0]
+       };
+     }
+@@ -550,10 +541,10 @@ module.exports = class Tokenizer {
+   }
+ 
+-  autolink(src, mangle) {
++  autolink(src) {
+     const cap = this.rules.inline.autolink.exec(src);
+     if (cap) {
+       let text, href;
+       if (cap[2] === '@') {
+-        text = escape(this.options.mangle ? mangle(cap[1]) : cap[1]);
++        text = escape(cap[1]);
+         href = 'mailto:' + text;
+       } else {
+@@ -578,10 +569,10 @@ module.exports = class Tokenizer {
+   }
+ 
+-  url(src, mangle) {
++  url(src) {
+     let cap;
+     if (cap = this.rules.inline.url.exec(src)) {
+       let text, href;
+       if (cap[2] === '@') {
+-        text = escape(this.options.mangle ? mangle(cap[0]) : cap[0]);
++        text = escape(cap[0]);
+         href = 'mailto:' + text;
+       } else {
+@@ -615,12 +606,12 @@ module.exports = class Tokenizer {
+   }
+ 
+-  inlineText(src, inRawBlock, smartypants) {
++  inlineText(src, inRawBlock) {
+     const cap = this.rules.inline.text.exec(src);
+     if (cap) {
+       let text;
+       if (inRawBlock) {
+-        text = this.options.sanitize ? (this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0])) : cap[0];
++        text = cap[0];
+       } else {
+-        text = escape(this.options.smartypants ? smartypants(cap[0]) : cap[0]);
++        text = escape(cap[0]);
+       }
+       return {
+diff --git a/src/defaults.js b/src/defaults.js
+--- a/src/defaults.js
++++ b/src/defaults.js
+@@ -8,12 +8,8 @@ function getDefaults() {
+     highlight: null,
+     langPrefix: 'language-',
+-    mangle: true,
+     pedantic: false,
+     renderer: null,
+-    sanitize: false,
+-    sanitizer: null,
+     silent: false,
+     smartLists: false,
+-    smartypants: false,
+     tokenizer: null,
+     walkTokens: null,
+diff --git a/src/helpers.js b/src/helpers.js
+--- a/src/helpers.js
++++ b/src/helpers.js
+@@ -64,18 +64,5 @@ function edit(regex, opt) {
+ const nonWordAndColonTest = /[^\w:]/g;
+ const originIndependentUrl = /^$|^[a-z][a-z0-9+.-]*:|^[?#]/i;
+-function cleanUrl(sanitize, base, href) {
+-  if (sanitize) {
+-    let prot;
+-    try {
+-      prot = decodeURIComponent(unescape(href))
+-        .replace(nonWordAndColonTest, '')
+-        .toLowerCase();
+-    } catch (e) {
+-      return null;
+-    }
+-    if (prot.indexOf('javascript:') === 0 || prot.indexOf('vbscript:') === 0 || prot.indexOf('data:') === 0) {
+-      return null;
+-    }
+-  }
++function cleanUrl(base, href) {
+   if (base && !originIndependentUrl.test(href)) {
+     href = resolveUrl(base, href);
+@@ -223,10 +210,4 @@ function findClosingBracket(str, b) {
+ }
+ 
+-function checkSanitizeDeprecation(opt) {
+-  if (opt && opt.sanitize && !opt.silent) {
+-    console.warn('marked(): sanitize and sanitizer parameters are deprecated since version 0.7.0, should not be used and will be removed in the future. Read more here: https://marked.js.org/#/USING_ADVANCED.md#options');
+-  }
+-}
+-
+ module.exports = {
+   escape,
+@@ -239,5 +220,4 @@ module.exports = {
+   splitCells,
+   rtrim,
+-  findClosingBracket,
+-  checkSanitizeDeprecation
++  findClosingBracket
+ };
+diff --git a/src/marked.js b/src/marked.js
+--- a/src/marked.js
++++ b/src/marked.js
+@@ -7,5 +7,4 @@ const Slugger = require('./Slugger.js');
+ const {
+   merge,
+-  checkSanitizeDeprecation,
+   escape
+ } = require('./helpers.js');
+@@ -35,5 +34,4 @@ function marked(src, opt, callback) {
+ 
+   opt = merge({}, marked.defaults, opt || {});
+-  checkSanitizeDeprecation(opt);
+ 
+   if (callback) {
+@@ -108,5 +106,5 @@ function marked(src, opt, callback) {
+     return Parser.parse(tokens, opt);
+   } catch (e) {
+-    e.message += '\nPlease report this to https://github.com/markedjs/marked.';
++    e.message += '\nmake issue @ https://github.com/9001/copyparty';
+     if (opt.silent) {
+       return '<p>An error occurred:</p><pre>'
+diff --git a/test/bench.js b/test/bench.js
+--- a/test/bench.js
++++ b/test/bench.js
+@@ -33,5 +33,4 @@ async function runBench(options) {
+     breaks: false,
+     pedantic: false,
+-    sanitize: false,
+     smartLists: false
+   });
+@@ -45,5 +44,4 @@ async function runBench(options) {
+     breaks: false,
+     pedantic: false,
+-    sanitize: false,
+     smartLists: false
+   });
+@@ -58,5 +56,4 @@ async function runBench(options) {
+     breaks: false,
+     pedantic: false,
+-    sanitize: false,
+     smartLists: false
+   });
+@@ -70,5 +67,4 @@ async function runBench(options) {
+     breaks: false,
+     pedantic: false,
+-    sanitize: false,
+     smartLists: false
+   });
+@@ -83,5 +79,4 @@ async function runBench(options) {
+     breaks: false,
+     pedantic: true,
+-    sanitize: false,
+     smartLists: false
+   });
+@@ -95,5 +90,4 @@ async function runBench(options) {
+     breaks: false,
+     pedantic: true,
+-    sanitize: false,
+     smartLists: false
+   });
+diff --git a/test/specs/run-spec.js b/test/specs/run-spec.js
+--- a/test/specs/run-spec.js
++++ b/test/specs/run-spec.js
+@@ -22,8 +22,4 @@ function runSpecs(title, dir, showCompletionTable, options) {
+           }
+ 
+-          if (spec.options.sanitizer) {
+-            // eslint-disable-next-line no-eval
+-            spec.options.sanitizer = eval(spec.options.sanitizer);
+-          }
+ 
+           (spec.only ? fit : (spec.skip ? xit : it))('should ' + passFail + example, async() => {
+@@ -53,3 +49,2 @@ runSpecs('Original', './original', false, { gfm: false, pedantic: true });
+ runSpecs('New', './new');
+ runSpecs('ReDOS', './redos');
+-runSpecs('Security', './security', false, { silent: true }); // silent - do not show deprecation warning
+diff --git a/test/unit/Lexer-spec.js b/test/unit/Lexer-spec.js
+--- a/test/unit/Lexer-spec.js
++++ b/test/unit/Lexer-spec.js
+@@ -465,5 +465,5 @@ a | b
+     });
+ 
+-    it('sanitize', () => {
++    /*it('sanitize', () => {
+       expectTokens({
+         md: '<div>html</div>',
+@@ -483,5 +483,5 @@ a | b
+         ]
+       });
+-    });
++    });*/
+   });
+ 
+@@ -587,5 +587,5 @@ a | b
+       });
+ 
+-      it('html sanitize', () => {
++      /*it('html sanitize', () => {
+         expectInlineTokens({
+           md: '<div>html</div>',
+@@ -597,5 +597,5 @@ a | b
+           ]
+         });
+-      });
++      });*/
+ 
+       it('link', () => {
+@@ -909,5 +909,5 @@ a | b
+         });
+ 
+-        it('autolink mangle email', () => {
++        /*it('autolink mangle email', () => {
+           expectInlineTokens({
+             md: '<test@example.com>',
+@@ -929,5 +929,5 @@ a | b
+             ]
+           });
+-        });
++        });*/
+ 
+         it('url', () => {
+@@ -966,5 +966,5 @@ a | b
+         });
+ 
+-        it('url mangle email', () => {
++        /*it('url mangle email', () => {
+           expectInlineTokens({
+             md: 'test@example.com',
+@@ -986,5 +986,5 @@ a | b
+             ]
+           });
+-        });
++        });*/
+       });
+ 
+@@ -1002,5 +1002,5 @@ a | b
+       });
+ 
+-      describe('smartypants', () => {
++      /*describe('smartypants', () => {
+         it('single quotes', () => {
+           expectInlineTokens({
+@@ -1072,5 +1072,5 @@ a | b
+           });
+         });
+-      });
++      });*/
+     });
+   });

scripts/deps-docker/mini-fa.css

@@ -0,0 +1,53 @@
+
+/*
+that was the original copyright ^
+now here's a tiny subset of fontawesome
+*/
+
+@font-face {
+font-family: 'fa';
+font-style: normal;
+font-weight: 400;
+font-display: block;
+src: url("/.cpr/deps/mini-fa.woff") format("woff");
+}
+
+.fa,
+.fas,
+.far,
+.fal,
+.fad,
+.fab {
+-moz-osx-font-smoothing: grayscale;
+-webkit-font-smoothing: antialiased;
+display: inline-block;
+font-style: normal;
+font-variant: normal;
+text-rendering: auto;
+line-height: 1;
+font-family: 'fa';
+font-weight: 400;
+}
+
+:add
+arrows-alt
+bold
+code
+columns
+eraser
+eye
+heading
+image
+italic
+lightbulb
+link
+list-ol
+list-ul
+minus
+question-circle
+quote-left
+redo
+save
+strikethrough
+table
+undo

scripts/deps-docker/mini-fa.sh

@@ -0,0 +1,31 @@
+#!/bin/ash
+set -e
+
+orig_css="$(find /z/fontawesome-fre* -name fontawesome.css | head -n 1)"
+orig_woff="$(find /z/fontawesome-fre* -name fa-solid-900.woff | head -n 1)"
+
+# first grab the copyright meme
+awk '1; / *\*\// {exit}' <"$orig_css" >/z/dist/mini-fa.css
+
+# then add the static part of our css template
+awk '/^:add/ {exit} 1' </z/mini-fa.css >>/z/dist/mini-fa.css
+
+# then take the list of icons to include
+awk 'o; /^:add/ {o=1}' </z/mini-fa.css |
+while IFS= read -r g; do
+    # and grab them from the upstream css
+    awk 'o{gsub(/[ ;]+/,"");print;exit} /^\.fa-'$g':before/ {o=1;printf "%s",$0}' <"$orig_css"
+done >>/z/dist/mini-fa.css
+
+# expecting this input btw:
+# .fa-python:before {
+#  content: "\f3e2"; }
+
+# get the codepoints (should produce lines like "f3e2")
+awk '/:before .content:"\\/ {sub(/[^"]+"./,""); sub(/".*/,""); print}' </z/dist/mini-fa.css >/z/icon.list
+
+# and finally create a woff with just our icons
+pyftsubset "$orig_woff" --unicodes-file=/z/icon.list --no-ignore-missing-unicodes --flavor=woff --with-zopfli --output-file=/z/dist/no-pk/mini-fa.woff --verbose
+
+# scp is easier, just want basic latin
+pyftsubset /z/scp.woff2 --unicodes="20-7e,ab,b7,bb,2022" --no-ignore-missing-unicodes --flavor=woff2 --output-file=/z/dist/no-pk/scp.woff2 --verbose

scripts/deps-docker/showdown.patch

@@ -0,0 +1,214 @@
+diff -NarU1 showdown-orig/Gruntfile.js showdown-mod/Gruntfile.js
+--- showdown-orig/Gruntfile.js	2020-04-23 06:22:01.486676149 +0000
++++ showdown-mod/Gruntfile.js	2020-04-23 08:03:56.700219788 +0000
+@@ -27,3 +27,2 @@
+           'src/subParsers/*.js',
+-          'src/subParsers/makeMarkdown/*.js',
+           'src/loader.js'
+diff -NarU1 showdown-orig/src/converter.js showdown-mod/src/converter.js
+--- showdown-orig/src/converter.js	2020-04-23 06:22:01.496676150 +0000
++++ showdown-mod/src/converter.js	2020-04-23 08:20:11.056920123 +0000
+@@ -84,5 +84,5 @@
+ 
+-    if (options.extensions) {
++    /*if (options.extensions) {
+       showdown.helper.forEach(options.extensions, _parseExtension);
+-    }
++    }*/
+   }
+@@ -95,3 +95,3 @@
+    */
+-  function _parseExtension (ext, name) {
++  /*function _parseExtension (ext, name) {
+ 
+@@ -159,3 +159,3 @@
+    */
+-  function legacyExtensionLoading (ext, name) {
++  /*function legacyExtensionLoading (ext, name) {
+     if (typeof ext === 'function') {
+@@ -351,3 +351,3 @@
+    */
+-  this.makeMarkdown = this.makeMd = function (src, HTMLParser) {
++  /*this.makeMarkdown = this.makeMd = function (src, HTMLParser) {
+ 
+@@ -482,3 +482,3 @@
+    */
+-  this.addExtension = function (extension, name) {
++  /*this.addExtension = function (extension, name) {
+     name = name || null;
+@@ -491,3 +491,3 @@
+    */
+-  this.useExtension = function (extensionName) {
++  /*this.useExtension = function (extensionName) {
+     _parseExtension(extensionName);
+@@ -526,3 +526,3 @@
+    */
+-  this.removeExtension = function (extension) {
++  /*this.removeExtension = function (extension) {
+     if (!showdown.helper.isArray(extension)) {
+@@ -549,3 +549,3 @@
+    */
+-  this.getAllExtensions = function () {
++  /*this.getAllExtensions = function () {
+     return {
+diff -NarU1 showdown-orig/src/options.js showdown-mod/src/options.js
+--- showdown-orig/src/options.js	2020-04-23 06:22:01.496676150 +0000
++++ showdown-mod/src/options.js	2020-04-23 08:24:29.176929018 +0000
+@@ -118,3 +118,3 @@
+     },
+-    ghMentions: {
++    /*ghMentions: {
+       defaultValue: false,
+@@ -127,3 +127,3 @@
+       type: 'string'
+-    },
++    },*/
+     encodeEmails: {
+diff -NarU1 showdown-orig/src/showdown.js showdown-mod/src/showdown.js
+--- showdown-orig/src/showdown.js	2020-04-23 06:22:01.496676150 +0000
++++ showdown-mod/src/showdown.js	2020-04-23 08:25:01.976930148 +0000
+@@ -7,3 +7,2 @@
+     parsers = {},
+-    extensions = {},
+     globalOptions = getDefaultOpts(true),
+@@ -25,5 +24,4 @@
+         ghCompatibleHeaderId:                 true,
+-        ghMentions:                           true,
++        //ghMentions:                           true,
+         backslashEscapesHTMLTags:             true,
+-        emoji:                                true,
+         splitAdjacentBlockquotes:             true
+@@ -48,3 +46,3 @@
+         requireSpaceBeforeHeadingText:        true,
+-        ghMentions:                           false,
++        //ghMentions:                           false,
+         encodeEmails:                         true
+@@ -65,3 +63,2 @@
+  */
+-showdown.extensions = {};
+ 
+@@ -193,3 +190,3 @@
+  */
+-showdown.extension = function (name, ext) {
++/*showdown.extension = function (name, ext) {
+   'use strict';
+@@ -235,3 +232,3 @@
+  */
+-showdown.getAllExtensions = function () {
++/*showdown.getAllExtensions = function () {
+   'use strict';
+@@ -244,3 +241,3 @@
+  */
+-showdown.removeExtension = function (name) {
++/*showdown.removeExtension = function (name) {
+   'use strict';
+@@ -252,3 +249,3 @@
+  */
+-showdown.resetExtensions = function () {
++/*showdown.resetExtensions = function () {
+   'use strict';
+@@ -263,3 +260,3 @@
+  */
+-function validate (extension, name) {
++/*function validate (extension, name) {
+   'use strict';
+@@ -370,3 +367,3 @@
+  */
+-showdown.validateExtension = function (ext) {
++/*showdown.validateExtension = function (ext) {
+   'use strict';
+@@ -380 +377,2 @@
+ };
++*/
+diff -NarU1 showdown-orig/src/subParsers/anchors.js showdown-mod/src/subParsers/anchors.js
+--- showdown-orig/src/subParsers/anchors.js	2020-04-23 06:22:01.496676150 +0000
++++ showdown-mod/src/subParsers/anchors.js	2020-04-23 08:25:26.880264347 +0000
+@@ -76,3 +76,3 @@
+   // Lastly handle GithubMentions if option is enabled
+-  if (options.ghMentions) {
++  /*if (options.ghMentions) {
+     text = text.replace(/(^|\s)(\\)?(@([a-z\d]+(?:[a-z\d.-]+?[a-z\d]+)*))/gmi, function (wm, st, escape, mentions, username) {
+@@ -93,3 +93,3 @@
+     });
+-  }
++  }*/
+ 
+diff -NarU1 showdown-orig/src/subParsers/spanGamut.js showdown-mod/src/subParsers/spanGamut.js
+--- showdown-orig/src/subParsers/spanGamut.js	2020-04-23 06:22:01.496676150 +0000
++++ showdown-mod/src/subParsers/spanGamut.js	2020-04-23 08:07:50.460227880 +0000
+@@ -22,3 +22,2 @@
+   text = showdown.subParser('simplifiedAutoLinks')(text, options, globals);
+-  text = showdown.subParser('emoji')(text, options, globals);
+   text = showdown.subParser('underline')(text, options, globals);
+@@ -26,3 +25,2 @@
+   text = showdown.subParser('strikethrough')(text, options, globals);
+-  text = showdown.subParser('ellipsis')(text, options, globals);
+ 
+diff -NarU1 showdown-orig/test/node/showdown.Converter.js showdown-mod/test/node/showdown.Converter.js
+--- showdown-orig/test/node/showdown.Converter.js	2020-04-23 06:22:01.520009484 +0000
++++ showdown-mod/test/node/showdown.Converter.js	2020-04-23 08:14:58.086909318 +0000
+@@ -29,3 +29,3 @@
+ 
+-  describe('Converter.options extensions', function () {
++  /*describe('Converter.options extensions', function () {
+     var runCount;
+@@ -48,3 +48,3 @@
+     });
+-  });
++  });*/
+ 
+@@ -115,3 +115,3 @@
+ 
+-  describe('extension methods', function () {
++  /*describe('extension methods', function () {
+     var extObjMock = {
+@@ -145,3 +145,3 @@
+     });
+-  });
++  });*/
+ 
+diff -NarU1 showdown-orig/test/node/showdown.js showdown-mod/test/node/showdown.js
+--- showdown-orig/test/node/showdown.js	2020-04-23 06:22:01.523342816 +0000
++++ showdown-mod/test/node/showdown.js	2020-04-23 08:14:31.733575073 +0000
+@@ -25,3 +25,3 @@
+ 
+-describe('showdown.extension()', function () {
++/*describe('showdown.extension()', function () {
+   'use strict';
+@@ -110,3 +110,3 @@
+   });
+-});
++});*/
+ 
+diff -NarU1 showdown-orig/test/node/testsuite.features.js showdown-mod/test/node/testsuite.features.js
+--- showdown-orig/test/node/testsuite.features.js	2020-04-23 06:22:01.523342816 +0000
++++ showdown-mod/test/node/testsuite.features.js	2020-04-23 08:25:48.880265106 +0000
+@@ -13,3 +13,2 @@
+     rawPrefixHeaderIdSuite = bootstrap.getTestSuite('test/features/rawPrefixHeaderId/'),
+-    emojisSuite = bootstrap.getTestSuite('test/features/emojis/'),
+     underlineSuite = bootstrap.getTestSuite('test/features/underline/'),
+@@ -69,4 +68,4 @@
+         converter = new showdown.Converter({ghCompatibleHeaderId: true});
+-      } else if (testsuite[i].name === 'ghMentions') {
+-        converter = new showdown.Converter({ghMentions: true});
++      //} else if (testsuite[i].name === 'ghMentions') {
++      //  converter = new showdown.Converter({ghMentions: true});
+       } else if (testsuite[i].name === 'disable-email-encoding') {
+@@ -185,17 +184,2 @@
+       it(suite[i].name.replace(/-/g, ' '), assertion(suite[i], converter));
+-    }
+-  });
+-
+-  /** test emojis support **/
+-  describe('emojis support', function () {
+-    var converter,
+-        suite = emojisSuite;
+-    for (var i = 0; i < suite.length; ++i) {
+-      if (suite[i].name === 'simplifiedautolinks') {
+-        converter = new showdown.Converter({emoji: true, simplifiedAutoLink: true});
+-      } else {
+-        converter = new showdown.Converter({emoji: true});
+-      }
+-
+-      it(suite[i].name.replace(/-/g, ' '), assertion(suite[i], converter));
+     }

scripts/deps-docker/zopfli.makefile

@@ -1,6 +1,10 @@
 all: $(addsuffix .gz, $(wildcard *.*))
 
 %.gz: %
+	#brotli -q 11 $<
 	pigz -11 -J 34 -I 573 $<
 
 # pigz -11 -J 34 -I 100 -F < $< > $@.first
+
+# disabling brotli after all since the gain is meh
+#   and it bloats sfx and wheels by like 70%

scripts/fusefuzz.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import time
+
+"""
+td=/dev/shm/; [ -e $td ] || td=$HOME; mkdir -p $td/fusefuzz/{r,v}
+PYTHONPATH=.. python3 -m copyparty -v $td/fusefuzz/r::r -i 127.0.0.1
+../bin/copyparty-fuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
+(d="$PWD"; cd $td/fusefuzz && "$d"/fusefuzz.py)
+"""
+
+
+def chk(fsz, rsz, ofs0, shift, ofs, rf, vf):
+    if ofs != rf.tell():
+        rf.seek(ofs)
+        vf.seek(ofs)
+
+    rb = rf.read(rsz)
+    vb = vf.read(rsz)
+
+    print(f"fsz {fsz} rsz {rsz} ofs {ofs0} shift {shift} ofs {ofs} = {len(rb)}")
+
+    if rb != vb:
+        for n, buf in enumerate([rb, vb]):
+            with open("buf." + str(n), "wb") as f:
+                f.write(buf)
+
+        raise Exception(f"{len(rb)} != {len(vb)}")
+
+    return rb, vb
+
+
+def main():
+    v = "v"
+    for n in range(5):
+        with open(f"r/{n}", "wb") as f:
+            f.write(b"h" * n)
+
+    rand = os.urandom(7919)  # prime
+    for fsz in range(1024 * 1024 * 2 - 3, 1024 * 1024 * 2 + 3):
+        with open("r/f", "wb", fsz) as f:
+            f.write((rand * int(fsz / len(rand) + 1))[:fsz])
+
+        for rsz in range(64 * 1024 - 2, 64 * 1024 + 2):
+            ofslist = [0, 1, 2]
+            for n in range(3):
+                ofslist.append(fsz - n)
+                ofslist.append(fsz - (rsz * 1 + n))
+                ofslist.append(fsz - (rsz * 2 + n))
+
+            for ofs0 in ofslist:
+                for shift in range(-3, 3):
+                    print(f"fsz {fsz} rsz {rsz} ofs {ofs0} shift {shift}")
+                    ofs = ofs0
+                    if ofs < 0 or ofs >= fsz:
+                        continue
+
+                    for n in range(1, 3):
+                        with open(f"{v}/{n}", "rb") as f:
+                            f.read()
+
+                    prev_ofs = -99
+                    with open("r/f", "rb", rsz) as rf:
+                        with open(f"{v}/f", "rb", rsz) as vf:
+                            while True:
+                                ofs += shift
+                                if ofs < 0 or ofs > fsz or ofs == prev_ofs:
+                                    break
+
+                                prev_ofs = ofs
+
+                                rb, vb = chk(fsz, rsz, ofs0, shift, ofs, rf, vf)
+
+                                if not rb:
+                                    break
+
+                                ofs += len(rb)
+
+                    for n in range(1, 3):
+                        with open(f"{v}/{n}", "rb") as f:
+                            f.read()
+
+                    with open("r/f", "rb", rsz) as rf:
+                        with open(f"{v}/f", "rb", rsz) as vf:
+                            for n in range(2):
+                                ofs += shift
+                                if ofs < 0 or ofs > fsz:
+                                    break
+
+                                rb, vb = chk(fsz, rsz, ofs0, shift, ofs, rf, vf)
+
+                                ofs -= rsz
+
+        # bumping fsz, sleep away the dentry cache in cppf
+        time.sleep(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/make-pypi-release.sh

@@ -3,9 +3,16 @@ set -e
 echo
 
 # osx support
-sed=$( which gsed  2>/dev/null || which sed)
-find=$(which gfind 2>/dev/null || which find)
-sort=$(which gsort 2>/dev/null || which sort)
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
+	sed()  { gsed  "$@"; }
+	find() { gfind "$@"; }
+	sort() { gsort "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
+}
 
 which md5sum 2>/dev/null >/dev/null &&
 	md5sum=md5sum ||
@@ -79,6 +86,8 @@ function have() {
 	python -c "import $1; $1; $1.__version__"
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 . buildenv/bin/activate
 have setuptools
 have wheel

scripts/make-sfx.sh

@@ -0,0 +1,241 @@
+#!/bin/bash
+set -e
+echo
+
+
+# optional args:
+#
+# `clean` uses files from git (everything except web/deps),
+#   so local changes won't affect the produced sfx
+#
+# `re` does a repack of an sfx which you already executed once
+#   (grabs files from the sfx-created tempdir), overrides `clean`
+#
+# `no-ogv` saves ~500k by removing the opus/vorbis audio codecs
+#   (only affects apple devices; everything else has native support)
+#
+# `no-cm` saves ~90k by removing easymde/codemirror
+#   (the fancy markdown editor)
+
+
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
+	sed()  { gsed  "$@"; }
+	find() { gfind "$@"; }
+	sort() { gsort "$@"; }
+	unexpand() { gunexpand "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
+
+	[ -e /opt/local/bin/bzip2 ] &&
+		bzip2() { /opt/local/bin/bzip2 "$@"; }
+}
+pybin=$(command -v python3 || command -v python) || {
+	echo need python
+	exit 1
+}
+
+[ -e copyparty/__main__.py ] || cd ..
+[ -e copyparty/__main__.py ] ||
+{
+	echo "run me from within the project root folder"
+	echo
+	exit 1
+}
+
+do_sh=1
+do_py=1
+while [ ! -z "$1" ]; do
+	[ "$1" = clean  ] && clean=1  && shift && continue
+	[ "$1" = re     ] && repack=1 && shift && continue
+	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
+	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
+	[ "$1" = no-sh  ] && do_sh=   && shift && continue
+	[ "$1" = no-py  ] && do_py=   && shift && continue
+	break
+done
+
+tmv() {
+	touch -r "$1" t
+	mv t "$1"
+}
+
+rm -rf sfx/*
+mkdir -p sfx build
+cd sfx
+
+[ $repack ] && {
+	old="$(
+		printf '%s\n' "$TMPDIR" /tmp |
+		awk '/./ {print; exit}'
+	)/pe-copyparty"
+
+	echo "repack of files in $old"
+	cp -pR "$old/"*{dep-j2,copyparty} .
+}
+
+[ $repack ] || {
+	echo collecting jinja2
+	f="../build/Jinja2-2.11.3.tar.gz"
+	[ -e "$f" ] ||
+		(url=https://files.pythonhosted.org/packages/4f/e7/65300e6b32e69768ded990494809106f87da1d436418d5f1367ed3966fd7/Jinja2-2.11.3.tar.gz;
+		wget -O$f "$url" || curl -L "$url" >$f)
+
+	tar -zxf $f
+	mv Jinja2-*/src/jinja2 .
+	rm -rf Jinja2-*
+	
+	echo collecting markupsafe
+	f="../build/MarkupSafe-1.1.1.tar.gz"
+	[ -e "$f" ] ||
+		(url=https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz;
+		wget -O$f "$url" || curl -L "$url" >$f)
+
+	tar -zxf $f
+	mv MarkupSafe-*/src/markupsafe .
+	rm -rf MarkupSafe-* markupsafe/_speedups.c
+
+	mkdir dep-j2/
+	mv {markupsafe,jinja2} dep-j2/
+
+	# msys2 tar is bad, make the best of it
+	echo collecting source
+	[ $clean ] && {
+		(cd .. && git archive master >tar) && tar -xf ../tar copyparty
+		(cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar
+	}
+	[ $clean ] || {
+		(cd .. && tar -cf tar copyparty) && tar -xf ../tar
+	}
+	rm -f ../tar
+}
+
+ver=
+git describe --tags >/dev/null 2>/dev/null && {
+	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//; s/-g?/./g')";
+	t_ver=
+
+	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
+		# short format (exact version number)
+		t_ver="$(printf '%s\n' "$ver" | sed -r 's/\./, /g')";
+	}
+
+	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+-[0-9]+-g[0-9a-f]+$' && {
+		# long format (unreleased commit)
+		t_ver="$(printf '%s\n' "$ver" | sed -r 's/\./, /g; s/(.*) (.*)/\1 "\2"/')"
+	}
+
+	[ -z "$t_ver" ] && {
+		printf 'unexpected git version format: [%s]\n' "$git_ver"
+		exit 1
+	}
+
+	dt="$(git log -1 --format=%cd --date=short | sed -E 's/-0?/, /g')"
+	printf 'git %3s: \033[36m%s\033[0m\n' ver "$ver" dt "$dt"
+	sed -ri '
+		s/^(VERSION =)(.*)/#\1\2\n\1 ('"$t_ver"')/;
+		s/^(S_VERSION =)(.*)/#\1\2\n\1 "'"$ver"'"/;
+		s/^(BUILD_DT =)(.*)/#\1\2\n\1 ('"$dt"')/;
+	' copyparty/__version__.py
+}
+
+[ -z "$ver" ] && 
+	ver="$(awk '/^VERSION *= \(/ {
+		gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < copyparty/__version__.py)"
+
+ts=$(date -u +%s)
+hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
+
+mkdir -p ../dist
+sfx_out=../dist/copyparty-sfx
+
+echo cleanup
+find .. -name '*.pyc' -delete
+find .. -name __pycache__ -delete
+
+# especially prevent osx from leaking your lan ip (wtf apple)
+find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
+find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
+
+echo use smol web deps
+rm -f copyparty/web/deps/*.full.*
+
+# it's fine dw
+grep -lE '\.full\.(js|css)' copyparty/web/* |
+while IFS= read -r x; do
+	sed -r 's/\.full\.(js|css)/.\1/g' <"$x" >t
+	tmv "$x"
+done
+
+[ $no_ogv ] &&
+	rm -rf copyparty/web/deps/{dynamicaudio,ogv}*
+
+[ $no_cm ] && {
+	rm -rf copyparty/web/mde.* copyparty/web/deps/easymde*
+	echo h > copyparty/web/mde.html
+	f=copyparty/web/md.html
+	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
+}
+
+[ $repack ] ||
+find | grep -E '\.py$' |
+  grep -vE '__version__' |
+  tr '\n' '\0' |
+  xargs -0 $pybin ../scripts/uncomment.py
+
+f=dep-j2/jinja2/constants.py
+awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
+tmv "$f"
+
+# up2k goes from 28k to 22k laff
+echo entabbening
+find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
+	unexpand -t 4 --first-only <"$f" >t
+	tmv "$f"
+done
+
+echo creating tar
+args=(--owner=1000 --group=1000)
+[ "$OSTYPE" = msys ] &&
+	args=()
+
+tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
+
+echo compressing tar
+# detect best level; bzip2 -7 is usually better than -9
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2; }
+[ $do_sh ] && { for n in {2..9}; do cp tar t.$n;  xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
+rm t.* || true
+exts=()
+
+
+[ $do_sh ] && {
+exts+=(sh)
+echo creating unix sfx
+(
+	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
+	grep -E '^sfx_eof$' -B 9001;
+	cat tar.xz
+) >$sfx_out.sh
+}
+
+
+[ $do_py ] && {
+exts+=(py)
+echo creating generic sfx
+$pybin ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
+mv sfx.out $sfx_out.py
+chmod 755 $sfx_out.*
+}
+
+
+printf "done:\n"
+for ext in ${exts[@]}; do
+	printf "  %s\n" "$(realpath $sfx_out)."$ext
+done
+
+# apk add bash python3 tar xz bzip2
+# while true; do ./make-sfx.sh; for f in ..//dist/copyparty-sfx.{sh,py}; do mv $f $f.$(wc -c <$f | awk '{print$1}'); done; done

scripts/make-tgz-release.sh

@@ -2,9 +2,17 @@
 set -e
 echo
 
-sed=$( which gsed  2>/dev/null || which sed)
-find=$(which gfind 2>/dev/null || which find)
-sort=$(which gsort 2>/dev/null || which sort)
+# osx support
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
+	sed()  { gsed  "$@"; }
+	find() { gfind "$@"; }
+	sort() { gsort "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
+}
 
 which md5sum 2>/dev/null >/dev/null &&
 	md5sum=md5sum ||
@@ -12,27 +20,29 @@ which md5sum 2>/dev/null >/dev/null &&
 
 ver="$1"
 
-[[ "x$ver" == x ]] &&
+[ "x$ver" = x ] &&
 {
 	echo "need argument 1:  version"
 	echo
 	exit 1
 }
 
-[[ -e copyparty/__main__.py ]] || cd ..
-[[ -e copyparty/__main__.py ]] ||
+[ -e copyparty/__main__.py ] || cd ..
+[ -e copyparty/__main__.py ] ||
 {
-	echo "run me from within the copyparty folder"
+	echo "run me from within the project root folder"
 	echo
 	exit 1
 }
 
-out_dir="$(pwd | $sed -r 's@/[^/]+$@@')"
-zip_path="$out_dir/copyparty-$ver.zip"
-tgz_path="$out_dir/copyparty-$ver.tar.gz"
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
 
-[[ -e "$zip_path" ]] ||
-[[ -e "$tgz_path" ]] &&
+mkdir -p dist
+zip_path="$(pwd)/dist/copyparty-$ver.zip"
+tgz_path="$(pwd)/dist/copyparty-$ver.tar.gz"
+
+[ -e "$zip_path" ] ||
+[ -e "$tgz_path" ] &&
 {
 	echo "found existing archives for this version"
 	echo "  $zip_path"
@@ -44,23 +54,25 @@ tgz_path="$out_dir/copyparty-$ver.tar.gz"
 rm "$zip_path" 2>/dev/null || true
 rm "$tgz_path" 2>/dev/null || true
 
-#$sed -ri "s/^(ADMIN_PWD *= *u).*/\1'hunter2'/" copyparty/config.py
+#sed -ri "s/^(ADMIN_PWD *= *u).*/\1'hunter2'/" copyparty/config.py
 
 tmp="$(mktemp -d)"
 rls_dir="$tmp/copyparty-$ver"
 mkdir "$rls_dir"
 
-echo ">>> export"
-git archive master |
-tar -x -C "$rls_dir"
+echo ">>> export from git"
+git archive master | tar -xC "$rls_dir"
+
+echo ">>> export untracked deps"
+tar -c copyparty/web/deps | tar -xC "$rls_dir"
 
 cd "$rls_dir"
-$find -type d -exec chmod 755 '{}' \+
-$find -type f -exec chmod 644 '{}' \+
+find -type d -exec chmod 755 '{}' \+
+find -type f -exec chmod 644 '{}' \+
 
 commaver="$(
 	printf '%s\n' "$ver" |
-	sed -r 's/\./,/g'
+	sed -r 's/\./, /g'
 )"
 
 grep -qE "^VERSION *= \(${commaver}\)$" copyparty/__version__.py ||
@@ -85,16 +97,23 @@ rm \
 
 mv LICENSE LICENSE.txt
 
-# messy because osx support
-$find -type f -exec $md5sum '{}' \+ |
-$sed -r 's/(.{32})(.*)/\2\1/' | LC_COLLATE=c $sort |
-$sed -r 's/(.*)(.{32})/\2\1/' |
-$sed -r 's/^(.{32}) \./\1  ./' > ../.sums.md5
+# the regular cleanup memes
+find -name '*.pyc' -delete
+find -name __pycache__ -delete
+find -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
+find -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
+
+# also messy because osx support
+find -type f -exec $md5sum '{}' \+ |
+sed -r 's/(.{32})(.*)/\2\1/' | LC_COLLATE=c sort |
+sed -r 's/(.*)(.{32})/\2\1/' |
+sed -r 's/^(.{32}) \./\1  ./' > ../.sums.md5
 mv ../.sums.md5 .
 
 cd ..
-echo ">>> tar"; tar -czf "$tgz_path" "copyparty-$ver"
-echo ">>> zip"; zip -qr  "$zip_path" "copyparty-$ver"
+pwd
+echo ">>> tar"; tar -czf "$tgz_path" --owner=1000 --group=1000 --numeric-owner "copyparty-$ver"
+echo ">>> zip"; zip  -qr "$zip_path" "copyparty-$ver"
 
 rm -rf "$tmp"
 echo
@@ -103,5 +122,5 @@ echo "  $zip_path"
 echo "  $tgz_path"
 echo
 
-# function alr() { ls -alR copyparty-$1 | $sed -r "s/copyparty-$1/copyparty/" | $sed -r 's/[A-Z][a-z]{2} [0-9 ]{2} [0-9]{2}:[0-9]{2}//' > $1; }; for x in master rls src ; do alr $x; done
+# function alr() { ls -alR copyparty-$1 | sed -r "s/copyparty-$1/copyparty/" | sed -r 's/[A-Z][a-z]{2} [0-9 ]{2} [0-9]{2}:[0-9]{2}//' > $1; }; for x in master rls src ; do alr $x; done
 

scripts/sfx.py

@@ -0,0 +1,438 @@
+#!/usr/bin/env python
+# coding: latin-1
+from __future__ import print_function, unicode_literals
+
+import os, sys, time, shutil, runpy, tarfile, hashlib, platform, tempfile, traceback
+
+"""
+run me with any version of python, i will unpack and run copyparty
+
+(but please don't edit this file with a text editor
+  since that would probably corrupt the binary stuff at the end)
+
+there's zero binaries! just plaintext python scripts all the way down
+  so you can easily unpack the archive and inspect it for shady stuff
+
+the archive data is attached after the b"\n# eof\n" archive marker,
+  b"\n#n" decodes to b"\n"
+  b"\n#r" decodes to b"\r"
+  b"\n# " decodes to b""
+"""
+
+# set by make-sfx.sh
+VER = None
+SIZE = None
+CKSUM = None
+STAMP = None
+
+PY2 = sys.version_info[0] == 2
+sys.dont_write_bytecode = True
+me = os.path.abspath(os.path.realpath(__file__))
+cpp = None
+
+
+def eprint(*args, **kwargs):
+    kwargs["file"] = sys.stderr
+    print(*args, **kwargs)
+
+
+def msg(*args, **kwargs):
+    if args:
+        args = ["[SFX]", args[0]] + list(args[1:])
+
+    eprint(*args, **kwargs)
+
+
+# skip 1
+
+
+def testptn1():
+    """test: creates a test-pattern for encode()"""
+    import struct
+
+    buf = b""
+    for c in range(256):
+        buf += struct.pack("B", c)
+
+    yield buf
+
+
+def testptn2():
+    import struct
+
+    for a in range(256):
+        if a % 16 == 0:
+            msg(a)
+
+        for b in range(256):
+            buf = b""
+            for c in range(256):
+                buf += struct.pack("BBBB", a, b, c, b)
+            yield buf
+
+
+def testptn3():
+    with open("C:/Users/ed/Downloads/python-3.8.1-amd64.exe", "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(512 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
+testptn = testptn2
+
+
+def testchk(cdata):
+    """test: verifies that `data` yields testptn"""
+    import struct
+
+    cbuf = b""
+    mbuf = b""
+    checked = 0
+    t0 = time.time()
+    mdata = testptn()
+    while True:
+        if not mbuf:
+            try:
+                mbuf += next(mdata)
+            except:
+                break
+
+        if not cbuf:
+            try:
+                cbuf += next(cdata)
+            except:
+                expect = mbuf[:8]
+                expect = "".join(
+                    " {:02x}".format(x)
+                    for x in struct.unpack("B" * len(expect), expect)
+                )
+                raise Exception(
+                    "truncated at {}, expected{}".format(checked + len(cbuf), expect)
+                )
+
+        ncmp = min(len(cbuf), len(mbuf))
+        # msg("checking {:x}H bytes, {:x}H ok so far".format(ncmp, checked))
+        for n in range(ncmp):
+            checked += 1
+            if cbuf[n] != mbuf[n]:
+                expect = mbuf[n : n + 8]
+                expect = "".join(
+                    " {:02x}".format(x)
+                    for x in struct.unpack("B" * len(expect), expect)
+                )
+                cc = struct.unpack(b"B", cbuf[n : n + 1])[0]
+                raise Exception(
+                    "byte {:x}H bad, got {:02x}, expected{}".format(checked, cc, expect)
+                )
+
+        cbuf = cbuf[ncmp:]
+        mbuf = mbuf[ncmp:]
+
+    td = time.time() - t0
+    txt = "all {}d bytes OK in {:.3f} sec, {:.3f} MB/s".format(
+        checked, td, (checked / (1024 * 1024.0)) / td
+    )
+    msg(txt)
+
+
+def encode(data, size, cksum, ver, ts):
+    """creates a new sfx; `data` should yield bufs to attach"""
+    nin = 0
+    nout = 0
+    skip = False
+    with open(me, "rb") as fi:
+        unpk = ""
+        src = fi.read().replace(b"\r", b"").rstrip(b"\n").decode("utf-8")
+        for ln in src.split("\n"):
+            if ln.endswith("# skip 0"):
+                skip = False
+                continue
+
+            if ln.endswith("# skip 1") or skip:
+                skip = True
+                continue
+
+            unpk += ln + "\n"
+
+        for k, v in [
+            ["VER", '"' + ver + '"'],
+            ["SIZE", size],
+            ["CKSUM", '"' + cksum + '"'],
+            ["STAMP", ts],
+        ]:
+            v1 = "\n{} = None\n".format(k)
+            v2 = "\n{} = {}\n".format(k, v)
+            unpk = unpk.replace(v1, v2)
+
+        unpk = unpk.replace("\n    ", "\n\t")
+        for _ in range(16):
+            unpk = unpk.replace("\t    ", "\t\t")
+
+    with open("sfx.out", "wb") as f:
+        f.write(unpk.encode("utf-8") + b"\n\n# eof\n# ")
+        for buf in data:
+            ebuf = buf.replace(b"\n", b"\n#n").replace(b"\r", b"\n#r")
+            f.write(ebuf)
+            nin += len(buf)
+            nout += len(ebuf)
+
+    msg("wrote {:x}H bytes ({:x}H after encode)".format(nin, nout))
+
+
+def makesfx(tar_src, ver, ts):
+    sz = os.path.getsize(tar_src)
+    cksum = hashfile(tar_src)
+    encode(yieldfile(tar_src), sz, cksum, ver, ts)
+
+
+# skip 0
+
+
+def u8(gen):
+    try:
+        for s in gen:
+            yield s.decode("utf-8", "ignore")
+    except:
+        yield s
+        for s in gen:
+            yield s
+
+
+def yieldfile(fn):
+    with open(fn, "rb") as f:
+        for block in iter(lambda: f.read(64 * 1024), b""):
+            yield block
+
+
+def hashfile(fn):
+    hasher = hashlib.md5()
+    for block in yieldfile(fn):
+        hasher.update(block)
+
+    return hasher.hexdigest()
+
+
+def unpack():
+    """unpacks the tar yielded by `data`"""
+    name = "pe-copyparty"
+    tag = "v" + str(STAMP)
+    withpid = "{}.{}".format(name, os.getpid())
+    top = tempfile.gettempdir()
+    final = os.path.join(top, name)
+    mine = os.path.join(top, withpid)
+    tar = os.path.join(mine, "tar")
+
+    try:
+        if tag in os.listdir(final):
+            msg("found early")
+            return final
+    except:
+        pass
+
+    nwrite = 0
+    os.mkdir(mine)
+    with open(tar, "wb") as f:
+        for buf in get_payload():
+            nwrite += len(buf)
+            f.write(buf)
+
+    if nwrite != SIZE:
+        t = "\n\n  bad file:\n    expected {} bytes, got {}\n".format(SIZE, nwrite)
+        raise Exception(t)
+
+    cksum = hashfile(tar)
+    if cksum != CKSUM:
+        t = "\n\n  bad file:\n    {} expected,\n    {} obtained\n".format(CKSUM, cksum)
+        raise Exception(t)
+
+    with tarfile.open(tar, "r:bz2") as tf:
+        tf.extractall(mine)
+
+    os.remove(tar)
+
+    with open(os.path.join(mine, tag), "wb") as f:
+        f.write(b"h\n")
+
+    try:
+        if tag in os.listdir(final):
+            msg("found late")
+            return final
+    except:
+        pass
+
+    try:
+        if os.path.islink(final):
+            os.remove(final)
+        else:
+            shutil.rmtree(final)
+    except:
+        pass
+
+    try:
+        os.symlink(mine, final)
+    except:
+        try:
+            os.rename(mine, final)
+        except:
+            msg("reloc fail,", mine)
+            return mine
+
+    for fn in u8(os.listdir(top)):
+        if fn.startswith(name) and fn not in [name, withpid]:
+            try:
+                old = os.path.join(top, fn)
+                if time.time() - os.path.getmtime(old) > 10:
+                    shutil.rmtree(old)
+            except:
+                pass
+
+    return final
+
+
+def get_payload():
+    """yields the binary data attached to script"""
+    with open(me, "rb") as f:
+        ptn = b"\n# eof\n# "
+        buf = b""
+        for n in range(64):
+            buf += f.read(4096)
+            ofs = buf.find(ptn)
+            if ofs >= 0:
+                break
+
+        if ofs < 0:
+            raise Exception("could not find archive marker")
+
+        # start reading from the final b"\n"
+        fpos = ofs + len(ptn) - 3
+        # msg("tar found at", fpos)
+        f.seek(fpos)
+        dpos = 0
+        leftovers = b""
+        while True:
+            rbuf = f.read(1024 * 32)
+            if rbuf:
+                buf = leftovers + rbuf
+                ofs = buf.rfind(b"\n")
+                if len(buf) <= 4:
+                    leftovers = buf
+                    continue
+
+                if ofs >= len(buf) - 4:
+                    leftovers = buf[ofs:]
+                    buf = buf[:ofs]
+                else:
+                    leftovers = b"\n# "
+            else:
+                buf = leftovers
+
+            fpos += len(buf) + 1
+            buf = (
+                buf.replace(b"\n# ", b"")
+                .replace(b"\n#r", b"\r")
+                .replace(b"\n#n", b"\n")
+            )
+            dpos += len(buf) - 1
+
+            yield buf
+
+            if not rbuf:
+                break
+
+
+def confirm(rv):
+    msg()
+    msg(traceback.format_exc())
+    msg("*** hit enter to exit ***")
+    try:
+        raw_input() if PY2 else input()
+    except:
+        pass
+
+    sys.exit(rv)
+
+
+def run(tmp, j2ver):
+    global cpp
+
+    msg("jinja2:", j2ver or "bundled")
+    msg("sfxdir:", tmp)
+    msg()
+
+    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    try:
+        import fcntl
+
+        fd = os.open(tmp, os.O_RDONLY)
+        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
+    except:
+        pass
+
+    ld = [tmp, os.path.join(tmp, "dep-j2")]
+    if j2ver:
+        del ld[-1]
+
+    for x in ld:
+        sys.path.insert(0, x)
+
+    try:
+        runpy.run_module(str("copyparty"), run_name=str("__main__"))
+    except SystemExit as ex:
+        if ex.code:
+            confirm(ex.code)
+    except:
+        confirm(1)
+
+
+def main():
+    sysver = str(sys.version).replace("\n", "\n" + " " * 18)
+    pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
+    msg()
+    msg("   this is: copyparty", VER)
+    msg(" packed at:", pktime, "UTC,", STAMP)
+    msg("archive is:", me)
+    msg("python bin:", sys.executable)
+    msg("python ver:", platform.python_implementation(), sysver)
+    msg()
+
+    arg = ""
+    try:
+        arg = sys.argv[1]
+    except:
+        pass
+
+    # skip 1
+
+    if arg == "--sfx-testgen":
+        return encode(testptn(), 1, "x", "x", 1)
+
+    if arg == "--sfx-testchk":
+        return testchk(get_payload())
+
+    if arg == "--sfx-make":
+        tar, ver, ts = sys.argv[2:]
+        return makesfx(tar, ver, ts)
+
+    # skip 0
+
+    tmp = unpack()
+
+    try:
+        from jinja2 import __version__ as j2ver
+    except:
+        j2ver = None
+
+    run(tmp, j2ver)
+
+
+if __name__ == "__main__":
+    main()
+
+
+# skip 1
+# python sfx.py --sfx-testgen && python test.py --sfx-testchk
+# c:\Python27\python.exe sfx.py --sfx-testgen && c:\Python27\python.exe test.py --sfx-testchk

scripts/sfx.sh

@@ -0,0 +1,76 @@
+# use current/default shell
+set -e
+
+dir="$(
+	printf '%s\n' "$TMPDIR" /tmp |
+	awk '/./ {print; exit}'
+)/pe-copyparty"
+
+[ -e "$dir/vPACK_TS" ] || (
+	printf '\033[36munpacking copyparty vCPP_VER (sfx-PACK_HTS)\033[1;30m\n\n'
+	mkdir -p "$dir.$$"
+	ofs=$(awk '$0=="sfx_eof" {print NR+1; exit}' < "$0")
+	
+	[ -z "$ofs" ] && {
+		printf '\033[31mabort: could not find SFX boundary\033[0m\n'
+		exit 1
+	}
+	tail -n +$ofs "$0" | tar -JxC "$dir.$$"
+	ln -nsf "$dir.$$" "$dir"
+	printf '\033[0m'
+	
+	now=$(date -u +%s)
+	for d in "$dir".*; do
+		ts=$(stat -c%Y -- "$d" 2>/dev/null) ||
+		ts=$(stat -f %m%n -- "$d" 2>/dev/null)
+
+		[ $((now-ts)) -gt 300 ] &&
+			rm -rf "$d"
+	done
+	echo h > "$dir/vPACK_TS"
+) >&2 || exit 1
+
+# detect available pythons
+(IFS=:; for d in $PATH; do
+	printf '%s\n' "$d"/python* "$d"/pypy*;
+done) |
+(sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) |
+(sort -nr || cat) |
+(sed -E 's/([^ ]*) (.*)/\2\1/' || cat) |
+grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
+
+# see if we made a choice before
+[ -z "$pybin" ] && pybin="$(cat $dir/py 2>/dev/null || true)"
+
+# otherwise find a python with jinja2
+[ -z "$pybin" ] && pybin="$(cat $dir/pys | while IFS= read -r _py; do
+	printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
+	$_py -c 'import jinja2' 2>/dev/null || continue
+	printf '%s\n' "$_py"
+	mv $dir/{,x.}jinja2
+	break
+done)"
+
+# otherwise find python2 (bundled jinja2 is way old)
+[ -z "$pybin" ] && {
+	printf '\033[0;33mcould not find jinja2; will use py2 + the bundled version\033[0m\n' >&2
+	pybin="$(cat $dir/pys | while IFS= read -r _py; do
+		printf '\033[1;30mtesting if py2 [%s]\033[0m\n' "$_py" >&2
+		_ver=$($_py -c 'import sys; sys.stdout.write(str(sys.version_info[0]))' 2>/dev/null) || continue
+		[ $_ver = 2 ] || continue
+		printf '%s\n' "$_py"
+		break
+	done)"
+}
+
+[ -z "$pybin" ] && {
+	printf '\033[1;31m\ncould not find a python with jinja2 installed; please do one of these:\n\n  pip install --user jinja2\n\n  install python2\033[0m\n\n' >&2
+	exit 1
+}
+
+printf '\033[1;30musing [%s]. you can reset with this:\n  rm -rf %s*\033[0m\n\n' "$pybin" "$dir"
+printf '%s\n' "$pybin" > $dir/py
+
+PYTHONPATH=$dir exec "$pybin" -m copyparty "$@"
+
+sfx_eof

scripts/speedtest-fs.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import stat
+import time
+import signal
+import traceback
+import threading
+from queue import Queue
+
+
+"""speedtest-fs: filesystem performance estimate"""
+__author__ = "ed <copyparty@ocv.me>"
+__copyright__ = 2020
+__license__ = "MIT"
+__url__ = "https://github.com/9001/copyparty/"
+
+
+def get_spd(nbyte, nsec):
+    if not nsec:
+        return "0.000 MB   0.000 sec   0.000 MB/s"
+
+    mb = nbyte / (1024 * 1024.0)
+    spd = mb / nsec
+
+    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+
+
+class Inf(object):
+    def __init__(self, t0):
+        self.msgs = []
+        self.errors = []
+        self.reports = []
+        self.mtx_msgs = threading.Lock()
+        self.mtx_reports = threading.Lock()
+
+        self.n_byte = 0
+        self.n_sec = 0
+        self.n_done = 0
+        self.t0 = t0
+
+        thr = threading.Thread(target=self.print_msgs)
+        thr.daemon = True
+        thr.start()
+
+    def msg(self, fn, n_read):
+        with self.mtx_msgs:
+            self.msgs.append(f"{fn} {n_read}")
+
+    def err(self, fn):
+        with self.mtx_reports:
+            self.errors.append(f"{fn}\n{traceback.format_exc()}")
+
+    def print_msgs(self):
+        while True:
+            time.sleep(0.02)
+            with self.mtx_msgs:
+                msgs = self.msgs
+                self.msgs = []
+
+            if not msgs:
+                continue
+
+            msgs = msgs[-64:]
+            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            print("\n".join(msgs))
+
+    def report(self, fn, n_byte, n_sec):
+        with self.mtx_reports:
+            self.reports.append([n_byte, n_sec, fn])
+            self.n_byte += n_byte
+            self.n_sec += n_sec
+
+    def done(self):
+        with self.mtx_reports:
+            self.n_done += 1
+
+
+def get_files(dir_path):
+    for fn in os.listdir(dir_path):
+        fn = os.path.join(dir_path, fn)
+        st = os.stat(fn).st_mode
+
+        if stat.S_ISDIR(st):
+            yield from get_files(fn)
+
+        if stat.S_ISREG(st):
+            yield fn
+
+
+def worker(q, inf, read_sz):
+    while True:
+        fn = q.get()
+        if not fn:
+            break
+
+        n_read = 0
+        try:
+            t0 = time.time()
+            with open(fn, "rb") as f:
+                while True:
+                    buf = f.read(read_sz)
+                    if not buf:
+                        break
+
+                    n_read += len(buf)
+                    inf.msg(fn, n_read)
+
+            inf.report(fn, n_read, time.time() - t0)
+        except:
+            inf.err(fn)
+
+    inf.done()
+
+
+def sighandler(signo, frame):
+    os._exit(0)
+
+
+def main():
+    signal.signal(signal.SIGINT, sighandler)
+
+    root = "."
+    if len(sys.argv) > 1:
+        root = sys.argv[1]
+
+    t0 = time.time()
+    q = Queue(256)
+    inf = Inf(t0)
+
+    num_threads = 8
+    read_sz = 32 * 1024
+    for _ in range(num_threads):
+        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr.daemon = True
+        thr.start()
+
+    for fn in get_files(root):
+        q.put(fn)
+
+    for _ in range(num_threads):
+        q.put(None)
+
+    while inf.n_done < num_threads:
+        time.sleep(0.1)
+
+    t2 = time.time()
+    print("\n")
+
+    log = inf.reports
+    log.sort()
+    for nbyte, nsec, fn in log[-64:]:
+        print(f"{get_spd(nbyte, nsec)}   {fn}")
+
+    print()
+    print("\n".join(inf.errors))
+
+    print(get_spd(inf.n_byte, t2 - t0))
+
+
+if __name__ == "__main__":
+    main()
+

scripts/uncomment.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import io
+import sys
+import tokenize
+
+
+def uncomment(fpath):
+    """ modified https://stackoverflow.com/a/62074206 """
+
+    with open(fpath, "rb") as f:
+        orig = f.read().decode("utf-8")
+
+    out = ""
+    for ln in orig.split("\n"):
+        if not ln.startswith("#"):
+            break
+
+        out += ln + "\n"
+
+    io_obj = io.StringIO(orig)
+    prev_toktype = tokenize.INDENT
+    last_lineno = -1
+    last_col = 0
+    for tok in tokenize.generate_tokens(io_obj.readline):
+        # print(repr(tok))
+        token_type = tok[0]
+        token_string = tok[1]
+        start_line, start_col = tok[2]
+        end_line, end_col = tok[3]
+
+        if start_line > last_lineno:
+            last_col = 0
+
+        if start_col > last_col:
+            out += " " * (start_col - last_col)
+
+        is_legalese = (
+            "copyright" in token_string.lower() or "license" in token_string.lower()
+        )
+
+        if token_type == tokenize.STRING:
+            if (
+                prev_toktype != tokenize.INDENT
+                and prev_toktype != tokenize.NEWLINE
+                and start_col > 0
+                or is_legalese
+            ):
+                out += token_string
+            else:
+                out += '"a"'
+        elif token_type != tokenize.COMMENT or is_legalese:
+            out += token_string
+
+        prev_toktype = token_type
+        last_lineno = end_line
+        last_col = end_col
+
+    # out = "\n".join(x for x in out.splitlines() if x.strip())
+
+    with open(fpath, "wb") as f:
+        f.write(out.encode("utf-8"))
+
+
+def main():
+    print("uncommenting", end="")
+    for f in sys.argv[1:]:
+        print(".", end="")
+        uncomment(f)
+
+    print("k")
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -2,10 +2,8 @@
 # coding: utf-8
 from __future__ import print_function
 
-import io
 import os
 import sys
-from glob import glob
 from shutil import rmtree
 
 setuptools_available = True
@@ -49,7 +47,7 @@ with open(here + "/README.md", "rb") as f:
 about = {}
 if not VERSION:
     with open(os.path.join(here, NAME, "__version__.py"), "rb") as f:
-        exec(f.read().decode("utf-8").split("\n\n", 1)[1], about)
+        exec (f.read().decode("utf-8").split("\n\n", 1)[1], about)
 else:
     about["__version__"] = VERSION
 
@@ -110,13 +108,13 @@ args = {
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Environment :: Console",
@@ -137,16 +135,14 @@ if setuptools_available:
             "entry_points": {
                 "console_scripts": ["copyparty = copyparty.__main__:main"]
             },
-            "scripts": [
-                "bin/copyparty-fuse.py"
-            ]
+            "scripts": ["bin/copyparty-fuse.py"],
         }
     )
 else:
     args.update(
         {
             "packages": ["copyparty", "copyparty.stolen"],
-            "scripts": ["bin/copyparty", "bin/copyparty-fuse.py"]
+            "scripts": ["bin/copyparty-fuse.py"],
         }
     )
 

srv/ceditable.html

@@ -0,0 +1,84 @@
+<!DOCTYPE html><html><head>
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=0.8">
+<style>
+
+* {
+    margin: 0;
+    padding: 0;
+    outline: 0;
+    border: none;
+    font-size: 1em;
+    line-height: 1em;
+    font-family: monospace, monospace;
+    color: #333;
+}
+html, body {
+    width: 100%;
+    height: 100%;
+    background: #ddd;
+}
+html {
+    font-size: 1.3em;
+}
+li, #edit {
+    list-style-type: none;
+    white-space: pre-wrap;
+    word-break: break-all;
+    overflow-wrap: break-word;
+    word-wrap: break-word; /*ie*/
+}
+li:nth-child(even) {
+    background: #ddd;
+}
+#edit, #html, #txt1, #txt2 {
+    background: #eee;
+    position: fixed;
+    width: calc(50% - .8em);
+    height: calc(50% - .8em);
+}
+#txt1 { top: .5em; left: .5em }
+#edit { top: .5em; right: .5em }
+#html { bottom: .5em; left: .5em }
+#txt2 { bottom: .5em; right: .5em }
+
+</style></head><body>
+<pre id="edit" contenteditable="true"></pre>
+<textarea id="html"></textarea>
+<ul id="txt1"></ul>
+<ul id="txt2"></ul>
+<script>
+
+var edit = document.getElementById('edit'),
+    html = document.getElementById('html'),
+    txt1 = document.getElementById('txt1'),
+    txt2 = document.getElementById('txt2');
+
+var oh = null;
+function fun() {
+    var h = edit.innerHTML;
+    if (oh != h) {
+        oh = h;
+        html.value = h;
+        var t = edit.innerText;
+        if (h.indexOf('<div><br></div>') >= 0)
+            t = t.replace(/\n\n/g, "\n");
+        
+        t = '<li>' + t.
+            replace(/&/g, "&amp;").
+            replace(/</g, "&lt;").
+            replace(/>/g, "&gt;").
+            split('\n').join('</li>\n<li>') + '</li>';
+        
+        t = t.replace(/<li><\/li>/g, '<li> </li>');
+        txt1.innerHTML = t;
+        txt2.innerHTML = t;
+    }
+    setTimeout(fun, 100);
+}
+fun();
+
+</script>
+</body>
+</html>

srv/extend.md

@@ -0,0 +1,141 @@
+# hi
+this showcases my worst idea yet; *extending markdown with inline javascript*
+
+due to obvious reasons it's disabled by default, and can be enabled with `-emp`
+
+the examples are by no means correct, they're as much of a joke as this feature itself
+
+
+### sub-header
+nothing special about this one
+
+
+## except/
+this one becomes a hyperlink to ./except/ thanks to
+* the `copyparty_pre` plugin at the end of this file
+* which is invoked as a markdown filter every time the document is modified
+* which looks for headers ending with a `/` and erwrites all headers below that
+
+it is a passthrough to the markdown extension api, see https://marked.js.org/using_pro
+
+in addition to the markdown extension functions, `ctor` will be called on document init
+
+
+### these/
+and this one becomes ./except/these/
+
+
+#### ones.md
+finally ./except/these/ones.md
+
+
+### also-this.md
+whic hshoud be ./except/also-this.md
+
+
+
+
+# ok
+now for another extension type, `copyparty_post` which is called to manipulate the generated dom instead
+
+`copyparty_post` can have the following functions, all optional
+* `ctor` is called on document init
+* `render` is called when the dom is done but still in-memory
+* `render2` is called with the live browser dom as-displayed
+
+## post example
+
+the values in the `ex:` columns are linkified to `example.com/$value`
+
+| ex:foo       | bar      | ex:baz |
+| ------------ | -------- | ------ |
+| asdf         | nice     | fgsfds |
+| more one row | hi hello | aaa    |
+
+and the table can be sorted by clicking the headers
+
+the difference is that with `copyparty_pre` you'll probably break various copyparty features but if you use `copyparty_post` then future copyparty versions will probably break you
+
+
+
+
+# heres the plugins
+if there is anything below ths line in the preview then the plugin feature is disabled (good)
+
+
+
+
+```copyparty_pre
+ctor() {
+    md_plug['h'] = {
+        on: false,
+        lv: -1,
+        path: []
+    }
+},
+walkTokens(token) {
+    if (token.type == 'heading') {
+        var h = md_plug['h'],
+            is_dir = token.text.endsWith('/');
+        
+        if (h.lv >= token.depth) {
+            h.on = false;
+        }
+        if (!h.on && is_dir) {
+            h.on = true;
+            h.lv = token.depth;
+            h.path = [token.text];
+        }
+        else if (h.on && h.lv < token.depth) {
+            h.path = h.path.slice(0, token.depth - h.lv);
+            h.path.push(token.text);
+        }
+        if (!h.on)
+            return false;
+
+        var path = h.path.join('');
+        var emoji = is_dir ? '📂' : '📜';
+        token.tokens[0].text = '<a href="' + path + '">' + emoji + ' ' + path + '</a>';
+    }
+    if (token.type == 'paragraph') {
+        //console.log(JSON.parse(JSON.stringify(token.tokens)));
+        for (var a = 0; a < token.tokens.length; a++) {
+            var t = token.tokens[a];
+            if (t.type == 'text' || t.type == 'strong' || t.type == 'em') {
+                var ret = '', text = t.text;
+                for (var b = 0; b < text.length; b++)
+                    ret += (Math.random() > 0.5) ? text[b] : text[b].toUpperCase();
+                
+                t.text = ret;
+            }
+        }
+    }
+    return true;
+}
+```
+
+
+
+```copyparty_post
+render(dom) {
+    var ths = dom.querySelectorAll('th');
+    for (var a = 0; a < ths.length; a++) {
+        var th = ths[a];
+        if (th.textContent.indexOf('ex:') === 0) {
+            th.textContent = th.textContent.slice(3);
+            var nrow = 0;
+            while ((th = th.previousSibling) != null)
+                nrow++;
+            
+            var trs = ths[a].parentNode.parentNode.parentNode.querySelectorAll('tr');
+            for (var b = 1; b < trs.length; b++) {
+                var td = trs[b].childNodes[nrow];
+                td.innerHTML = '<a href="//example.com/' + td.innerHTML + '">' + td.innerHTML + '</a>';
+            }
+        }
+    }
+},
+render2(dom) {
+    window.makeSortable(dom.getElementsByTagName('table')[0]);
+}
+```

srv/test.md

@@ -0,0 +1,237 @@
+### hello world
+
+* qwe
+* asd
+  * zxc
+  * 573
+    * one
+    * two
+    
+  * |||
+    |--|--|
+    |listed|table|
+
+```
+[72....................................................................]
+[80............................................................................]
+```
+
+* foo
+  ```
+  [72....................................................................]
+  [80............................................................................]
+  ```
+
+  * bar
+    ```
+    [72....................................................................]
+    [80............................................................................]
+    ```
+
+```
+l[i]=1I;(){}o0O</> var foo = "$(`bar`)"; a's'd
+```
+
+🔍🌽.📕.🍙🔎
+
+[](#s1)
+[s1](#s1)
+[#s1](#s1)
+
+a123456789b123456789c123456789d123456789e123456789f123456789g123456789h123456789i123456789j123456789k123456789l123456789m123456789n123456789o123456789p123456789q123456789r123456789s123456789t123456789u123456789v123456789w123456789x123456789y123456789z123456789
+
+<foo> &nbsp; bar &amp; <span>baz</span>
+<a href="?foo=bar&baz=qwe&amp;rty">?foo=bar&baz=qwe&amp;rty</a>
+<!-- hidden -->
+```
+<foo> &nbsp; bar &amp; <span>baz</span>
+<a href="?foo=bar&baz=qwe&amp;rty">?foo=bar&baz=qwe&amp;rty</a>
+<!-- visible -->
+```
+
+*fails marked/showdown/tui/simplemde (just italics), **OK: markdown-it/simplemde:***  
+testing just google.com and underscored _google.com_ also with _google.com,_ trailing comma and _google.com_, comma after
+
+*fails tui (just italics), **OK: marked/showdown/markdown-it/simplemde:***  
+testing just https://google.com and underscored _https://google.com_ links like that
+
+*fails marked (no markup) and showdown/tui/simplemde (no links at all), **OK: markdown-it:***  
+let's try <google.com> bracketed and _<google.com>_ underscored bracketed
+
+*fails marked (literal underscore), **OK: showdown/markdown-it/simplemde:***  
+let's try <https://google.com> bracketed and _<https://google.com>_ underscored bracketed
+
+*fails none:*  
+and then [google](google.com) verbose and _[google](google.com)_ underscored  
+
+*fails none:*  
+and then [google](https://google.com/) verbose and _[google](https://google.com/)_ underscored
+
+*all behave similarly (only verbose ones):*  
+and then <local> or maybe <./local> fsgfds </absolute> fsgfds  
+and then [local] or maybe [./local] fsgfds [/absolute] fsgfds  
+and then (local) or maybe (./local) fsgfds (/absolute) fsgfds  
+and then [](local) or maybe [](./local) fsgfds [](/absolute) fsgfds  
+and then [.](local) or maybe [.](./local) fsgfds [.](/absolute) fsgfds  
+and then [asdf](local) or maybe [asdf](./local) fsgfds [asdf](/absolute) fsgfds
+
+*`ng/OK/OK/OK markdown-it`  
+`ng/OK/ng/OK marked`  
+`ng/OK/OK/OK showdown`
+`OK/OK/OK/OK simplemde`*  
+[with spaces](/with spaces) plain, [with spaces](/with%20spaces) %20, [with spaces](</with spaces>) brackets, [with spaces](/with%20spaces) %20
+
+*this fails marked, **OK: markdown-it, simplemde:***
+
+* testing a list with:
+  `some code after a newline`
+
+* testing a list with:
+  just a newline
+
+and here is really just
+a newline toplevel
+
+*this fails showdown/hypermd, **OK: marked/markdown-it/simplemde:***
+
+* testing a list with
+
+      code here
+      and a newline
+        this should have two leading spaces
+
+  * second list level
+
+        more code here
+        and a newline
+          this should have two leading spaces
+
+.
+
+* testing a list with
+
+      code here
+      and a newline
+        this should have two leading spaces
+
+    * second list level
+
+          more code here
+          and a newline
+            this should have two leading spaces
+
+*this fails stackedit, **OK: showdown/marked/markdown-it/simplemde:***
+
+|||
+|--|--|
+| a table | with no header |
+| second row | foo bar |
+
+*this fails showdown/stackedit, **OK: marked/markdown-it/simplemde:***
+
+|||
+|--|--:|
+| a table | on the right |
+| second row | foo bar |
+
+||
+--|:-:|-:
+a table | big text in this | aaakbfddd
+second row | centred | bbb
+
+||
+--|--|--
+foo
+
+* list entry
+* [x] yes
+* [ ] no
+* another entry
+
+# s1
+## ep1
+## ep2
+# s2
+## ep1
+## ep2
+# s3
+## ep1
+## ep2
+
+
+
+#######################################################################
+
+
+
+marked:
+  works in last ff/chrome for xp
+  bug: config{breaks:true} does nothing in 1.0
+  use whitespace, no tabs
+
+showdown:
+  ie6 and ie8 broken, works in last ff/chrome for xp
+
+markdown-it:
+  works in last ff/chrome for xp
+  use whitespace, no tabs
+  no header anchors
+
+tui wysiwyg:
+  requires links to be <http://> or [title](location)
+
+
+
+links:
+  http://demo.showdownjs.com/
+  https://marked.js.org/demo/
+  https://markdown-it.github.io/
+  https://simplemde.com/
+
+
+
+all-pass:
+
+https://github.com/joemccann/dillinger
+  https://dillinger.io/
+  uses markdown-it
+
+https://github.com/markdown-it/markdown-it
+  https://markdown-it.github.io/
+
+
+
+almost-all-pass:
+
+https://github.com/Ionaru/easy-markdown-editor
+  https://easymde.tk/
+  simplemde fork (the most active)
+
+https://github.com/Inscryb/inscryb-markdown-editor
+  simplemde fork
+
+other simplemde forks:
+  pulkitmittal
+
+https://simplemde.com/
+  (dead)
+
+https://github.com/nhn/tui.editor
+  https://nhn.github.io/tui.editor/latest/tutorial-example01-editor-basic
+  ie10 and up
+
+
+
+unrelated neat stuff:
+  https://github.com/gnab/remark
+
+
+
+```sh
+awk '/./ {printf "%s %d\n", $0, NR; next} 1' <test.md >ln.md
+gawk '{print gensub(/([a-zA-Z\.])/,NR" \\1","1")}' <test.md >ln.md
+```
+
+a|b|c
+--|--|--
+foo

tests/test_vfs.py

@@ -3,8 +3,10 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import time
 import json
 import shutil
+import tempfile
 import unittest
 import subprocess as sp  # nosec
 
@@ -14,6 +16,14 @@ from copyparty.authsrv import AuthSrv
 from copyparty import util
 
 
+class Cfg(Namespace):
+    def __init__(self, a=[], v=[], c=None):
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
+        super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
+
+
 class TestVFS(unittest.TestCase):
     def dump(self, vfs):
         print(json.dumps(vfs, indent=4, sort_keys=True, default=lambda o: o.__dict__))
@@ -30,13 +40,16 @@ class TestVFS(unittest.TestCase):
         response = self.unfoo(response)
         self.assertEqual(util.undot(query), response)
 
-    def absify(self, root, names):
-        return ["{}/{}".format(root, x).replace("//", "/") for x in names]
-
     def ls(self, vfs, vpath, uname):
         """helper for resolving and listing a folder"""
         vn, rem = vfs.get(vpath, uname, True, False)
-        return vn.ls(rem, uname)
+        r1 = vn.ls(rem, uname, False)
+        r2 = vn.ls(rem, uname, False)
+        self.assertEqual(r1, r2)
+
+        fsdir, real, virt = r1
+        real = [x[0] for x in real]
+        return fsdir, real, virt
 
     def runcmd(self, *argv):
         p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
@@ -59,13 +72,31 @@ class TestVFS(unittest.TestCase):
 
         if os.path.exists("/Volumes"):
             devname, _ = self.chkcmd("hdiutil", "attach", "-nomount", "ram://8192")
-            _, _ = self.chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
-            return "/Volumes/cptd"
+            devname = devname.strip()
+            print("devname: [{}]".format(devname))
+            for _ in range(10):
+                try:
+                    _, _ = self.chkcmd(
+                        "diskutil", "eraseVolume", "HFS+", "cptd", devname
+                    )
+                    return "/Volumes/cptd"
+                except Exception as ex:
+                    print(repr(ex))
+                    time.sleep(0.25)
 
-        raise Exception("TODO support windows")
+            raise Exception("ramdisk creation failed")
+
+        ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
+        try:
+            os.mkdir(ret)
+        finally:
+            return ret
+
+    def log(self, src, msg, c=0):
+        pass
 
     def test(self):
-        td = self.get_ramdisk() + "/vfs"
+        td = os.path.join(self.get_ramdisk(), "vfs")
         try:
             shutil.rmtree(td)
         except OSError:
@@ -85,7 +116,7 @@ class TestVFS(unittest.TestCase):
                             f.write(fn)
 
         # defaults
-        vfs = AuthSrv(Namespace(c=None, a=[], v=[]), None).vfs
+        vfs = AuthSrv(Cfg(), self.log).vfs
         self.assertEqual(vfs.nodes, {})
         self.assertEqual(vfs.vpath, "")
         self.assertEqual(vfs.realpath, td)
@@ -93,24 +124,25 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(vfs.uwrite, ["*"])
 
         # single read-only rootfs (relative path)
-        vfs = AuthSrv(Namespace(c=None, a=[], v=["a/ab/::r"]), None).vfs
+        vfs = AuthSrv(Cfg(v=["a/ab/::r"]), self.log).vfs
         self.assertEqual(vfs.nodes, {})
         self.assertEqual(vfs.vpath, "")
-        self.assertEqual(vfs.realpath, td + "/a/ab")
+        self.assertEqual(vfs.realpath, os.path.join(td, "a", "ab"))
         self.assertEqual(vfs.uread, ["*"])
         self.assertEqual(vfs.uwrite, [])
 
         # single read-only rootfs (absolute path)
-        vfs = AuthSrv(Namespace(c=None, a=[], v=[td + "//a/ac/../aa//::r"]), None).vfs
+        vfs = AuthSrv(Cfg(v=[td + "//a/ac/../aa//::r"]), self.log).vfs
         self.assertEqual(vfs.nodes, {})
         self.assertEqual(vfs.vpath, "")
-        self.assertEqual(vfs.realpath, td + "/a/aa")
+        self.assertEqual(vfs.realpath, os.path.join(td, "a", "aa"))
         self.assertEqual(vfs.uread, ["*"])
         self.assertEqual(vfs.uwrite, [])
 
         # read-only rootfs with write-only subdirectory (read-write for k)
         vfs = AuthSrv(
-            Namespace(c=None, a=["k:k"], v=[".::r:ak", "a/ac/acb:a/ac/acb:w:ak"]), None
+            Cfg(a=["k:k"], v=[".::r:ak", "a/ac/acb:a/ac/acb:w:ak"]),
+            self.log,
         ).vfs
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(vfs.vpath, "")
@@ -132,47 +164,69 @@ class TestVFS(unittest.TestCase):
         n = n.nodes["acb"]
         self.assertEqual(n.nodes, {})
         self.assertEqual(n.vpath, "a/ac/acb")
-        self.assertEqual(n.realpath, td + "/a/ac/acb")
+        self.assertEqual(n.realpath, os.path.join(td, "a", "ac", "acb"))
         self.assertEqual(n.uread, ["k"])
         self.assertEqual(n.uwrite, ["*", "k"])
 
+        # something funky about the windows path normalization,
+        # doesn't really matter but makes the test messy, TODO?
+
         fsdir, real, virt = self.ls(vfs, "/", "*")
         self.assertEqual(fsdir, td)
         self.assertEqual(real, ["b", "c"])
-        self.assertEqual(virt, ["a"])
+        self.assertEqual(list(virt), ["a"])
 
         fsdir, real, virt = self.ls(vfs, "a", "*")
-        self.assertEqual(fsdir, td + "/a")
+        self.assertEqual(fsdir, os.path.join(td, "a"))
         self.assertEqual(real, ["aa", "ab"])
-        self.assertEqual(virt, ["ac"])
+        self.assertEqual(list(virt), ["ac"])
 
         fsdir, real, virt = self.ls(vfs, "a/ab", "*")
-        self.assertEqual(fsdir, td + "/a/ab")
+        self.assertEqual(fsdir, os.path.join(td, "a", "ab"))
         self.assertEqual(real, ["aba", "abb", "abc"])
-        self.assertEqual(virt, [])
+        self.assertEqual(list(virt), [])
 
         fsdir, real, virt = self.ls(vfs, "a/ac", "*")
-        self.assertEqual(fsdir, td + "/a/ac")
+        self.assertEqual(fsdir, os.path.join(td, "a", "ac"))
         self.assertEqual(real, ["aca", "acc"])
-        self.assertEqual(virt, [])
+        self.assertEqual(list(virt), [])
 
         fsdir, real, virt = self.ls(vfs, "a/ac", "k")
-        self.assertEqual(fsdir, td + "/a/ac")
+        self.assertEqual(fsdir, os.path.join(td, "a", "ac"))
         self.assertEqual(real, ["aca", "acc"])
-        self.assertEqual(virt, ["acb"])
+        self.assertEqual(list(virt), ["acb"])
 
         self.assertRaises(util.Pebkac, vfs.get, "a/ac/acb", "*", True, False)
 
         fsdir, real, virt = self.ls(vfs, "a/ac/acb", "k")
-        self.assertEqual(fsdir, td + "/a/ac/acb")
+        self.assertEqual(fsdir, os.path.join(td, "a", "ac", "acb"))
         self.assertEqual(real, ["acba", "acbb", "acbc"])
-        self.assertEqual(virt, [])
+        self.assertEqual(list(virt), [])
+
+        # admin-only rootfs with all-read-only subfolder
+        vfs = AuthSrv(
+            Cfg(a=["k:k"], v=[".::ak", "a:a:r"]),
+            self.log,
+        ).vfs
+        self.assertEqual(len(vfs.nodes), 1)
+        self.assertEqual(vfs.vpath, "")
+        self.assertEqual(vfs.realpath, td)
+        self.assertEqual(vfs.uread, ["k"])
+        self.assertEqual(vfs.uwrite, ["k"])
+        n = vfs.nodes["a"]
+        self.assertEqual(len(vfs.nodes), 1)
+        self.assertEqual(n.vpath, "a")
+        self.assertEqual(n.realpath, os.path.join(td, "a"))
+        self.assertEqual(n.uread, ["*"])
+        self.assertEqual(n.uwrite, [])
+        self.assertEqual(vfs.can_access("/", "*"), [False, False])
+        self.assertEqual(vfs.can_access("/", "k"), [True, True])
+        self.assertEqual(vfs.can_access("/a", "*"), [True, False])
+        self.assertEqual(vfs.can_access("/a", "k"), [True, False])
 
         # breadth-first construction
         vfs = AuthSrv(
-            Namespace(
-                c=None,
-                a=[],
+            Cfg(
                 v=[
                     "a/ac/acb:a/ac/acb:w",
                     "a:a:w",
@@ -181,7 +235,7 @@ class TestVFS(unittest.TestCase):
                     "a/ac:a/ac:w",
                 ],
             ),
-            None,
+            self.log,
         ).vfs
 
         # sanitizing relative paths
@@ -193,28 +247,28 @@ class TestVFS(unittest.TestCase):
         self.undot(vfs, "./.././foo/..", "")
 
         # shadowing
-        vfs = AuthSrv(Namespace(c=None, a=[], v=[".::r", "b:a/ac:r"]), None).vfs
+        vfs = AuthSrv(Cfg(v=[".::r", "b:a/ac:r"]), self.log).vfs
 
         fsp, r1, v1 = self.ls(vfs, "", "*")
         self.assertEqual(fsp, td)
         self.assertEqual(r1, ["b", "c"])
-        self.assertEqual(v1, ["a"])
+        self.assertEqual(list(v1), ["a"])
 
         fsp, r1, v1 = self.ls(vfs, "a", "*")
-        self.assertEqual(fsp, td + "/a")
+        self.assertEqual(fsp, os.path.join(td, "a"))
         self.assertEqual(r1, ["aa", "ab"])
-        self.assertEqual(v1, ["ac"])
+        self.assertEqual(list(v1), ["ac"])
 
         fsp1, r1, v1 = self.ls(vfs, "a/ac", "*")
         fsp2, r2, v2 = self.ls(vfs, "b", "*")
-        self.assertEqual(fsp1, td + "/b")
-        self.assertEqual(fsp2, td + "/b")
+        self.assertEqual(fsp1, os.path.join(td, "b"))
+        self.assertEqual(fsp2, os.path.join(td, "b"))
         self.assertEqual(r1, ["ba", "bb", "bc"])
         self.assertEqual(r1, r2)
-        self.assertEqual(v1, v2)
+        self.assertEqual(list(v1), list(v2))
 
         # config file parser
-        cfg_path = self.get_ramdisk() + "/test.cfg"
+        cfg_path = os.path.join(self.get_ramdisk(), "test.cfg")
         with open(cfg_path, "wb") as f:
             f.write(
                 dedent(
@@ -230,7 +284,7 @@ class TestVFS(unittest.TestCase):
                 ).encode("utf-8")
             )
 
-        au = AuthSrv(Namespace(c=[cfg_path], a=[], v=[]), None)
+        au = AuthSrv(Cfg(c=[cfg_path]), self.log)
         self.assertEqual(au.user["a"], "123")
         self.assertEqual(au.user["asd"], "fgh:jkl")
         n = au.vfs
@@ -242,10 +296,11 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(len(n.nodes), 1)
         n = n.nodes["dst"]
         self.assertEqual(n.vpath, "dst")
-        self.assertEqual(n.realpath, td + "/src")
+        self.assertEqual(n.realpath, os.path.join(td, "src"))
         self.assertEqual(n.uread, ["a", "asd"])
         self.assertEqual(n.uwrite, ["asd"])
         self.assertEqual(len(n.nodes), 0)
 
+        os.chdir(tempfile.gettempdir())
         shutil.rmtree(td)
         os.unlink(cfg_path)