v0.11.24

navigating somewhere and back can return a REALLY OLD copy of the page

.eslintrc.json

@@ -0,0 +1,12 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 12
+    },
+    "rules": {
+    }
+}

.gitattributes

@@ -1,4 +1,6 @@
 * text eol=lf
 
+*.reg text eol=crlf
+
 *.png binary
 *.gif binary

.vscode/launch.json

@@ -10,12 +10,24 @@
             "cwd": "${workspaceFolder}",
             "args": [
                 //"-nw",
-                "-a",
-                "ed:wark",
-                "-v",
-                "srv::r:aed"
+                "-ed",
+                "-emp",
+                "-e2dsa",
+                "-e2ts",
+                "-mtp",
+                ".bpm=f,bin/mtag/audio-bpm.py",
+                "-aed:wark",
+                "-vsrv::r:aed:cnodupe",
+                "-vdist:dist:r"
             ]
         },
+        {
+            "name": "No debug",
+            "preLaunchTask": "no_dbg",
+            "type": "python",
+            //"request": "attach", "port": 42069
+            // fork: nc -l 42069 </dev/null
+        },
         {
             "name": "Run active unit test",
             "type": "python",
@@ -28,5 +40,13 @@
                 "${file}"
             ]
         },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
     ]
 }

.vscode/launch.py

@@ -0,0 +1,45 @@
+# takes arguments from launch.json
+# is used by no_dbg in tasks.json
+# launches 10x faster than mspython debugpy
+# and is stoppable with ^C
+
+import re
+import os
+import sys
+
+print(sys.executable)
+
+import shlex
+import jstyleson
+import subprocess as sp
+
+
+with open(".vscode/launch.json", "r", encoding="utf-8") as f:
+    tj = f.read()
+
+oj = jstyleson.loads(tj)
+argv = oj["configurations"][0]["args"]
+
+try:
+    sargv = " ".join([shlex.quote(x) for x in argv])
+    print(sys.executable + " -m copyparty " + sargv + "\n")
+except:
+    pass
+
+argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
+
+if re.search(" -j ?[0-9]", " ".join(argv)):
+    argv = [sys.executable, "-m", "copyparty"] + argv
+    sp.check_call(argv)
+else:
+    sys.path.insert(0, os.getcwd())
+    from copyparty.__main__ import main as copyparty
+
+    try:
+        copyparty(["a"] + argv)
+    except SystemExit as ex:
+        if ex.code:
+            raise
+
+print("\n\033[32mokke\033[0m")
+sys.exit(1)

.vscode/settings.json

@@ -50,11 +50,9 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "editor.codeActionsOnSaveTimeout": 9001,
-    "editor.formatOnSaveTimeout": 9001,
-    //
-    //  things you may wanna edit:
-    //
-    "python.pythonPath": "/usr/bin/python3",
-    //"python.linting.enabled": true,
+    "python.formatting.blackArgs": [
+        "-t",
+        "py27"
+    ],
+    "python.linting.enabled": true,
 }

.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "pre",
+            "command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
+            "type": "shell"
+        },
+        {
+            "label": "no_dbg",
+            "type": "shell",
+            "command": "${config:python.pythonPath} .vscode/launch.py"
+        }
+    ]
+}

README.md

@@ -8,39 +8,419 @@
 
 turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
 
-* server runs on anything with `py2.7` or `py3.2+`
-* *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
+* server runs on anything with `py2.7` or `py3.3+`
+* browse/upload with IE4 / netscape4.0 on win3.11 (heh)
+* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+` for full speed
 * code standard: `black`
 
+📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [thumbnails](#thumbnails) // [md-viewer](#markdown-viewer) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [ie4](#browser-support)
+
+
+## readme toc
+
+* top
+    * [quickstart](#quickstart)
+    * [notes](#notes)
+    * [status](#status)
+* [bugs](#bugs)
+    * [general bugs](#general-bugs)
+    * [not my bugs](#not-my-bugs)
+* [the browser](#the-browser)
+    * [tabs](#tabs)
+    * [hotkeys](#hotkeys)
+    * [tree-mode](#tree-mode)
+    * [thumbnails](#thumbnails)
+    * [zip downloads](#zip-downloads)
+    * [uploading](#uploading)
+        * [file-search](#file-search)
+    * [markdown viewer](#markdown-viewer)
+    * [other tricks](#other-tricks)
+* [searching](#searching)
+    * [search configuration](#search-configuration)
+    * [database location](#database-location)
+    * [metadata from audio files](#metadata-from-audio-files)
+    * [file parser plugins](#file-parser-plugins)
+    * [complete examples](#complete-examples)
+* [browser support](#browser-support)
+* [client examples](#client-examples)
+* [up2k](#up2k)
+* [dependencies](#dependencies)
+    * [optional dependencies](#optional-dependencies)
+    * [install recommended deps](#install-recommended-deps)
+    * [optional gpl stuff](#optional-gpl-stuff)
+* [sfx](#sfx)
+    * [sfx repack](#sfx-repack)
+* [install on android](#install-on-android)
+* [building](#building)
+    * [dev env setup](#dev-env-setup)
+    * [just the sfx](#just-the-sfx)
+    * [complete release](#complete-release)
+* [todo](#todo)
+
+
+## quickstart
+
+download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and you're all set!
+
+running the sfx without arguments (for example doubleclicking it on Windows) will give everyone full access to the current folder; see `-h` for help if you want accounts and volumes etc
+
+some recommended options:
+* `-e2dsa` enables general file indexing, see [search configuration](#search-configuration)
+* `-e2ts` enables audio metadata indexing (needs either FFprobe or mutagen), see [optional dependencies](#optional-dependencies)
+* `-v /mnt/music:/music:r:afoo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, with user `foo` as `a`dmin (read/write), password `bar`
+  * replace `:r:afoo` with `:rfoo` to only make the folder readable by `foo` and nobody else
+  * in addition to `r`ead and `a`dmin, `w`rite makes a folder write-only, so cannot list/access files in it
+* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access
+
+you may also want these, especially on servers:
+* [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
+* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
+
 
 ## notes
 
-* iPhone/iPad: use Firefox to download files
-* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
-* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
-* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+general:
 * paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
   * because no browsers currently implement the media-query to do this properly orz
 
+browser-specific:
+* iPhone/iPad: use Firefox to download files
+* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
+* Android-Firefox: takes a while to select files (their fix for ☝️)
+* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
+* Desktop-Firefox: may stop you from deleting folders you've uploaded until you visit `about:memory` and click `Minimize memory usage`
+
 
 ## status
 
-* [x] sanic multipart parser
-* [x] load balancer (multiprocessing)
-* [x] upload (plain multipart, ie6 support)
-* [x] upload (js, resumable, multithreaded)
-* [x] download
-* [x] browser
-* [x] media player
-* [ ] thumbnails
-* [ ] download as zip
-* [x] volumes
-* [x] accounts
-* [x] markdown viewer
-* [x] markdown editor
-* [x] FUSE client (read-only)
+summary: all planned features work! now please enjoy the bloatening
 
-summary: it works! you can use it! (but technically not even close to beta)
+* backend stuff
+  * ☑ sanic multipart parser
+  * ☑ load balancer (multiprocessing)
+  * ☑ volumes (mountpoints)
+  * ☑ accounts
+* upload
+  * ☑ basic: plain multipart, ie6 support
+  * ☑ up2k: js, resumable, multithreaded
+  * ☑ stash: simple PUT filedropper
+  * ☑ symlink/discard existing files (content-matching)
+* download
+  * ☑ single files in browser
+  * ☑ folders as zip / tar files
+  * ☑ FUSE client (read-only)
+* browser
+  * ☑ tree-view
+  * ☑ audio player (with OS media controls)
+  * ☑ thumbnails
+    * ☑ images using Pillow
+    * ☑ videos using FFmpeg
+    * ☑ cache eviction (max-age; maybe max-size eventually)
+  * ☑ image gallery
+  * ☑ SPA (browse while uploading)
+    * if you use the file-tree on the left only, not folders in the file list
+* server indexing
+  * ☑ locate files by contents
+  * ☑ search by name/path/date/size
+  * ☑ search by ID3-tags etc.
+* markdown
+  * ☑ viewer
+  * ☑ editor (sure why not)
+
+
+# bugs
+
+* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
+* Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* MacOS: `--th-ff-jpg` may fix thumbnails using macports-FFmpeg
+
+## general bugs
+
+* all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
+* probably more, pls let me know
+
+## not my bugs
+
+* Windows: folders cannot be accessed if the name ends with `.`
+  * python or windows bug
+
+* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
+  * this is an msys2 bug, the regular windows edition of python is fine
+
+
+# the browser
+
+![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/115978054-65106380-a57d-11eb-98f8-59e3dee73557.png)
+
+
+## tabs
+
+* `[🔎]` search by size, date, path/name, mp3-tags ... see [searching](#searching)
+* `[🚀]` and `[🎈]` are the uploaders, see [uploading](#uploading)
+* `[📂]` mkdir, create directories
+* `[📝]` new-md, create a new markdown document
+* `[📟]` send-msg, either to server-log or into textfiles if `--urlform save`
+* `[⚙️]` client configuration options
+
+
+## hotkeys
+
+the browser has the following hotkeys
+* `B` toggle breadcrumbs / directory tree
+* `I/K` prev/next folder
+* `M` parent folder
+* `G` toggle list / grid view
+* `T` toggle thumbnails / icons
+* when playing audio:
+  * `0..9` jump to 10%..90%
+  * `U/O` skip 10sec back/forward
+  * `J/L` prev/next song
+  * `P` play/pause (also starts playing the folder)
+* in the grid view:
+  * `S` toggle multiselect
+  * `A/D` zoom
+
+
+## tree-mode
+
+by default there's a breadcrumbs path; you can replace this with a tree-browser sidebar thing by clicking the `🌲` or pressing the `B` hotkey
+
+click `[-]` and `[+]` (or hotkeys `A`/`D`) to adjust the size, and the `[a]` toggles if the tree should widen dynamically as you go deeper or stay fixed-size
+
+
+## thumbnails
+
+![copyparty-thumbs-fs8](https://user-images.githubusercontent.com/241032/120070302-10836b00-c08a-11eb-8eb4-82004a34c342.png)
+
+it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how destructive your users are
+
+images named `folder.jpg` and `folder.png` become the thumbnail of the folder they're in
+
+
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+you can also zip a selection of files or folders by clicking them in the browser, that brings up a selection editor and zip button in the bottom right
+
+![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/116008321-372a2e00-a614-11eb-9a4a-4a1fd9074224.png)
+
+## uploading
+
+two upload methods are available in the html client:
+* `🎈 bup`, the basic uploader, supports almost every browser since netscape 4.0
+* `🚀 up2k`, the fancy one
+
+up2k has several advantages:
+* you can drop folders into the browser (files are added recursively)
+* files are processed in chunks, and each chunk is checksummed
+  * uploads resume if they are interrupted (for example by a reboot)
+  * server detects any corruption; the client reuploads affected chunks
+  * the client doesn't upload anything that already exists on the server
+* the last-modified timestamp of the file is preserved
+
+see [up2k](#up2k) for details on how it works
+
+![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/115978061-680b5400-a57d-11eb-9ef6-cbb5f60aeccc.png)
+
+**protip:** you can avoid scaring away users with [docs/minimal-up2k.html](docs/minimal-up2k.html) which makes it look [much simpler](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+the up2k UI is the epitome of polished inutitive experiences:
+* "parallel uploads" specifies how many chunks to upload at the same time
+* `[🏃]` analysis of other files should continue while one is uploading
+* `[💭]` ask for confirmation before files are added to the list
+* `[💤]` sync uploading between other copyparty tabs so only one is active
+* `[🔎]` switch between upload and file-search mode
+
+and then theres the tabs below it,
+* `[ok]` is uploads which completed successfully
+* `[ng]` is the uploads which failed / got rejected (already exists, ...)
+* `[done]` shows a combined list of `[ok]` and `[ng]`, chronological order
+* `[busy]` files which are currently hashing, pending-upload, or uploading
+  * plus up to 3 entries each from `[done]` and `[que]` for context
+* `[que]` is all the files that are still queued
+
+### file-search
+
+![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/116008320-36919780-a614-11eb-803f-04162326a700.png)
+
+in the `[🚀 up2k]` tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere already
+
+files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
+* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else, this is no longer the case but now i've warmed up to the idea too much
+
+adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files
+
+note that since up2k has to read the file twice, `[🎈 bup]` can be up to 2x faster in extreme cases (if your internet connection is faster than the read-speed of your HDD)
+
+up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well thanks to tls also functioning as an integrity check
+
+
+## markdown viewer
+
+![copyparty-md-read-fs8](https://user-images.githubusercontent.com/241032/115978057-66419080-a57d-11eb-8539-d2be843991aa.png)
+
+* the document preview has a max-width which is the same as an A4 paper when printed
+
+
+## other tricks
+
+* you can link a particular timestamp in an audio file by adding it to the URL, such as `&20` / `&20s` / `&1m20` / `&t=1:20` after the `.../#af-c8960dab`
+
+* if you are using media hotkeys to switch songs and are getting tired of seeing the OSD popup which Windows doesn't let you disable, consider https://ocv.me/dev/?media-osd-bgone.ps1
+
+
+# searching
+
+![copyparty-search-fs8](https://user-images.githubusercontent.com/241032/115978060-6772bd80-a57d-11eb-81d3-174e869b72c3.png)
+
+when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
+* make search queries by `size`/`date`/`directory-path`/`filename`, or...
+* drag/drop a local file to see if the same contents exist somewhere on the server, see [file-search](#file-search)
+
+path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
+* path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
+* name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9)
+
+add `-e2ts` to also scan/index tags from music files:
+
+
+## search configuration
+
+searching relies on two databases, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`). Configuration can be done through arguments, volume flags, or a mix of both.
+
+through arguments:
+* `-e2d` enables file indexing on upload
+* `-e2ds` scans writable folders on startup
+* `-e2dsa` scans all mounted volumes (including readonly ones)
+* `-e2t` enables metadata indexing on upload
+* `-e2ts` scans for tags in all files that don't have tags yet
+* `-e2tsr` deletes all existing tags, so a full reindex
+
+the same arguments can be set as volume flags, in addition to `d2d` and `d2t` for disabling:
+* `-v ~/music::r:ce2dsa:ce2tsr` does a full reindex of everything on startup
+* `-v ~/music::r:cd2d` disables **all** indexing, even if any `-e2*` are on
+* `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
+
+note:
+* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those
+* the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
+
+you can choose to only index filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash` or the volume-flag `cdhash`, this has the following consequences:
+* initial indexing is way faster, especially when the volume is on a networked disk
+* makes it impossible to [file-search](#file-search)
+* if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
+
+if you set `--no-hash`, you can enable hashing for specific volumes using flag `cehash`
+
+
+## database location
+
+copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
+
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+* `--hist ~/.cache/copyparty -v ~/music::r:chist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
+
+note:
+* markdown edits are always stored in a local `.hist` subdirectory
+* on windows the volflag path is cyglike, so `/c/temp` means `C:\temp` but use regular paths for `--hist`
+  * you can use cygpaths for volumes too, `-v C:\Users::r` and `-v /c/users::r` both work
+
+
+## metadata from audio files
+
+`-mte` decides which tags to index and display in the browser (and also the display order), this can be changed per-volume:
+* `-v ~/music::r:cmte=title,artist` indexes and displays *title* followed by *artist*
+
+if you add/remove a tag from `mte` you will need to run with `-e2tsr` once to rebuild the database, otherwise only new files will be affected
+
+`-mtm` can be used to add or redefine a metadata mapping, say you have media files with `foo` and `bar` tags and you want them to display as `qux` in the browser (preferring `foo` if both are present), then do `-mtm qux=foo,bar` and now you can `-mte artist,title,qux`
+
+tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric value
+
+see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
+
+`--no-mutagen` disables mutagen and uses ffprobe instead, which...
+* is about 20x slower than mutagen
+* catches a few tags that mutagen doesn't
+  * melodic key, video resolution, framerate, pixfmt
+* avoids pulling any GPL code into copyparty
+* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
+
+
+## file parser plugins
+
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (as argument or volume flag), there is a default timeout of 30sec
+
+* `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
+* `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
+* `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
+
+*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` only do non-audio files, or `ad` do all files (d as in dontcare) 
+
+* `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`)
+* `-mtp arch,built,ver,orig=an,eexe,edll,~/bin/exe.py` runs `~/bin/exe.py` to get properties about windows-binaries only if file is not audio (`an`) and file extension is exe or dll
+
+
+## complete examples
+
+* read-only music server with bpm and key scanning  
+  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+
+
+# browser support
+
+![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/118192791-fb31fe00-b446-11eb-9647-898ea8efc1f7.png)
+
+`ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
+
+| feature         | ie6 | ie9 | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+| --------------- | --- | --- | ---- | ---- | ----- | ---- | --- | ---- |
+| browse files    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| basic uploader  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| make directory  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| send message    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| set sort order  |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| zip selection   |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| directory tree  |  -  |  -  | `*1` | yep  | yep   | yep  | yep | yep  |
+| up2k            |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| icons work      |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown editor |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown viewer |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| play mp3/m4a    |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| play ogg/opus   |  -  |  -  |  -   |  -   | yep   | yep  | `*2` | yep |
+
+* internet explorer 6 to 8 behave the same
+* firefox 52 and chrome 49 are the last winxp versions
+* `*1` only public folders (login session is dropped) and no history / back-button
+* `*2` using a wasm decoder which can sometimes get stuck and consumes a bit more power
+
+quick summary of more eccentric web-browsers trying to view a directory index:
+
+| browser | will it blend |
+| ------- | ------------- |
+| **safari** (14.0.3/macos) | is chrome with janky wasm, so playing opus can deadlock the javascript engine |
+| **safari** (14.0.1/iOS)   | same as macos, except it recovers from the deadlocks if you poke it a bit |
+| **links** (2.21/macports) | can browse, login, upload/mkdir/msg |
+| **lynx** (2.8.9/macports) | can browse, login, upload/mkdir/msg |
+| **w3m** (0.5.3/macports)  | can browse, login, upload at 100kB/s, mkdir/msg |
+| **netsurf** (3.10/arch)   | is basically ie6 with much better css (javascript has almost no effect) | 
+| **ie4** and **netscape** 4.0  | can browse (text is yellow on white), upload with `?b=u` |
+| **SerenityOS** (22d13d8)  | hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying |
 
 
 # client examples
@@ -49,25 +429,84 @@ summary: it works! you can use it! (but technically not even close to beta)
   * `await fetch('https://127.0.0.1:3923/', {method:"PUT", body: JSON.stringify(foo)});`
   * `var xhr = new XMLHttpRequest(); xhr.open('POST', 'https://127.0.0.1:3923/msgs?raw'); xhr.send('foo');`
 
+* curl/wget: upload some files (post=file, chunk=stdin)
+  * `post(){ curl -b cppwd=wark http://127.0.0.1:3923/ -F act=bput -F f=@"$1";}`  
+    `post movie.mkv`
+  * `post(){ wget --header='Cookie: cppwd=wark' http://127.0.0.1:3923/?raw --post-file="$1" -O-;}`  
+    `post movie.mkv`
+  * `chunk(){ curl -b cppwd=wark http://127.0.0.1:3923/ -T-;}`  
+    `chunk <movie.mkv`
+
 * FUSE: mount a copyparty server as a local filesystem
   * cross-platform python client available in [./bin/](bin/)
   * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
 
+* sharex (screenshot utility): see [./contrib/sharex.sxcu](contrib/#sharexsxcu)
+
+copyparty returns a truncated sha512sum of your PUT/POST as base64; you can generate the same checksum locally to verify uplaods:
+
+    b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|tr '+/' '-_'|head -c44;}
+    b512 <movie.mkv
+
+
+# up2k
+
+quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+* the up2k client splits a file into an "optimal" number of chunks
+  * 1 MiB each, unless that becomes more than 256 chunks
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
+* client posts the list of hashes, filename, size, last-modified
+* server creates the `wark`, an identifier for this upload
+  * `sha512( salt + filesize + chunk_hashes )`
+  * and a sparse file is created for the chunks to drop into
+* client uploads each chunk
+  * header entries for the chunk-hash and wark
+  * server writes chunks into place based on the hash
+* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+
 
 # dependencies
 
-* `jinja2`
-  * pulls in `markupsafe` as of v2.7; use jinja 2.6 on py3.2
+* `jinja2` (is built into the SFX)
 
-optional, enables thumbnails:
+
+## optional dependencies
+
+enable music tags:
+* either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
+* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
+
+enable image thumbnails:
 * `Pillow` (requires py2.7 or py3.5+)
 
+enable video thumbnails:
+* `ffmpeg` and `ffprobe` somewhere in `$PATH`
+
+enable reading HEIF pictures:
+* `pyheif-pillow-opener` (requires Linux or a C compiler)
+
+enable reading AVIF pictures:
+* `pillow-avif-plugin`
+
+
+## install recommended deps
+```
+python -m pip install --user -U jinja2 mutagen Pillow
+```
+
+
+## optional gpl stuff
+
+some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
+
+these are standalone programs and will never be imported / evaluated by copyparty
+
 
 # sfx
 
-currently there are two self-contained binaries:
-* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust
-* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta
+currently there are two self-contained "binaries":
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere, **recommended**
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos, kinda deprecated
 
 launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
 
@@ -87,34 +526,64 @@ the features you can opt to drop are
 
 for the `re`pack to work, first run one of the sfx'es once to unpack it
 
+**note:** you can also just download and run [scripts/copyparty-repack.sh](scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a `no-ogv no-cm` repack; works on linux/macos (and windows with msys2 or WSL)
+
 
 # install on android
 
 install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
 ```sh
-apt update && apt -y full-upgrade && termux-setup-storage && apt -y install curl && cd && curl -L https://github.com/9001/copyparty/raw/master/scripts/copyparty-android.sh > copyparty-android.sh && chmod 755 copyparty-android.sh && ./copyparty-android.sh -h
+apt update && apt -y full-upgrade && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install -U copyparty
 echo $?
 ```
 
-after the initial setup (and restarting bash), you can launch copyparty at any time by running "copyparty" in Termux
+after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
 
 
-# dev env setup
+# building
+
+## dev env setup
+
+mostly optional; if you need a working env for vscode or similar
 
 ```sh
 python3 -m venv .venv
 . .venv/bin/activate
-pip install jinja2  # mandatory deps
-pip install Pillow  # thumbnail deps
+pip install jinja2  # mandatory
+pip install mutagen  # audio metadata
+pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
 pip install black bandit pylint flake8  # vscode tooling
 ```
 
 
-# how to release
+## just the sfx
+
+unless you need to modify something in the web-dependencies, it's faster to grab those from a previous release:
+
+```sh
+rm -rf copyparty/web/deps
+curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
+python3 x.py -h
+rm x.py
+mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/
+```
+
+then build the sfx using any of the following examples:
+
+```sh
+./scripts/make-sfx.sh  # both python and sh editions
+./scripts/make-sfx.sh no-sh gz  # just python with gzip
+```
+
+
+## complete release
+
+also builds the sfx so disregard the sfx section above
 
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies
+* `git tag v1.2.3 && git push origin --tags`
 * create github release with `make-tgz-release.sh`
 * upload to pypi with `make-pypi-release.(sh|bat)`
 * create sfx with `make-sfx.sh`
@@ -124,13 +593,25 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
-* up2k handle filename too long
-* up2k fails on empty files? alert then stuck
-* drop onto folders
-* look into android thumbnail cache file format
-* support pillow-simd
+* readme.md as epilogue
+* single sha512 across all up2k chunks? maybe
+* reduce up2k roundtrips
+  * start from a chunk index and just go
+  * terminate client on bad data
+
+discarded ideas
+
+* separate sqlite table per tag
+  * performance fixed by skipping some indexes (`+mt.k`)
+* audio fingerprinting
+  * only makes sense if there can be a wasm client and that doesn't exist yet (except for olaf which is agpl hence counts as not existing)
+* `os.copy_file_range` for up2k cloning
+  * almost never hit this path anyways
+* up2k partials ui
+  * feels like there isn't much point
 * cache sha512 chunks on client
-* symlink existing files on upload
+  * too dangerous
 * comment field
-* figure out the deal with pixel3a not being connectable as hotspot
-  * pixel3a having unpredictable 3sec latency in general :||||
+  * nah
+* look into android thumbnail cache file format
+  * absolutely not

bin/README.md

@@ -1,4 +1,4 @@
-# copyparty-fuse.py
+# [`copyparty-fuse.py`](copyparty-fuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
 * **supports Linux** -- expect `117 MiB/s` sequential read
@@ -29,8 +29,35 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse🅱️.py
+# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
 * mount a copyparty server as a local filesystem (read-only)
 * does the same thing except more correct, `samba` approves
 * **supports Linux** -- expect `18 MiB/s` (wait what)
 * **supports Macos** -- probably
+
+
+
+# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
+* pretend this doesn't exist
+
+
+
+# [`mtag/`](mtag/)
+* standalone programs which perform misc. file analysis
+* copyparty can Popen programs like these during file indexing to collect additional metadata
+
+
+# [`dbtool.py`](dbtool.py)
+upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
+
+for that example (upgrading to v0.11.20), first launch the new version of copyparty like usual, let it make a backup of the old db and rebuild the new db until the point where it starts running mtp (colored messages as it adds the mtp tags), that's when you hit CTRL-C and patch in the old mtp tags from the old db instead
+
+so assuming you have `-mtp` parsers to provide the tags `key` and `.bpm`:
+
+```
+cd /mnt/nas/music/.hist
+~/src/copyparty/bin/dbtool.py -ls up2k.db
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -cmp
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy key
+~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
+```

bin/copyparty-fuse.py

@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
 mount a copyparty server (local or remote) as a filesystem
 
 usage:
-  python copyparty-fuse.py ./music http://192.168.1.69:3923/
+  python copyparty-fuse.py http://192.168.1.69:3923/  ./music
 
 dependencies:
   python3 -m pip install --user fusepy
@@ -20,6 +20,10 @@ dependencies:
   + on Macos: https://osxfuse.github.io/
   + on Windows: https://github.com/billziss-gh/winfsp/releases/latest
 
+note:
+  you probably want to run this on windows clients:
+  https://github.com/9001/copyparty/blob/master/contrib/explorer-nothumbs-nofoldertypes.reg
+
 get server cert:
   awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
 """
@@ -29,6 +33,7 @@ import re
 import os
 import sys
 import time
+import json
 import stat
 import errno
 import struct
@@ -49,6 +54,12 @@ MACOS = platform.system() == "Darwin"
 info = log = dbg = None
 
 
+print("{} v{} @ {}".format(
+    platform.python_implementation(),
+    ".".join([str(x) for x in sys.version_info]),
+    sys.executable))
+
+
 try:
     from fuse import FUSE, FuseOSError, Operations
 except:
@@ -100,7 +111,7 @@ def rice_tid():
 
 
 def fancy_log(msg):
-    print("{} {}\n".format(rice_tid(), msg), end="")
+    print("{:10.6f} {} {}\n".format(time.time() % 900, rice_tid(), msg), end="")
 
 
 def null_log(msg):
@@ -159,7 +170,7 @@ class RecentLog(object):
         thr.start()
 
     def put(self, msg):
-        msg = "{} {}\n".format(rice_tid(), msg)
+        msg = "{:10.6f} {} {}\n".format(time.time() % 900, rice_tid(), msg)
         if self.f:
             fmsg = " ".join([datetime.utcnow().strftime("%H%M%S.%f"), str(msg)])
             self.f.write(fmsg.encode("utf-8"))
@@ -319,7 +330,7 @@ class Gateway(object):
         if bad_good:
             path = dewin(path)
 
-        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
+        web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
         r = self.sendreq("GET", web_path)
         if r.status != 200:
             self.closeconn()
@@ -330,12 +341,17 @@ class Gateway(object):
             )
             raise FuseOSError(errno.ENOENT)
 
-        if not r.getheader("Content-Type", "").startswith("text/html"):
+        ctype = r.getheader("Content-Type", "")
+        if ctype == "application/json":
+            parser = self.parse_jls
+        elif ctype.startswith("text/html"):
+            parser = self.parse_html
+        else:
             log("listdir on file: {}".format(path))
             raise FuseOSError(errno.ENOENT)
 
         try:
-            return self.parse_html(r)
+            return parser(r)
         except:
             info(repr(path) + "\n" + traceback.format_exc())
             raise
@@ -363,11 +379,34 @@ class Gateway(object):
 
         return r.read()
 
+    def parse_jls(self, datasrc):
+        rsp = b""
+        while True:
+            buf = datasrc.read(1024 * 32)
+            if not buf:
+                break
+
+            rsp += buf
+
+        rsp = json.loads(rsp.decode("utf-8"))
+        ret = []
+        for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
+            for n in nodes:
+                fname = unquote(n["href"]).rstrip(b"/")
+                fname = fname.decode("wtf-8")
+                if bad_good:
+                    fname = enwin(fname)
+
+                fun = self.stat_dir if is_dir else self.stat_file
+                ret.append([fname, fun(n["ts"], n["sz"]), 0])
+
+        return ret
+
     def parse_html(self, datasrc):
         ret = []
         remainder = b""
         ptn = re.compile(
-            r'^<tr><td>(-|DIR)</td><td><a[^>]* href="([^"]+)"[^>]*>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$'
+            r'^<tr><td>(-|DIR|<a [^<]+</a>)</td><td><a[^>]* href="([^"]+)"[^>]*>([^<]+)</a></td><td>([^<]+)</td><td>[^<]+</td><td>([^<]+)</td></tr>$'
         )
 
         while True:
@@ -405,7 +444,7 @@ class Gateway(object):
                     info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
                     # python cannot strptime(1959-01-01) on windows
 
-                if ftype == "-":
+                if ftype != "DIR":
                     ret.append([fname, self.stat_file(ts, sz), 0])
                 else:
                     ret.append([fname, self.stat_dir(ts, sz), 0])
@@ -658,8 +697,18 @@ class CPPF(Operations):
 
         else:
             if get2 - get1 <= 1024 * 1024:
-                h_ofs = get1 - 256 * 1024
-                h_end = get2 + 1024 * 1024
+                # unless the request is for the last n bytes of the file,
+                # grow the start to cache some stuff around the range
+                if get2 < file_sz - 1:
+                    h_ofs = get1 - 1024 * 256
+                else:
+                    h_ofs = get1 - 1024 * 32
+
+                # likewise grow the end unless start is 0
+                if get1 > 0:
+                    h_end = get2 + 1024 * 1024
+                else:
+                    h_end = get2 + 1024 * 64
             else:
                 # big enough, doesn't need pads
                 h_ofs = get1
@@ -705,6 +754,7 @@ class CPPF(Operations):
             self.dircache.append(cn)
             self.clean_dircache()
 
+        # import pprint; pprint.pprint(ret)
         return ret
 
     def readdir(self, path, fh=None):
@@ -802,7 +852,11 @@ class CPPF(Operations):
                 # dbg("=" + repr(cache_stat))
                 return cache_stat
 
-        info("=ENOENT ({})".format(hexler(path)))
+        fun = info
+        if MACOS and path.split("/")[-1].startswith("._"):
+            fun = dbg
+
+        fun("=ENOENT ({})".format(hexler(path)))
         raise FuseOSError(errno.ENOENT)
 
     access = None
@@ -906,6 +960,7 @@ class TheArgparseFormatter(
 
 def main():
     global info, log, dbg
+    time.strptime("19970815", "%Y%m%d")  # python#7980
 
     # filecache helps for reads that are ~64k or smaller;
     #   linux generally does 128k so the cache is a slowdown,
@@ -959,8 +1014,14 @@ def main():
         log = null_log
         dbg = null_log
 
+    if ar.a and ar.a.startswith("$"):
+        fn = ar.a[1:]
+        log("reading password from file [{}]".format(fn))
+        with open(fn, "rb") as f:
+            ar.a = f.read().decode("utf-8").strip()
+
     if WINDOWS:
-        os.system("")
+        os.system("rem")
 
         for ch in '<>:"\\|?*':
             # microsoft maps illegal characters to f0xx

bin/copyparty-fuseb.py

@@ -567,6 +567,8 @@ class CPPF(Fuse):
 
 
 def main():
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+
     server = CPPF()
     server.parser.add_option(mountopt="url", metavar="BASE_URL", default=None)
     server.parse(values=server, errex=1)

bin/dbtool.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import shutil
+import sqlite3
+import argparse
+
+DB_VER1 = 3
+DB_VER2 = 4
+
+
+def die(msg):
+    print("\033[31m\n" + msg + "\n\033[0m")
+    sys.exit(1)
+
+
+def read_ver(db):
+    for tab in ["ki", "kv"]:
+        try:
+            c = db.execute(r"select v from {} where k = 'sver'".format(tab))
+        except:
+            continue
+
+        rows = c.fetchall()
+        if rows:
+            return int(rows[0][0])
+
+    return "corrupt"
+
+
+def ls(db):
+    nfiles = next(db.execute("select count(w) from up"))[0]
+    ntags = next(db.execute("select count(w) from mt"))[0]
+    print(f"{nfiles} files")
+    print(f"{ntags} tags\n")
+
+    print("number of occurences for each tag,")
+    print(" 'x' = file has no tags")
+    print(" 't:mtp' = the mtp flag (file not mtp processed yet)")
+    print()
+    for k, nk in db.execute("select k, count(k) from mt group by k order by k"):
+        print(f"{nk:9} {k}")
+
+
+def compare(n1, d1, n2, d2, verbose):
+    nt = next(d1.execute("select count(w) from up"))[0]
+    n = 0
+    miss = 0
+    for w1, rd, fn in d1.execute("select w, rd, fn from up"):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} files in {n1} against {n2}\033[0m"
+            print(m)
+
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select w from up where rd = ? and fn = ?"
+        hit = d2.execute(q, (rd, fn)).fetchone()
+        if not hit:
+            miss += 1
+            if verbose:
+                print(f"file in {n1} missing in {n2}: [{w1}] {rd}/{fn}")
+
+    print(f" {miss} files in {n1} missing in {n2}\n")
+
+    nt = next(d1.execute("select count(w) from mt"))[0]
+    n = 0
+    miss = {}
+    nmiss = 0
+    for w1, k, v in d1.execute("select * from mt"):
+
+        n += 1
+        if n % 100_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
+            print(m)
+
+        q = "select rd, fn from up where substr(w,1,16) = ?"
+        rd, fn = d1.execute(q, (w1,)).fetchone()
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
+        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if w2:
+            w2 = w2[0]
+
+        v2 = None
+        if w2:
+            v2 = d2.execute(
+                "select v from mt where w = ? and +k = ?", (w2, k)
+            ).fetchone()
+            if v2:
+                v2 = v2[0]
+
+        # if v != v2 and v2 and k in [".bpm", "key"] and n2 == "src":
+        #    print(f"{w} [{rd}/{fn}] {k} = [{v}] / [{v2}]")
+
+        if v2 is not None:
+            if k.startswith("."):
+                try:
+                    diff = abs(float(v) - float(v2))
+                    if diff > float(v) / 0.9:
+                        v2 = None
+                    else:
+                        v2 = v
+                except:
+                    pass
+
+            if v != v2:
+                v2 = None
+
+        if v2 is None:
+            nmiss += 1
+            try:
+                miss[k] += 1
+            except:
+                miss[k] = 1
+
+            if verbose:
+                print(f"missing in {n2}: [{w1}] [{rd}/{fn}] {k} = {v}")
+
+    for k, v in sorted(miss.items()):
+        if v:
+            print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
+
+    print(f"in total, {nmiss} missing tags in {n2}\n")
+
+
+def copy_mtp(d1, d2, tag, rm):
+    nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
+    n = 0
+    ndone = 0
+    for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
+            print(m)
+
+        q = "select rd, fn from up where substr(w,1,16) = ?"
+        rd, fn = d1.execute(q, (w1,)).fetchone()
+        if rd.split("/", 1)[0] == ".hist":
+            continue
+
+        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
+        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if not w2:
+            continue
+
+        w2 = w2[0]
+        hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
+        if hit:
+            hit = hit[0]
+
+        if hit != v:
+            ndone += 1
+            if hit is not None:
+                d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
+
+            d2.execute("insert into mt values (?,?,?)", (w2, k, v))
+            if rm:
+                d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
+
+    d2.commit()
+    print(f"copied {ndone} {tag} tags over")
+
+
+def main():
+    os.system("")
+    print()
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db", help="database to work on")
+    ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
+
+    ap2 = ap.add_argument_group("informational / read-only stuff")
+    ap2.add_argument("-v", action="store_true", help="verbose")
+    ap2.add_argument("-ls", action="store_true", help="list summary for db")
+    ap2.add_argument("-cmp", action="store_true", help="compare databases")
+
+    ap2 = ap.add_argument_group("options which modify target db")
+    ap2.add_argument("-copy", metavar="TAG", type=str, help="mtp tag to copy over")
+    ap2.add_argument(
+        "-rm-mtp-flag",
+        action="store_true",
+        help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
+    )
+    ap2.add_argument("-vac", action="store_true", help="optimize DB")
+
+    ar = ap.parse_args()
+
+    for v in [ar.db, ar.src]:
+        if v and not os.path.exists(v):
+            die("database must exist")
+
+    db = sqlite3.connect(ar.db)
+    ds = sqlite3.connect(ar.src) if ar.src else None
+
+    # revert journals
+    for d, p in [[db, ar.db], [ds, ar.src]]:
+        if not d:
+            continue
+
+        pj = "{}-journal".format(p)
+        if not os.path.exists(pj):
+            continue
+
+        d.execute("create table foo (bar int)")
+        d.execute("drop table foo")
+
+    if ar.copy:
+        db.close()
+        shutil.copy2(ar.db, "{}.bak.dbtool.{:x}".format(ar.db, int(time.time())))
+        db = sqlite3.connect(ar.db)
+
+    for d, n in [[ds, "src"], [db, "dst"]]:
+        if not d:
+            continue
+
+        ver = read_ver(d)
+        if ver == "corrupt":
+            die("{} database appears to be corrupt, sorry")
+
+        if ver < DB_VER1 or ver > DB_VER2:
+            m = f"{n} db is version {ver}, this tool only supports versions between {DB_VER1} and {DB_VER2}, please upgrade it with copyparty first"
+            die(m)
+
+    if ar.ls:
+        ls(db)
+
+    if ar.cmp:
+        if not ds:
+            die("need src db to compare against")
+
+        compare("src", ds, "dst", db, ar.v)
+        compare("dst", db, "src", ds, ar.v)
+
+    if ar.copy:
+        copy_mtp(ds, db, ar.copy, ar.rm_mtp_flag)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/README.md

@@ -0,0 +1,34 @@
+standalone programs which take an audio file as argument
+
+some of these rely on libraries which are not MIT-compatible
+
+* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
+* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
+
+
+# dependencies
+
+run [`install-deps.sh`](install-deps.sh) to build/install most dependencies required by these programs (supports windows/linux/macos)
+
+*alternatively* (or preferably) use packages from your distro instead, then you'll need at least these:
+
+* from distro: `numpy vamp-plugin-sdk beatroot-vamp mixxx-keyfinder ffmpeg`
+* from pypy: `keyfinder vamp`
+
+
+# usage from copyparty
+
+`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
+
+* `f,` makes the detected value replace any existing values
+* the `.` in `.bpm` indicates numeric value
+* assumes the python files are in the folder you're launching copyparty from, replace the filename with a relative/absolute path if that's not the case
+* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
+
+
+## usage with volume-flags
+
+instead of affecting all volumes, you can set the options for just one volume like so:
+```
+copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
+```

bin/mtag/audio-bpm.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import vamp
+import tempfile
+import numpy as np
+import subprocess as sp
+
+from copyparty.util import fsenc
+
+"""
+dep: vamp
+dep: beatroot-vamp
+dep: ffmpeg
+"""
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-ss", "13",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-ac", "1",
+        "-ar", "22050",
+        "-t", "300",
+        "-f", "f32le",
+        tf
+    ])
+    # fmt: on
+
+    with open(tf, "rb") as f:
+        d = np.fromfile(f, dtype=np.float32)
+        try:
+            # 98% accuracy on jcore
+            c = vamp.collect(d, 22050, "beatroot-vamp:beatroot")
+            cl = c["list"]
+        except:
+            # fallback; 73% accuracy
+            plug = "vamp-example-plugins:fixedtempo"
+            c = vamp.collect(d, 22050, plug, parameters={"maxdflen": 40})
+            print(c["list"][0]["label"].split(" ")[0])
+            return
+
+        # throws if detection failed:
+        bpm = float(cl[-1]["timestamp"] - cl[1]["timestamp"])
+        bpm = round(60 * ((len(cl) - 1) / bpm), 2)
+        print(f"{bpm:.2f}")
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import sys
+import keyfinder
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+
+note: cannot fsenc
+"""
+
+
+try:
+    print(keyfinder.key(sys.argv[1]).camelot())
+except:
+    pass

bin/mtag/exe.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+
+import sys
+import time
+import json
+import pefile
+
+"""
+retrieve exe info,
+example for multivalue providers
+"""
+
+
+def unk(v):
+    return "unk({:04x})".format(v)
+
+
+class PE2(pefile.PE):
+    def __init__(self, *a, **ka):
+        for k in [
+            # -- parse_data_directories:
+            "parse_import_directory",
+            "parse_export_directory",
+            # "parse_resources_directory",
+            "parse_debug_directory",
+            "parse_relocations_directory",
+            "parse_directory_tls",
+            "parse_directory_load_config",
+            "parse_delay_import_directory",
+            "parse_directory_bound_imports",
+            # -- full_load:
+            "parse_rich_header",
+        ]:
+            setattr(self, k, self.noop)
+
+        super(PE2, self).__init__(*a, **ka)
+
+    def noop(*a, **ka):
+        pass
+
+
+try:
+    pe = PE2(sys.argv[1], fast_load=False)
+except:
+    sys.exit(0)
+
+arch = pe.FILE_HEADER.Machine
+if arch == 0x14C:
+    arch = "x86"
+elif arch == 0x8664:
+    arch = "x64"
+else:
+    arch = unk(arch)
+
+try:
+    buildtime = time.gmtime(pe.FILE_HEADER.TimeDateStamp)
+    buildtime = time.strftime("%Y-%m-%d_%H:%M:%S", buildtime)
+except:
+    buildtime = "invalid"
+
+ui = pe.OPTIONAL_HEADER.Subsystem
+if ui == 2:
+    ui = "GUI"
+elif ui == 3:
+    ui = "cmdline"
+else:
+    ui = unk(ui)
+
+extra = {}
+if hasattr(pe, "FileInfo"):
+    for v1 in pe.FileInfo:
+        for v2 in v1:
+            if v2.name != "StringFileInfo":
+                continue
+
+            for v3 in v2.StringTable:
+                for k, v in v3.entries.items():
+                    v = v.decode("utf-8", "replace").strip()
+                    if not v:
+                        continue
+
+                    if k in [b"FileVersion", b"ProductVersion"]:
+                        extra["ver"] = v
+
+                    if k in [b"OriginalFilename", b"InternalName"]:
+                        extra["orig"] = v
+
+r = {
+    "arch": arch,
+    "built": buildtime,
+    "ui": ui,
+    "cksum": "{:08x}".format(pe.OPTIONAL_HEADER.CheckSum),
+}
+r.update(extra)
+
+print(json.dumps(r, indent=4))

bin/mtag/file-ext.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+
+"""
+example that just prints the file extension
+"""
+
+print(sys.argv[1].split(".")[-1])

bin/mtag/install-deps.sh

@@ -0,0 +1,265 @@
+#!/bin/bash
+set -e
+
+
+# install dependencies for audio-*.py
+#
+# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
+# win64: requires msys2-mingw64 environment
+# macos: requires macports
+#
+# has the following manual dependencies, especially on mac:
+#   https://www.vamp-plugins.org/pack.html
+#
+# installs stuff to the following locations:
+#   ~/pe/
+#   whatever your python uses for --user packages
+#
+# does the following terrible things:
+#   modifies the keyfinder python lib to load the .so in ~/pe
+
+
+linux=1
+
+win=
+[ ! -z "$MSYSTEM" ] || [ -e /msys2.exe ] && {
+	[ "$MSYSTEM" = MINGW64 ] || {
+		echo windows detected, msys2-mingw64 required
+		exit 1
+	}
+	pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	win=1
+	linux=
+}
+
+mac=
+[ $(uname -s) = Darwin ] && {
+	#pybin="$(printf '%s\n' /opt/local/bin/python* | (sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) | (sort -nr || cat) | (sed -E 's/([^ ]*) (.*)/\2\1/' || cat) | grep -E '/(python|pypy)[0-9\.-]*$' | head -n 1)"
+	pybin=/opt/local/bin/python3.9
+	[ -e "$pybin" ] || {
+		echo mac detected, python3 from macports required
+		exit 1
+	}
+	pkgs='ffmpeg python39 py39-wheel'
+	ninst=$(port installed | awk '/^  /{print$1}' | sort | uniq | grep -E '^('"$(echo "$pkgs" | tr ' ' '|')"')$' | wc -l)
+	[ $ninst -eq 3 ] || {
+		sudo port install $pkgs
+	}
+	mac=1
+	linux=
+}
+
+hash -r
+
+[ $mac ] || {
+	command -v python3 && pybin=python3 || pybin=python
+}
+
+$pybin -m pip install --user numpy
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+
+
+need() {
+	command -v $1 >/dev/null || {
+		echo need $1
+		exit 1
+	}
+}
+need cmake
+need ffmpeg
+need $pybin
+#need patchelf
+
+
+td="$(mktemp -d)"
+cln() {
+	rm -rf "$td"
+}
+trap cln EXIT
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	local yolo= ex=
+	[ $1 = "yolo" ] && yolo=1 && ex=k && shift
+	command -v curl >/dev/null && exec curl -${ex}JOL "$@"
+	
+	[ $yolo ] && ex=--no-check-certificate
+	exec wget --trust-server-names $ex "$@"
+}
+export -f dl_files
+
+
+github_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.tarball_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"tarball_url": "/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+gitlab_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.[0].assets.sources[]|select(.format|test("tar.gz")).url' ||
+
+		# fallback to abomination
+		tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	tee links |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+install_keyfinder() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
+	
+	github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
+
+	tar -xf mixxxdj-libkeyfinder-*
+	rm -- *.tar.gz
+	cd mixxxdj-libkeyfinder*
+	
+	h="$HOME"
+	so="lib/libkeyfinder.so"
+	memes=()
+
+	[ $win ] &&
+		so="bin/libkeyfinder.dll" &&
+		h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
+		memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
+	
+	[ $mac ] &&
+		so="lib/libkeyfinder.dylib"
+
+	cmake -DCMAKE_INSTALL_PREFIX="$h/pe/keyfinder" "${memes[@]}" -S . -B build
+	cmake --build build --parallel $(nproc || echo 4)
+	cmake --install build
+
+	libpath="$h/pe/keyfinder/$so"
+	[ $linux ] && [ ! -e "$libpath" ] &&
+		so=lib64/libkeyfinder.so
+	
+	libpath="$h/pe/keyfinder/$so"
+	[ -e "$libpath" ] || {
+		echo "so not found at $sop"
+		exit 1
+	}
+	
+	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
+	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
+	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
+	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
+	$pybin -m pip install --user keyfinder
+
+	pypath="$($pybin -c 'import keyfinder; print(keyfinder.__file__)')"
+	for pyso in "${pypath%/*}"/*.so; do
+		[ -e "$pyso" ] || break
+		patchelf --set-rpath "${libpath%/*}" "$pyso" ||
+			echo "WARNING: patchelf failed (only fatal on musl-based distros)"
+	done
+	
+	mv "$pypath"{,.bak}
+	(
+		printf 'import ctypes\nctypes.cdll.LoadLibrary("%s")\n' "$libpath"
+		cat "$pypath.bak"
+	) >"$pypath"
+
+	echo
+	echo libkeyfinder successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+have_beatroot() {
+	$pybin -c 'import vampyhost, sys; plugs = vampyhost.list_plugins(); sys.exit(0 if "beatroot-vamp:beatroot" in plugs else 1)'
+}
+
+
+install_vamp() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	
+	$pybin -m pip install --user vamp
+
+	have_beatroot || {
+		printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
+		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
+		sha512sum -c <(
+			echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874  -"
+		) <beatroot-vamp-v1.0.tar.gz
+		tar -xf beatroot-vamp-v1.0.tar.gz 
+		cd beatroot-vamp-v1.0
+		make -f Makefile.linux -j4
+		# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
+		mkdir ~/vamp
+		cp -pv beatroot-vamp.* ~/vamp/
+	}
+	
+	have_beatroot &&
+		printf '\033[32mfound the vamp beatroot plugin, nice\033[0m\n' ||
+		printf '\033[31mWARNING: could not find the vamp beatroot plugin, please install it for optimal results\033[0m\n'
+}
+
+
+# not in use because it kinda segfaults, also no windows support
+install_soundtouch() {
+	gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
+	
+	tar -xvf soundtouch-*
+	rm -- *.tar.gz
+	cd soundtouch-*
+	
+	# https://github.com/jrising/pysoundtouch
+	./bootstrap
+	./configure --enable-integer-samples CXXFLAGS="-fPIC" --prefix="$HOME/pe/soundtouch"
+	make -j$(nproc || echo 4)
+	make install
+	
+	CFLAGS=-I$HOME/pe/soundtouch/include/ \
+	LDFLAGS=-L$HOME/pe/soundtouch/lib \
+	$pybin -m pip install --user git+https://github.com/snowxmas/pysoundtouch.git
+	
+	pypath="$($pybin -c 'import importlib; print(importlib.util.find_spec("soundtouch").origin)')"
+	libpath="$(echo "$HOME/pe/soundtouch/lib/")"
+	patchelf --set-rpath "$libpath" "$pypath"
+
+	echo
+	echo soundtouch successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+[ "$1" = keyfinder ] && { install_keyfinder; exit $?; }
+[ "$1" = soundtouch ] && { install_soundtouch; exit $?; }
+[ "$1" = vamp ] && { install_vamp; exit $?; }
+
+echo no args provided, installing keyfinder and vamp
+install_keyfinder
+install_vamp

bin/mtag/sleep.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+
+import time
+import random
+
+v = random.random() * 6
+time.sleep(v)
+print(f"{v:.2f}")

contrib/README.md

@@ -9,7 +9,29 @@
 * assumes the webserver and copyparty is running on the same server/IP
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
+### [`sharex.sxcu`](sharex.sxcu)
+* sharex config file to upload screenshots and grab the URL
+* `RequestURL`: full URL to the target folder
+* `pw`: password (remove the `pw` line if anon-write)
+
+however if your copyparty is behind a reverse-proxy, you may want to use [`sharex-html.sxcu`](sharex-html.sxcu) instead:
+* `RequestURL`: full URL to the target folder
+* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
+* `pw`: password (remove `Parameters` if anon-write)
+
+### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
+* disables thumbnails and folder-type detection in windows explorer
+* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+
+### [`cfssl.sh`](cfssl.sh)
+* creates CA and server certificates using cfssl
+* give a 3rd argument to install it to your copyparty config
+
 # OS integration
 init-scripts to start copyparty as a service
 * [`systemd/copyparty.service`](systemd/copyparty.service)
 * [`openrc/copyparty`](openrc/copyparty)
+
+# Reverse-proxy
+copyparty has basic support for running behind another webserver
+* [`nginx/copyparty.conf`](nginx/copyparty.conf)

contrib/cfssl.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e
+
+# ca-name and server-name
+ca_name="$1"
+srv_name="$2"
+
+[ -z "$srv_name" ] && {
+	echo "need arg 1: ca name"
+	echo "need arg 2: server name"
+	exit 1
+}
+
+
+gen_ca() {
+	(tee /dev/stderr <<EOF
+{"CN": "$ca_name ca",
+"CA": {"expiry":"87600h", "pathlen":0},
+"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name ca"}]}
+EOF
+	)|
+	cfssl gencert -initca - |
+	cfssljson -bare ca
+	
+	mv ca-key.pem ca.key
+	rm ca.csr
+}
+
+
+gen_srv() {
+	(tee /dev/stderr <<EOF
+{"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name - $srv_name"}]}
+EOF
+	)|
+	cfssl gencert -ca ca.pem -ca-key ca.key \
+		-profile=www -hostname="$srv_name.$ca_name" - |
+	cfssljson -bare "$srv_name"
+
+	mv "$srv_name-key.pem" "$srv_name.key"
+	rm "$srv_name.csr"
+}
+
+
+# create ca if not exist
+[ -e ca.key ] ||
+	gen_ca
+
+# always create server cert
+gen_srv
+
+
+# dump cert info
+show() {
+	openssl x509 -text -noout -in $1 |
+	awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
+}
+show ca.pem
+show "$srv_name.pem"
+
+
+# write cert into copyparty config
+[ -z "$3" ] || {
+	mkdir -p ~/.config/copyparty
+	cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
+}
+
+
+# rm *.key *.pem
+# cfssl print-defaults config
+# cfssl print-defaults csr

contrib/explorer-nothumbs-nofoldertypes.reg

@@ -0,0 +1,31 @@
+Windows Registry Editor Version 5.00
+
+; this will do 3 things, all optional:
+;  1) disable thumbnails
+;  2) delete all existing folder type settings/detections
+;  3) disable folder type detection (force default columns)
+;
+; this makes the file explorer way faster,
+; especially on slow/networked locations
+
+
+; =====================================================================
+; 1) disable thumbnails
+
+[HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced]
+"IconsOnly"=dword:00000001
+
+
+; =====================================================================
+; 2) delete all existing folder type settings/detections
+
+[-HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\Bags]
+
+[-HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\BagMRU]
+
+
+; =====================================================================
+; 3) disable folder type detection
+
+[HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\Bags\AllFolders\Shell]
+"FolderType"="NotSpecified"

contrib/nginx/copyparty.conf

@@ -0,0 +1,31 @@
+# when running copyparty behind a reverse-proxy,
+# make sure that copyparty allows at least as many clients as the proxy does,
+# so run copyparty with -nc 512 if your nginx has the default limits
+# (worker_processes 1, worker_connections 512)
+
+upstream cpp {
+	server 127.0.0.1:3923;
+	keepalive 120;
+}
+server {
+	listen 443 ssl;
+	listen [::]:443 ssl;
+
+	server_name fs.example.com;
+	
+	location / {
+		proxy_pass http://cpp;
+		proxy_redirect off;
+		# disable buffering (next 4 lines)
+		proxy_http_version 1.1;
+		client_max_body_size 0;
+		proxy_buffering off;
+		proxy_request_buffering off;
+
+		proxy_set_header   Host              $host;
+		proxy_set_header   X-Real-IP         $remote_addr;
+		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+		proxy_set_header   X-Forwarded-Proto $scheme;
+		proxy_set_header   Connection        "Keep-Alive";
+	}
+}

contrib/sharex-html.sxcu

@@ -0,0 +1,19 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty-html",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark"
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "RegexList": [
+    "bytes // <a href=\"/([^\"]+)\""
+  ],
+  "URL": "http://127.0.0.1:3923/$regex:1|1$"
+}

contrib/sharex.sxcu

@@ -0,0 +1,17 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark",
+    "j": null
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "URL": "$json:files[0].url$"
+}

contrib/systemd/copyparty.service

@@ -12,7 +12,7 @@
 Description=copyparty file server
 
 [Service]
-ExecStart=/usr/bin/python /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 [Install]

copyparty/__init__.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import platform
+import time
 import sys
 import os
 
@@ -16,12 +17,18 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 
 class EnvParams(object):
     def __init__(self):
+        self.t0 = time.time()
         self.mod = os.path.dirname(os.path.realpath(__file__))
+        if self.mod.endswith("__init__"):
+            self.mod = os.path.dirname(self.mod)
+
         if sys.platform == "win32":
             self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
         elif sys.platform == "darwin":

copyparty/__main__.py

@@ -8,17 +8,28 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
+import re
 import os
+import sys
+import time
 import shutil
 import filecmp
 import locale
 import argparse
+import threading
+import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, VT100
+from .__init__ import E, WINDOWS, VT100, PY2
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc
+from .util import py_desc, align_tab, IMPLICATIONS, alltrace
+
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
 
 
 class RiceFormatter(argparse.HelpFormatter):
@@ -44,6 +55,16 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+class Dodge11874(RiceFormatter):
+    def __init__(self, *args, **kwargs):
+        kwargs["width"] = 9003
+        super(Dodge11874, self).__init__(*args, **kwargs)
+
+
+def warn(msg):
+    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+
+
 def ensure_locale():
     for x in [
         "en_US.UTF-8",
@@ -84,9 +105,244 @@ def ensure_cert():
     # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
 
 
-def main():
+def configure_ssl_ver(al):
+    def terse_sslver(txt):
+        txt = txt.lower()
+        for c in ["_", "v", "."]:
+            txt = txt.replace(c, "")
+
+        return txt.replace("tls10", "tls1")
+
+    # oh man i love openssl
+    # check this out
+    # hold my beer
+    ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
+    sslver = terse_sslver(al.ssl_ver).split(",")
+    flags = [k for k in ssl.__dict__ if ptn.match(k)]
+    # SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
+    if "help" in sslver:
+        avail = [terse_sslver(x[6:]) for x in flags]
+        avail = " ".join(sorted(avail) + ["all"])
+        print("\navailable ssl/tls versions:\n  " + avail)
+        sys.exit(0)
+
+    al.ssl_flags_en = 0
+    al.ssl_flags_de = 0
+    for flag in sorted(flags):
+        ver = terse_sslver(flag[6:])
+        num = getattr(ssl, flag)
+        if ver in sslver:
+            al.ssl_flags_en |= num
+        else:
+            al.ssl_flags_de |= num
+
+    if sslver == ["all"]:
+        x = al.ssl_flags_en
+        al.ssl_flags_en = al.ssl_flags_de
+        al.ssl_flags_de = x
+
+    for k in ["ssl_flags_en", "ssl_flags_de"]:
+        num = getattr(al, k)
+        print("{}: {:8x} ({})".format(k, num, num))
+
+    # think i need that beer now
+
+
+def configure_ssl_ciphers(al):
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    if al.ssl_ver:
+        ctx.options &= ~al.ssl_flags_en
+        ctx.options |= al.ssl_flags_de
+
+    is_help = al.ciphers == "help"
+
+    if al.ciphers and not is_help:
+        try:
+            ctx.set_ciphers(al.ciphers)
+        except:
+            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+
+    if not hasattr(ctx, "get_ciphers"):
+        print("cannot read cipher list: openssl or python too old")
+    else:
+        ciphers = [x["description"] for x in ctx.get_ciphers()]
+        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+
+    if is_help:
+        sys.exit(0)
+
+
+def sighandler(sig=None, frame=None):
+    msg = [""] * 5
+    for th in threading.enumerate():
+        msg.append(str(th))
+        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
+
+    msg.append("\n")
+    print("\n".join(msg))
+
+
+def stackmon(fp, ival):
+    ctr = 0
+    while True:
+        ctr += 1
+        time.sleep(ival)
+        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
+        with open(fp, "wb") as f:
+            f.write(st.encode("utf-8", "replace"))
+
+
+def run_argparse(argv, formatter):
+    ap = argparse.ArgumentParser(
+        formatter_class=formatter,
+        prog="copyparty",
+        description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
+        epilog=dedent(
+            """
+            -a takes username:password,
+            -v takes src:dst:permset:permset:cflag:cflag:...
+               where "permset" is accesslevel followed by username (no separator)
+               and "cflag" is config flags to set on this volume
+            
+            list of cflags:
+              "cnodupe" rejects existing files (instead of symlinking them)
+              "ce2d" sets -e2d (all -e2* args can be set using ce2* cflags)
+              "cd2t" disables metadata collection, overrides -e2t*
+              "cd2d" disables all database stuff, overrides -e2*
+
+            example:\033[35m
+              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
+              mount current directory at "/" with
+               * r (read-only) for everyone
+               * a (read+write) for ed
+              mount ../inc at "/dump" with
+               * w (write-only) for everyone
+               * a (read+write) for ed
+               * reject duplicate files  \033[0m
+            
+            if no accounts or volumes are configured,
+            current folder will be read/write for everyone
+
+            consider the config file for more flexible account/volume management,
+            including dynamic reload at runtime (and being more readable w)
+
+            values for --urlform:
+              "stash" dumps the data to file and returns length + checksum
+              "save,get" dumps to file and returns the page like a GET
+              "print,get" prints the data in the log and returns GET
+              (leave out the ",get" to return an error instead)
+
+            values for --ls:
+              "USR" is a user to browse as; * is anonymous, ** is all users
+              "VOL" is a single volume to scan, default is * (all vols)
+              "FLAG" is flags;
+                "v" in addition to realpaths, print usernames and vpaths
+                "ln" only prints symlinks leaving the volume mountpoint
+                "p" exits 1 if any such symlinks are found
+                "r" resumes startup after the listing
+            examples:
+              --ls '**'          # list all files which are possible to read
+              --ls '**,*,ln'     # check for dangerous symlinks
+              --ls '**,*,ln,p,r' # check, then start normally if safe
+            \033[0m
+            """
+        ),
+    )
+    # fmt: off
+    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
+    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
+    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
+    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
+    ap.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
+    ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
+
+    ap2 = ap.add_argument_group('network options')
+    ap2.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
+    
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ssl/tls ciphers; [help] shows available ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+
+    ap2 = ap.add_argument_group('opt-outs')
+    ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
+    ap2.add_argument("-nih", action="store_true", help="no info hostname")
+    ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+
+    ap2 = ap.add_argument_group('safety options')
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
+    ap2.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+
+    ap2 = ap.add_argument_group('logging options')
+    ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
+
+    ap2 = ap.add_argument_group('admin panel options')
+    ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
+    ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
+
+    ap2 = ap.add_argument_group('thumbnail options')
+    ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
+    ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
+    ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
+    ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
+    ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
+    ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
+    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
+    ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
+    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval")
+    ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
+
+    ap2 = ap.add_argument_group('database options')
+    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
+    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
+    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
+    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
+    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
+    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
+    ap2.add_argument("--hist", metavar="PATH", type=str, help="where to store volume state")
+    ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
+    ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
+    ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
+    ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,ac,vc,res,.fps")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
+
+    ap2 = ap.add_argument_group('appearance options')
+    ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
+
+    ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
+    ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
+    ap2.add_argument("--stackmon", metavar="P,S", help="write stacktrace to Path every S second")
+    
+    return ap.parse_args(args=argv[1:])
+    # fmt: on
+
+
+def main(argv=None):
+    time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
-        os.system("")  # enables colors
+        os.system("rem")  # enables colors
+
+    if argv is None:
+        argv = sys.argv
 
     desc = py_desc().replace("[", "\033[1;30m[")
 
@@ -94,52 +350,70 @@ def main():
     print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
-    ensure_cert()
+    if HAVE_SSL:
+        ensure_cert()
 
-    ap = argparse.ArgumentParser(
-        formatter_class=RiceFormatter,
-        prog="copyparty",
-        description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
-        epilog=dedent(
-            """
-            -a takes username:password,
-            -v takes src:dst:permset:permset:... where "permset" is
-               accesslevel followed by username (no separator)
-            
-            example:\033[35m
-              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed  \033[36m
-              mount current directory at "/" with
-               * r (read-only) for everyone
-               * a (read+write) for ed
-              mount ../inc at "/dump" with
-               * w (write-only) for everyone
-               * a (read+write) for ed  \033[0m
-            
-            if no accounts or volumes are configured,
-            current folder will be read/write for everyone
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = argv.index(dk)
+        except:
+            continue
 
-            consider the config file for more flexible account/volume management,
-            including dynamic reload at runtime (and being more readable w)
-            """
-        ),
-    )
-    ap.add_argument(
-        "-c", metavar="PATH", type=str, action="append", help="add config file"
-    )
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind")
-    ap.add_argument("-p", metavar="PORT", type=int, default=3923, help="port to bind")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
-    ap.add_argument(
-        "-j", metavar="CORES", type=int, default=1, help="max num cpu cores"
-    )
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
-    ap.add_argument("-q", action="store_true", help="quiet")
-    ap.add_argument("-ed", action="store_true", help="enable ?dots")
-    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
-    ap.add_argument("-nih", action="store_true", help="no info hostname")
-    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
-    al = ap.parse_args()
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        argv[idx] = nk
+        time.sleep(2)
+
+    try:
+        al = run_argparse(argv, RiceFormatter)
+    except AssertionError:
+        al = run_argparse(argv, Dodge11874)
+
+    if al.stackmon:
+        fp, f = al.stackmon.rsplit(",", 1)
+        f = int(f)
+        t = threading.Thread(
+            target=stackmon,
+            args=(fp, f),
+        )
+        t.daemon = True
+        t.start()
+
+    # propagate implications
+    for k1, k2 in IMPLICATIONS:
+        if getattr(al, k1):
+            setattr(al, k2, True)
+
+    al.i = al.i.split(",")
+    try:
+        if "-" in al.p:
+            lo, hi = [int(x) for x in al.p.split("-")]
+            al.p = list(range(lo, hi + 1))
+        else:
+            al.p = [int(x) for x in al.p.split(",")]
+    except:
+        raise Exception("invalid value for -p")
+
+    if HAVE_SSL:
+        if al.ssl_ver:
+            configure_ssl_ver(al)
+
+        if al.ciphers:
+            configure_ssl_ciphers(al)
+    else:
+        warn("ssl module does not exist; cannot enable https")
+
+    if PY2 and WINDOWS and al.e2d:
+        warn(
+            "windows py2 cannot do unicode filenames with -e2d\n"
+            + "  (if you crash with codec errors then that is why)"
+        )
+
+    if sys.version_info < (3, 6):
+        al.no_scandir = True
+
+    # signal.signal(signal.SIGINT, sighandler)
 
     SvcHub(al).run()
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 5, 3)
-CODENAME = "fuse jelly"
-BUILD_DT = (2020, 11, 13)
+VERSION = (0, 11, 24)
+CODENAME = "the grid"
+BUILD_DT = (2021, 6, 22)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -1,23 +1,53 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
 import re
+import os
+import sys
+import stat
+import base64
+import hashlib
 import threading
 
-from .__init__ import PY2, WINDOWS
-from .util import undot, Pebkac, fsdec, fsenc
+from .__init__ import WINDOWS
+from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir, nuprint
 
 
 class VFS(object):
     """single level in the virtual fs"""
 
-    def __init__(self, realpath, vpath, uread=[], uwrite=[]):
+    def __init__(self, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}):
         self.realpath = realpath  # absolute path on host filesystem
         self.vpath = vpath  # absolute path in the virtual filesystem
         self.uread = uread  # users who can read this
         self.uwrite = uwrite  # users who can write this
+        self.uadm = uadm  # users who are regular admins
+        self.flags = flags  # config switches
         self.nodes = {}  # child nodes
+        self.histtab = None  # all realpath->histpath
+        self.dbv = None  # closest full/non-jump parent
+
+        if realpath:
+            self.histpath = os.path.join(realpath, ".hist")  # db / thumbcache
+            self.all_vols = {vpath: self}  # flattened recursive
+        else:
+            self.histpath = None
+            self.all_vols = None
+
+    def __repr__(self):
+        return "VFS({})".format(
+            ", ".join(
+                "{}={!r}".format(k, self.__dict__[k])
+                for k in "realpath vpath uread uwrite uadm flags".split()
+            )
+        )
+
+    def get_all_vols(self, outdict):
+        if self.realpath:
+            outdict[self.vpath] = self
+
+        for v in self.nodes.values():
+            v.get_all_vols(outdict)
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""
@@ -32,11 +62,14 @@ class VFS(object):
                 return self.nodes[name].add(src, dst)
 
             vn = VFS(
-                "{}/{}".format(self.realpath, name),
+                os.path.join(self.realpath, name) if self.realpath else None,
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
                 self.uwrite,
+                self.uadm,
+                self._copy_flags(name),
             )
+            vn.dbv = self.dbv or self
             self.nodes[name] = vn
             return vn.add(src, dst)
 
@@ -47,9 +80,27 @@ class VFS(object):
         # leaf does not exist; create and keep permissions blank
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
+        vn.dbv = self.dbv or self
         self.nodes[dst] = vn
         return vn
 
+    def _copy_flags(self, name):
+        flags = {k: v for k, v in self.flags.items()}
+        hist = flags.get("hist")
+        if hist and hist != "-":
+            flags["hist"] = "{}/{}".format(hist.rstrip("/"), name)
+
+        return flags
+
+    def bubble_flags(self):
+        if self.dbv:
+            for k, v in self.dbv.flags.items():
+                if k not in ["hist"]:
+                    self.flags[k] = v
+
+        for v in self.nodes.values():
+            v.bubble_flags()
+
     def _find(self, vpath):
         """return [vfs,remainder]"""
         vpath = undot(vpath)
@@ -76,6 +127,7 @@ class VFS(object):
         ]
 
     def get(self, vpath, uname, will_read, will_write):
+        # type: (str, str, bool, bool) -> tuple[VFS, str]
         """returns [vfsnode,fs_remainder] if user has the requested permissions"""
         vn, rem = self._find(vpath)
 
@@ -87,52 +139,170 @@ class VFS(object):
 
         return vn, rem
 
+    def get_dbv(self, vrem):
+        dbv = self.dbv
+        if not dbv:
+            return self, vrem
+
+        vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem]
+        vrem = "/".join([x for x in vrem if x])
+        return dbv, vrem
+
     def canonical(self, rem):
         """returns the canonical path (fully-resolved absolute fs path)"""
         rp = self.realpath
         if rem:
             rp += "/" + rem
 
-        return fsdec(os.path.realpath(fsenc(rp)))
+        try:
+            return fsdec(os.path.realpath(fsenc(rp)))
+        except:
+            if not WINDOWS:
+                raise
 
-    def ls(self, rem, uname):
+            # cpython bug introduced in 3.8, still exists in 3.9.1;
+            # some win7sp1 and win10:20H2 boxes cannot realpath a
+            # networked drive letter such as b"n:" or b"n:\\"
+            #
+            # requirements to trigger:
+            #  * bytestring (not unicode str)
+            #  * just the drive letter (subfolders are ok)
+            #  * networked drive (regular disks and vmhgfs are ok)
+            #  * on an enterprise network (idk, cannot repro with samba)
+            #
+            # hits the following exceptions in succession:
+            #  * access denied at L601: "path = _getfinalpathname(path)"
+            #  * "cant concat str to bytes" at L621: "return path + tail"
+            #
+            return os.path.realpath(rp)
+
+    def ls(self, rem, uname, scandir, incl_wo=False, lstat=False):
+        # type: (str, str, bool, bool, bool) -> tuple[str, str, dict[str, VFS]]
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
-        items = os.listdir(fsenc(abspath))
-        real = [fsdec(x) for x in items]
+        real = list(statdir(nuprint, scandir, lstat, abspath))
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
-                if uname in vn2.uread:
+                ok = uname in vn2.uread or "*" in vn2.uread
+
+                if not ok and incl_wo:
+                    ok = uname in vn2.uwrite or "*" in vn2.uwrite
+
+                if ok:
                     virt_vis[name] = vn2
 
             # no vfs nodes in the list of real inodes
-            real = [x for x in real if x not in self.nodes]
+            real = [x for x in real if x[0] not in self.nodes]
 
         return [abspath, real, virt_vis]
 
-    def user_tree(self, uname, readable=False, writable=False):
-        ret = []
-        opt1 = readable and (uname in self.uread or "*" in self.uread)
-        opt2 = writable and (uname in self.uwrite or "*" in self.uwrite)
-        if opt1 or opt2:
-            ret.append(self.vpath)
+    def walk(self, rel, rem, seen, uname, dots, scandir, lstat):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(
+            rem, uname, scandir, incl_wo=False, lstat=lstat
+        )
+
+        if seen and not fsroot.startswith(seen[-1]) and fsroot in seen:
+            print("bailing from symlink loop,\n  {}\n  {}".format(seen[-1], fsroot))
+            return
+
+        seen = seen[:] + [fsroot]
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, seen, uname, dots, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", seen, uname, dots, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk(
+            "", vrem, [], uname, dots, scandir, False
+        ):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+
+                rm = [x for x in rd if x[0] not in flt]
+                [rd.remove(x) for x in rm]
+
+                rm = [x for x in vd.keys() if x not in flt]
+                [vd.pop(x) for x in rm]
+
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
+    def user_tree(self, uname, readable, writable, admin):
+        is_readable = False
+        if uname in self.uread or "*" in self.uread:
+            readable.append(self.vpath)
+            is_readable = True
+
+        if uname in self.uwrite or "*" in self.uwrite:
+            writable.append(self.vpath)
+            if is_readable:
+                admin.append(self.vpath)
 
         for _, vn in sorted(self.nodes.items()):
-            ret.extend(vn.user_tree(uname, readable, writable))
-
-        return ret
+            vn.user_tree(uname, readable, writable, admin)
 
 
 class AuthSrv(object):
     """verifies users against given paths"""
 
-    def __init__(self, args, log_func):
-        self.log_func = log_func
+    def __init__(self, args, log_func, warn_anonwrite=True):
         self.args = args
-
-        self.warn_anonwrite = True
+        self.log_func = log_func
+        self.warn_anonwrite = warn_anonwrite
+        self.line_ctr = 0
 
         if WINDOWS:
             self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -142,14 +312,9 @@ class AuthSrv(object):
         self.mutex = threading.Lock()
         self.reload()
 
-    def log(self, msg):
-        self.log_func("auth", msg)
-
-    def invert(self, orig):
-        if PY2:
-            return {v: k for k, v in orig.iteritems()}
-        else:
-            return {v: k for k, v in orig.items()}
+    def log(self, msg, c=0):
+        if self.log_func:
+            self.log_func("auth", msg, c)
 
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
@@ -161,10 +326,12 @@ class AuthSrv(object):
 
         yield prev, True
 
-    def _parse_config_file(self, fd, user, mread, mwrite, mount):
+    def _parse_config_file(self, fd, user, mread, mwrite, madm, mflags, mount):
         vol_src = None
         vol_dst = None
+        self.line_ctr = 0
         for ln in [x.decode("utf-8").strip() for x in fd]:
+            self.line_ctr += 1
             if not ln and vol_src is not None:
                 vol_src = None
                 vol_dst = None
@@ -191,13 +358,57 @@ class AuthSrv(object):
                 mount[vol_dst] = vol_src
                 mread[vol_dst] = []
                 mwrite[vol_dst] = []
+                madm[vol_dst] = []
+                mflags[vol_dst] = {}
                 continue
 
-            lvl, uname = ln.split(" ")
-            if lvl in "ra":
-                mread[vol_dst].append(uname)
-            if lvl in "wa":
-                mwrite[vol_dst].append(uname)
+            if len(ln) > 1:
+                lvl, uname = ln.split(" ")
+            else:
+                lvl = ln
+                uname = "*"
+
+            self._read_vol_str(
+                lvl,
+                uname,
+                mread[vol_dst],
+                mwrite[vol_dst],
+                madm[vol_dst],
+                mflags[vol_dst],
+            )
+
+    def _read_vol_str(self, lvl, uname, mr, mw, ma, mf):
+        if lvl == "c":
+            cval = True
+            if "=" in uname:
+                uname, cval = uname.split("=", 1)
+
+            self._read_volflag(mf, uname, cval, False)
+            return
+
+        if uname == "":
+            uname = "*"
+
+        if lvl in "ra":
+            mr.append(uname)
+
+        if lvl in "wa":
+            mw.append(uname)
+
+        if lvl == "a":
+            ma.append(uname)
+
+    def _read_volflag(self, flags, name, value, is_list):
+        if name not in ["mtp"]:
+            flags[name] = value
+            return
+
+        if not is_list:
+            value = [value]
+        elif not value:
+            return
+
+        flags[name] = flags.get(name, []) + value
 
     def reload(self):
         """
@@ -210,6 +421,8 @@ class AuthSrv(object):
         user = {}  # username:password
         mread = {}  # mountpoint:[username]
         mwrite = {}  # mountpoint:[username]
+        madm = {}  # mountpoint:[username]
+        mflags = {}  # mountpoint:[flag]
         mount = {}  # dst:src (mountpoint:realpath)
 
         if self.args.a:
@@ -219,40 +432,61 @@ class AuthSrv(object):
 
         if self.args.v:
             # list of src:dst:permset:permset:...
-            # permset is [rwa]username
+            # permset is [rwa]username or [c]flag
             for v_str in self.args.v:
                 m = self.re_vol.match(v_str)
                 if not m:
                     raise Exception("invalid -v argument: [{}]".format(v_str))
 
                 src, dst, perms = m.groups()
+                if WINDOWS:
+                    src = uncyg(src)
+
                 # print("\n".join([src, dst, perms]))
                 src = fsdec(os.path.abspath(fsenc(src)))
                 dst = dst.strip("/")
                 mount[dst] = src
                 mread[dst] = []
                 mwrite[dst] = []
+                madm[dst] = []
+                mflags[dst] = {}
 
                 perms = perms.split(":")
                 for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
-                    if uname == "":
-                        uname = "*"
-                    if lvl in "ra":
-                        mread[dst].append(uname)
-                    if lvl in "wa":
-                        mwrite[dst].append(uname)
+                    self._read_vol_str(
+                        lvl, uname, mread[dst], mwrite[dst], madm[dst], mflags[dst]
+                    )
 
         if self.args.c:
             for cfg_fn in self.args.c:
                 with open(cfg_fn, "rb") as f:
-                    self._parse_config_file(f, user, mread, mwrite, mount)
+                    try:
+                        self._parse_config_file(
+                            f, user, mread, mwrite, madm, mflags, mount
+                        )
+                    except:
+                        m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
+                        print(m.format(cfg_fn, self.line_ctr))
+                        raise
+
+        # case-insensitive; normalize
+        if WINDOWS:
+            cased = {}
+            for k, v in mount.items():
+                try:
+                    cased[k] = fsdec(os.path.realpath(fsenc(v)))
+                except:
+                    cased[k] = v
+
+            mount = cased
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(os.path.abspath("."), "", [], [])
+            vfs = VFS(None, "")
+            vfs.flags["d2d"] = True
 
         maxdepth = 0
         for dst in sorted(mount.keys(), key=lambda x: (x.count("/"), len(x))):
@@ -262,12 +496,20 @@ class AuthSrv(object):
 
             if dst == "":
                 # rootfs was mapped; fully replaces the default CWD vfs
-                vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst])
+                vfs = VFS(
+                    mount[dst], dst, mread[dst], mwrite[dst], madm[dst], mflags[dst]
+                )
                 continue
 
             v = vfs.add(mount[dst], dst)
             v.uread = mread[dst]
             v.uwrite = mwrite[dst]
+            v.uadm = madm[dst]
+            v.flags = mflags[dst]
+            v.dbv = None
+
+        vfs.all_vols = {}
+        vfs.get_all_vols(vfs.all_vols)
 
         missing_users = {}
         for d in [mread, mwrite]:
@@ -278,28 +520,265 @@ class AuthSrv(object):
 
         if missing_users:
             self.log(
-                "\033[31myou must -a the following users: "
-                + ", ".join(k for k in sorted(missing_users))
-                + "\033[0m"
+                "you must -a the following users: "
+                + ", ".join(k for k in sorted(missing_users)),
+                c=1,
             )
             raise Exception("invalid config")
 
+        promote = []
+        demote = []
+        for vol in vfs.all_vols.values():
+            hid = hashlib.sha512(fsenc(vol.realpath)).digest()
+            hid = base64.b32encode(hid).decode("ascii").lower()
+            vflag = vol.flags.get("hist")
+            if vflag == "-":
+                pass
+            elif vflag:
+                vol.histpath = uncyg(vflag) if WINDOWS else vflag
+            elif self.args.hist:
+                for nch in range(len(hid)):
+                    hpath = os.path.join(self.args.hist, hid[: nch + 1])
+                    try:
+                        os.makedirs(hpath)
+                    except:
+                        pass
+
+                    powner = os.path.join(hpath, "owner.txt")
+                    try:
+                        with open(powner, "rb") as f:
+                            owner = f.read().rstrip()
+                    except:
+                        owner = None
+
+                    me = fsenc(vol.realpath).rstrip()
+                    if owner not in [None, me]:
+                        continue
+
+                    if owner is None:
+                        with open(powner, "wb") as f:
+                            f.write(me)
+
+                    vol.histpath = hpath
+                    break
+
+            vol.histpath = os.path.realpath(vol.histpath)
+            if vol.dbv:
+                if os.path.exists(os.path.join(vol.histpath, "up2k.db")):
+                    promote.append(vol)
+                    vol.dbv = None
+                else:
+                    demote.append(vol)
+
+        # discard jump-vols
+        for v in demote:
+            vfs.all_vols.pop(v.vpath)
+
+        if promote:
+            msg = [
+                "\n  the following jump-volumes were generated to assist the vfs.\n  As they contain a database (probably from v0.11.11 or older),\n  they are promoted to full volumes:"
+            ]
+            for vol in promote:
+                msg.append(
+                    "  /{}  ({})  ({})".format(vol.vpath, vol.realpath, vol.histpath)
+                )
+
+            self.log("\n\n".join(msg) + "\n", c=3)
+
+        vfs.histtab = {v.realpath: v.histpath for v in vfs.all_vols.values()}
+
+        all_mte = {}
+        errors = False
+        for vol in vfs.all_vols.values():
+            if (self.args.e2ds and vol.uwrite) or self.args.e2dsa:
+                vol.flags["e2ds"] = True
+
+            if self.args.e2d or "e2ds" in vol.flags:
+                vol.flags["e2d"] = True
+
+            if self.args.no_hash:
+                if "ehash" not in vol.flags:
+                    vol.flags["dhash"] = True
+
+            for k in ["e2t", "e2ts", "e2tsr"]:
+                if getattr(self.args, k):
+                    vol.flags[k] = True
+
+            for k1, k2 in IMPLICATIONS:
+                if k1 in vol.flags:
+                    vol.flags[k2] = True
+
+            # default tag-list if unset
+            if "mte" not in vol.flags:
+                vol.flags["mte"] = self.args.mte
+
+            # append parsers from argv to volume-flags
+            self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
+
+            # d2d drops all database features for a volume
+            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"]]:
+                if not vol.flags.get(grp, False):
+                    continue
+
+                vol.flags["d2t"] = True
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # mt* needs e2t so drop those too
+            for grp, rm in [["e2t", "mt"]]:
+                if vol.flags.get(grp, False):
+                    continue
+
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # verify tags mentioned by -mt[mp] are used by -mte
+            local_mtp = {}
+            local_only_mtp = {}
+            tags = vol.flags.get("mtp", []) + vol.flags.get("mtm", [])
+            tags = [x.split("=")[0] for x in tags]
+            tags = [y for x in tags for y in x.split(",")]
+            for a in tags:
+                local_mtp[a] = True
+                local = True
+                for b in self.args.mtp or []:
+                    b = b.split("=")[0]
+                    if a == b:
+                        local = False
+
+                if local:
+                    local_only_mtp[a] = True
+
+            local_mte = {}
+            for a in vol.flags.get("mte", "").split(","):
+                local = True
+                all_mte[a] = True
+                local_mte[a] = True
+                for b in self.args.mte.split(","):
+                    if not a or not b:
+                        continue
+
+                    if a == b:
+                        local = False
+
+            for mtp in local_only_mtp.keys():
+                if mtp not in local_mte:
+                    m = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)'
+                    self.log(m.format(vol.vpath, mtp), 1)
+                    errors = True
+
+        tags = self.args.mtp or []
+        tags = [x.split("=")[0] for x in tags]
+        tags = [y for x in tags for y in x.split(",")]
+        for mtp in tags:
+            if mtp not in all_mte:
+                m = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volume-flag)'
+                self.log(m.format(mtp), 1)
+                errors = True
+
+        if errors:
+            sys.exit(1)
+
+        vfs.bubble_flags()
+
         try:
             v, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == v.realpath:
                 self.warn_anonwrite = False
-                self.log(
-                    "\033[31manyone can read/write the current directory: {}\033[0m".format(
-                        v.realpath
-                    )
-                )
+                msg = "anyone can read/write the current directory: {}"
+                self.log(msg.format(v.realpath), c=1)
         except Pebkac:
             self.warn_anonwrite = True
 
         with self.mutex:
             self.vfs = vfs
             self.user = user
-            self.iuser = self.invert(user)
+            self.iuser = {v: k for k, v in user.items()}
 
         # import pprint
         # pprint.pprint({"usr": user, "rd": mread, "wr": mwrite, "mnt": mount})
+
+    def dbg_ls(self):
+        users = self.args.ls
+        vols = "*"
+        flags = []
+
+        try:
+            users, vols = users.split(",", 1)
+        except:
+            pass
+
+        try:
+            vols, flags = vols.split(",", 1)
+            flags = flags.split(",")
+        except:
+            pass
+
+        if users == "**":
+            users = list(self.user.keys()) + ["*"]
+        else:
+            users = [users]
+
+        for u in users:
+            if u not in self.user and u != "*":
+                raise Exception("user not found: " + u)
+
+        if vols == "*":
+            vols = ["/" + x for x in self.vfs.all_vols.keys()]
+        else:
+            vols = [vols]
+
+        for v in vols:
+            if not v.startswith("/"):
+                raise Exception("volumes must start with /")
+
+            if v[1:] not in self.vfs.all_vols:
+                raise Exception("volume not found: " + v)
+
+        self.log({"users": users, "vols": vols, "flags": flags})
+        for k, v in self.vfs.all_vols.items():
+            self.log("/{}: read({}) write({})".format(k, v.uread, v.uwrite))
+
+        flag_v = "v" in flags
+        flag_ln = "ln" in flags
+        flag_p = "p" in flags
+        flag_r = "r" in flags
+
+        n_bads = 0
+        for v in vols:
+            v = v[1:]
+            vtop = "/{}/".format(v) if v else "/"
+            for u in users:
+                self.log("checking /{} as {}".format(v, u))
+                try:
+                    vn, _ = self.vfs.get(v, u, True, False)
+                except:
+                    continue
+
+                atop = vn.realpath
+                g = vn.walk("", "", [], u, True, not self.args.no_scandir, False)
+                for vpath, apath, files, _, _ in g:
+                    fnames = [n[0] for n in files]
+                    vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+                    vpaths = [vtop + x for x in vpaths]
+                    apaths = [os.path.join(apath, n) for n in fnames]
+                    files = [[vpath + "/", apath + os.sep]] + list(zip(vpaths, apaths))
+
+                    if flag_ln:
+                        files = [x for x in files if not x[1].startswith(atop + os.sep)]
+                        n_bads += len(files)
+
+                    if flag_v:
+                        msg = [
+                            '# user "{}", vpath "{}"\n{}'.format(u, vp, ap)
+                            for vp, ap in files
+                        ]
+                    else:
+                        msg = [x[1] for x in files]
+
+                    if msg:
+                        nuprint("\n".join(msg))
+
+                if n_bads and flag_p:
+                    raise Exception("found symlink leaving volume, and strict is set")
+
+        if not flag_r:
+            sys.exit(0)

copyparty/broker_mp.py

@@ -44,22 +44,29 @@ class BrokerMp(object):
             proc.clients = {}
             proc.workload = 0
 
-            thr = threading.Thread(target=self.collector, args=(proc,))
+            thr = threading.Thread(
+                target=self.collector, args=(proc,), name="mp-collector"
+            )
             thr.daemon = True
             thr.start()
 
             self.procs.append(proc)
             proc.start()
 
-        if True:
-            thr = threading.Thread(target=self.debug_load_balancer)
+        if not self.args.q:
+            thr = threading.Thread(
+                target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
+            )
             thr.daemon = True
             thr.start()
 
     def shutdown(self):
         self.log("broker", "shutting down")
-        for proc in self.procs:
-            thr = threading.Thread(target=proc.q_pend.put([0, "shutdown", []]))
+        for n, proc in enumerate(self.procs):
+            thr = threading.Thread(
+                target=proc.q_pend.put([0, "shutdown", []]),
+                name="mp-shutdown-{}-{}".format(n, len(self.procs)),
+            )
             thr.start()
 
         with self.mutex:

copyparty/broker_mpw.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
+from copyparty.authsrv import AuthSrv
 
 import sys
 import time
@@ -27,20 +28,23 @@ class MpWorker(object):
         self.retpend = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
-        self.workload_thr_active = False
+        self.workload_thr_alive = False
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
         if not FAKE_MP:
             signal.signal(signal.SIGINT, self.signal_handler)
 
+        # starting to look like a good idea
+        self.asrv = AuthSrv(args, None, False)
+
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
+        self.httpsrv = HttpSrv(self, True)
         self.httpsrv.disconnect_func = self.httpdrop
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
-        thr = threading.Thread(target=self.main)
+        thr = threading.Thread(target=self.main, name="mpw-main")
         thr.daemon = True
         thr.start()
         thr.join()
@@ -49,11 +53,11 @@ class MpWorker(object):
         # print('k')
         pass
 
-    def log(self, src, msg):
-        self.q_yield.put([0, "log", [src, msg]])
+    def log(self, src, msg, c=0):
+        self.q_yield.put([0, "log", [src, msg, c]])
 
-    def logw(self, msg):
-        self.log("mp{}".format(self.n), msg)
+    def logw(self, msg, c=0):
+        self.log("mp{}".format(self.n), msg, c)
 
     def httpdrop(self, addr):
         self.q_yield.put([0, "httpdrop", [addr]])
@@ -64,6 +68,7 @@ class MpWorker(object):
 
             # self.logw("work: [{}]".format(d[0]))
             if dest == "shutdown":
+                self.httpsrv.shutdown()
                 self.logw("ok bye")
                 sys.exit(0)
                 return
@@ -73,13 +78,17 @@ class MpWorker(object):
                 if PY2:
                     sck = pickle.loads(sck)  # nosec
 
-                self.log("%s %s" % addr, "-" * 4 + "C-qpop")
+                if self.args.log_conn:
+                    self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:
-                    if not self.workload_thr_active:
+                    if not self.workload_thr_alive:
                         self.workload_thr_alive = True
-                        thr = threading.Thread(target=self.thr_workload)
+                        thr = threading.Thread(
+                            target=self.thr_workload, name="mpw-workload"
+                        )
                         thr.daemon = True
                         thr.start()
 

copyparty/broker_thr.py

@@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals
 
 import threading
 
+from .authsrv import AuthSrv
 from .httpsrv import HttpSrv
 from .broker_util import ExceptionalQueue, try_exec
 
@@ -14,6 +15,7 @@ class BrokerThr(object):
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
+        self.asrv = hub.asrv
 
         self.mutex = threading.Lock()
 
@@ -23,12 +25,15 @@ class BrokerThr(object):
 
     def shutdown(self):
         # self.log("broker", "shutting down")
+        self.httpsrv.shutdown()
         pass
 
     def put(self, want_retval, dest, *args):
         if dest == "httpconn":
             sck, addr = args
-            self.log("%s %s" % addr, "-" * 4 + "C-qpop")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+
             self.httpsrv.accept(sck, addr)
 
         else:

copyparty/httpconn.py

@@ -1,30 +1,24 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
-import sys
-import ssl
 import time
 import socket
 
+HAVE_SSL = True
 try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * python3 -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-"""
-    )
-    sys.exit(1)
+    import ssl
+except:
+    HAVE_SSL = False
 
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
+from .u2idx import U2idx
+from .th_cli import ThumbCli
+from .th_srv import HAVE_PIL
+from .ico import Ico
 
 
 class HttpConn(object):
@@ -39,32 +33,58 @@ class HttpConn(object):
         self.hsrv = hsrv
 
         self.args = hsrv.args
-        self.auth = hsrv.auth
+        self.asrv = hsrv.asrv
+        self.is_mp = hsrv.is_mp
         self.cert_path = hsrv.cert_path
 
+        enth = HAVE_PIL and not self.args.no_thumb
+        self.thumbcli = ThumbCli(hsrv.broker) if enth else None
+        self.ico = Ico(self.args)
+
         self.t0 = time.time()
+        self.stopping = False
         self.nbyte = 0
         self.workload = 0
+        self.u2idx = None
         self.log_func = hsrv.log
-        self.log_src = "{} \033[36m{}".format(addr[0], addr[1]).ljust(26)
+        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
+        self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
+    def shutdown(self):
+        self.stopping = True
+        try:
+            self.s.shutdown(socket.SHUT_RDWR)
+            self.s.close()
+        except:
+            pass
+
+    def set_rproxy(self, ip=None):
+        if ip is None:
+            color = 36
+            ip = self.addr[0]
+            self.rproxy = None
+        else:
+            color = 34
+            self.rproxy = ip
+
+        self.ip = ip
+        self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
+        return self.log_src
 
     def respath(self, res_name):
         return os.path.join(E.mod, "web", res_name)
 
-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)
 
-    def run(self):
+    def get_u2idx(self):
+        if not self.u2idx:
+            self.u2idx = U2idx(self)
+
+        return self.u2idx
+
+    def _detect_https(self):
         method = None
-        self.sr = None
         if self.cert_path:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
@@ -85,20 +105,64 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
-                self.log(err)
+                if method:
+                    self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 
-        if method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]:
+        return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
+
+    def run(self):
+        self.sr = None
+        if self.args.https_only:
+            is_https = True
+        elif self.args.http_only or not HAVE_SSL:
+            is_https = False
+        else:
+            is_https = self._detect_https()
+
+        if is_https:
             if self.sr:
-                self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
+                self.log("TODO: cannot do https in jython", c="1;31")
                 return
 
             self.log_src = self.log_src.replace("[36m", "[35m")
             try:
-                self.s = ssl.wrap_socket(
-                    self.s, server_side=True, certfile=self.cert_path
-                )
+                ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+                ctx.load_cert_chain(self.cert_path)
+                if self.args.ssl_ver:
+                    ctx.options &= ~self.args.ssl_flags_en
+                    ctx.options |= self.args.ssl_flags_de
+                    # print(repr(ctx.options))
+
+                if self.args.ssl_log:
+                    try:
+                        ctx.keylog_filename = self.args.ssl_log
+                    except:
+                        self.log("keylog failed; openssl or python too old")
+
+                if self.args.ciphers:
+                    ctx.set_ciphers(self.args.ciphers)
+
+                self.s = ctx.wrap_socket(self.s, server_side=True)
+                msg = [
+                    "\033[1;3{:d}m{}".format(c, s)
+                    for c, s in zip([0, 5, 0], self.s.cipher())
+                ]
+                self.log(" ".join(msg) + "\033[0m")
+
+                if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
+                    overlap = [y[::-1] for y in self.s.shared_ciphers()]
+                    lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
+                    self.log("\n".join(lines))
+                    for k, v in [
+                        ["compression", self.s.compression()],
+                        ["ALPN proto", self.s.selected_alpn_protocol()],
+                        ["NPN proto", self.s.selected_npn_protocol()],
+                    ]:
+                        self.log("TLS {}: {}".format(k, v or "nah"))
+
             except Exception as ex:
                 em = str(ex)
 
@@ -107,18 +171,23 @@ class HttpConn(object):
                     self.log("client rejected our certificate (nice)")
 
                 elif "ALERT_CERTIFICATE_UNKNOWN" in em:
-                    # chrome-android keeps doing this
+                    # android-chrome keeps doing this
                     pass
 
                 else:
-                    self.log("\033[35mhandshake\033[0m " + em)
+                    self.log("handshake\033[0m " + em, c=5)
 
                 return
 
         if not self.sr:
             self.sr = Unrecv(self.s)
 
-        while True:
+        while not self.stopping:
+            if self.is_mp:
+                self.workload += 50
+                if self.workload >= 2 ** 31:
+                    self.workload = 100
+
             cli = HttpCli(self)
             if not cli.run():
                 return

copyparty/httpsrv.py

@@ -2,13 +2,31 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
 from .__init__ import E, MACOS
-from .httpconn import HttpConn
 from .authsrv import AuthSrv
+from .httpconn import HttpConn
 
 
 class HttpSrv(object):
@@ -17,10 +35,12 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker):
+    def __init__(self, broker, is_mp=False):
         self.broker = broker
+        self.is_mp = is_mp
         self.args = broker.args
         self.log = broker.log
+        self.asrv = broker.asrv
 
         self.disconnect_func = None
         self.mutex = threading.Lock()
@@ -28,7 +48,13 @@ class HttpSrv(object):
         self.clients = {}
         self.workload = 0
         self.workload_thr_alive = False
-        self.auth = AuthSrv(self.args, self.log)
+
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+        }
 
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
@@ -38,8 +64,14 @@ class HttpSrv(object):
 
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log("%s %s" % addr, "-" * 5 + "C-cthr")
-        thr = threading.Thread(target=self.thr_client, args=(sck, addr))
+        if self.args.log_conn:
+            self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+
+        thr = threading.Thread(
+            target=self.thr_client,
+            args=(sck, addr),
+            name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+        )
         thr.daemon = True
         thr.start()
 
@@ -48,7 +80,14 @@ class HttpSrv(object):
             return len(self.clients)
 
     def shutdown(self):
-        self.log("ok bye")
+        clients = list(self.clients.keys())
+        for cli in clients:
+            try:
+                cli.shutdown()
+            except:
+                pass
+
+        self.log("httpsrv-n", "ok bye")
 
     def thr_client(self, sck, addr):
         """thread managing one tcp client"""
@@ -57,28 +96,47 @@ class HttpSrv(object):
         cli = HttpConn(sck, addr, self)
         with self.mutex:
             self.clients[cli] = 0
-            self.workload += 50
 
-            if not self.workload_thr_alive:
-                self.workload_thr_alive = True
-                thr = threading.Thread(target=self.thr_workload)
-                thr.daemon = True
-                thr.start()
+            if self.is_mp:
+                self.workload += 50
+                if not self.workload_thr_alive:
+                    self.workload_thr_alive = True
+                    thr = threading.Thread(
+                        target=self.thr_workload, name="httpsrv-workload"
+                    )
+                    thr.daemon = True
+                    thr.start()
 
+        fno = sck.fileno()
         try:
-            self.log("%s %s" % addr, "-" * 6 + "C-crun")
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+
             cli.run()
 
+        except (OSError, socket.error) as ex:
+            if ex.errno not in [10038, 10054, 107, 57, 9]:
+                self.log(
+                    "%s %s" % addr,
+                    "run({}): {}".format(fno, ex),
+                    c=6,
+                )
+
         finally:
-            self.log("%s %s" % addr, "-" * 7 + "C-done")
+            sck = cli.s
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+
             try:
+                fno = sck.fileno()
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
             except (OSError, socket.error) as ex:
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "shut_rdwr err:\n  {}\n  {}".format(repr(sck), ex),
+                        "shut({}): {}".format(fno, ex),
+                        c="1;30",
                     )
                 if ex.errno not in [10038, 10054, 107, 57, 9]:
                     # 10038 No longer considered a socket

copyparty/ico.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import hashlib
+import colorsys
+
+from .__init__ import PY2
+
+
+class Ico(object):
+    def __init__(self, args):
+        self.args = args
+
+    def get(self, ext, as_thumb):
+        """placeholder to make thumbnails not break"""
+
+        h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        if PY2:
+            h = [ord(x) for x in h]
+
+        c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
+        c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
+        c = list(c1) + list(c2)
+        c = [int(x * 255) for x in c]
+        c = "".join(["{:02x}".format(x) for x in c])
+
+        h = 30
+        if not self.args.th_no_crop and as_thumb:
+            w, h = self.args.th_size.split("x")
+            h = int(100 / (float(w) / float(h)))
+
+        svg = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<svg version="1.1" viewBox="0 0 100 {}" xmlns="http://www.w3.org/2000/svg"><g>
+<rect width="100%" height="100%" fill="#{}" />
+<text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" xml:space="preserve"
+  fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
+</g></svg>
+"""
+        svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
+
+        return ["image/svg+xml", svg]

copyparty/mtag.py

@@ -0,0 +1,468 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import sys
+import json
+import shutil
+import subprocess as sp
+
+from .__init__ import PY2, WINDOWS
+from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
+
+if not PY2:
+    unicode = str
+
+
+def have_ff(cmd):
+    if PY2:
+        print("# checking {}".format(cmd))
+        cmd = (cmd + " -version").encode("ascii").split(b" ")
+        try:
+            sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
+            return True
+        except:
+            return False
+    else:
+        return bool(shutil.which(cmd))
+
+
+HAVE_FFMPEG = have_ff("ffmpeg")
+HAVE_FFPROBE = have_ff("ffprobe")
+
+
+class MParser(object):
+    def __init__(self, cmdline):
+        self.tag, args = cmdline.split("=", 1)
+        self.tags = self.tag.split(",")
+
+        self.timeout = 30
+        self.force = False
+        self.audio = "y"
+        self.ext = []
+
+        while True:
+            try:
+                bp = os.path.expanduser(args)
+                if WINDOWS:
+                    bp = uncyg(bp)
+
+                if os.path.exists(bp):
+                    self.bin = bp
+                    return
+            except:
+                pass
+
+            arg, args = args.split(",", 1)
+            arg = arg.lower()
+
+            if arg.startswith("a"):
+                self.audio = arg[1:]  # [r]equire [n]ot [d]ontcare
+                continue
+
+            if arg == "f":
+                self.force = True
+                continue
+
+            if arg.startswith("t"):
+                self.timeout = int(arg[1:])
+                continue
+
+            if arg.startswith("e"):
+                self.ext.append(arg[1:])
+                continue
+
+            raise Exception()
+
+
+def ffprobe(abspath):
+    cmd = [
+        b"ffprobe",
+        b"-hide_banner",
+        b"-show_streams",
+        b"-show_format",
+        b"--",
+        fsenc(abspath),
+    ]
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    r = p.communicate()
+    txt = r[0].decode("utf-8", "replace")
+    return parse_ffprobe(txt)
+
+
+def parse_ffprobe(txt):
+    """ffprobe -show_format -show_streams"""
+    streams = []
+    fmt = {}
+    g = None
+    for ln in [x.rstrip("\r") for x in txt.split("\n")]:
+        try:
+            k, v = ln.split("=", 1)
+            g[k] = v
+            continue
+        except:
+            pass
+
+        if ln == "[STREAM]":
+            g = {}
+            streams.append(g)
+
+        if ln == "[FORMAT]":
+            g = {"codec_type": "format"}  # heh
+            fmt = g
+
+    streams = [fmt] + streams
+    ret = {}  # processed
+    md = {}  # raw tags
+
+    is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
+    if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
+        is_audio = True
+
+    # if audio file, ensure audio stream appears first
+    if (
+        is_audio
+        and len(streams) > 2
+        and streams[1].get("codec_type") != "audio"
+        and streams[2].get("codec_type") == "audio"
+    ):
+        streams = [fmt, streams[2], streams[1]] + streams[3:]
+
+    have = {}
+    for strm in streams:
+        typ = strm.get("codec_type")
+        if typ in have:
+            continue
+
+        have[typ] = True
+        kvm = []
+
+        if typ == "audio":
+            kvm = [
+                ["codec_name", "ac"],
+                ["channel_layout", "chs"],
+                ["sample_rate", ".hz"],
+                ["bit_rate", ".aq"],
+                ["duration", ".dur"],
+            ]
+
+        if typ == "video":
+            if strm.get("DISPOSITION:attached_pic") == "1" or is_audio:
+                continue
+
+            kvm = [
+                ["codec_name", "vc"],
+                ["pix_fmt", "pixfmt"],
+                ["r_frame_rate", ".fps"],
+                ["bit_rate", ".vq"],
+                ["width", ".resw"],
+                ["height", ".resh"],
+                ["duration", ".dur"],
+            ]
+
+        if typ == "format":
+            kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
+
+        for sk, rk in kvm:
+            v = strm.get(sk)
+            if v is None:
+                continue
+
+            if rk.startswith("."):
+                try:
+                    v = float(v)
+                    v2 = ret.get(rk)
+                    if v2 is None or v > v2:
+                        ret[rk] = v
+                except:
+                    # sqlite doesnt care but the code below does
+                    if v not in ["N/A"]:
+                        ret[rk] = v
+            else:
+                ret[rk] = v
+
+    if ret.get("vc") == "ansi":  # shellscript
+        return {}, {}
+
+    for strm in streams:
+        for k, v in strm.items():
+            if not k.startswith("TAG:"):
+                continue
+
+            k = k[4:].strip()
+            v = v.strip()
+            if k and v and k not in md:
+                md[k] = [v]
+
+    for k in [".q", ".vq", ".aq"]:
+        if k in ret:
+            ret[k] /= 1000  # bit_rate=320000
+
+    for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
+        if k in ret:
+            ret[k] = int(ret[k])
+
+    if ".fps" in ret:
+        fps = ret[".fps"]
+        if "/" in fps:
+            fa, fb = fps.split("/")
+            fps = int(fa) * 1.0 / int(fb)
+
+        if fps < 1000 and fmt.get("format_name") not in ["image2", "png_pipe"]:
+            ret[".fps"] = round(fps, 3)
+        else:
+            del ret[".fps"]
+
+    if ".dur" in ret:
+        if ret[".dur"] < 0.1:
+            del ret[".dur"]
+            if ".q" in ret:
+                del ret[".q"]
+
+    if ".resw" in ret and ".resh" in ret:
+        ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
+
+    ret = {k: [0, v] for k, v in ret.items()}
+
+    return ret, md
+
+
+class MTag(object):
+    def __init__(self, log_func, args):
+        self.log_func = log_func
+        self.usable = True
+        self.prefer_mt = False
+        mappings = args.mtm
+        self.backend = "ffprobe" if args.no_mutagen else "mutagen"
+        or_ffprobe = " or ffprobe"
+
+        if self.backend == "mutagen":
+            self.get = self.get_mutagen
+            try:
+                import mutagen
+            except:
+                self.log("could not load mutagen, trying ffprobe instead", c=3)
+                self.backend = "ffprobe"
+
+        if self.backend == "ffprobe":
+            self.get = self.get_ffprobe
+            self.prefer_mt = True
+            # about 20x slower
+            self.usable = HAVE_FFPROBE
+
+            if self.usable and WINDOWS and sys.version_info < (3, 8):
+                self.usable = False
+                or_ffprobe = " or python >= 3.8"
+                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                self.log(msg, c=1)
+
+        if not self.usable:
+            msg = "need mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
+            self.log(
+                msg.format(or_ffprobe, " " * 37, os.path.basename(sys.executable)), c=1
+            )
+            return
+
+        # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
+        tagmap = {
+            "album": ["album", "talb", "\u00a9alb", "original-album", "toal"],
+            "artist": [
+                "artist",
+                "tpe1",
+                "\u00a9art",
+                "composer",
+                "performer",
+                "arranger",
+                "\u00a9wrt",
+                "tcom",
+                "tpe3",
+                "original-artist",
+                "tope",
+            ],
+            "title": ["title", "tit2", "\u00a9nam"],
+            "circle": [
+                "album-artist",
+                "tpe2",
+                "aart",
+                "conductor",
+                "organization",
+                "band",
+            ],
+            ".tn": ["tracknumber", "trck", "trkn", "track"],
+            "genre": ["genre", "tcon", "\u00a9gen"],
+            "date": [
+                "original-release-date",
+                "release-date",
+                "date",
+                "tdrc",
+                "\u00a9day",
+                "original-date",
+                "original-year",
+                "tyer",
+                "tdor",
+                "tory",
+                "year",
+                "creation-time",
+            ],
+            ".bpm": ["bpm", "tbpm", "tmpo", "tbp"],
+            "key": ["initial-key", "tkey", "key"],
+            "comment": ["comment", "comm", "\u00a9cmt", "comments", "description"],
+        }
+
+        if mappings:
+            for k, v in [x.split("=") for x in mappings]:
+                tagmap[k] = v.split(",")
+
+        self.tagmap = {}
+        for k, vs in tagmap.items():
+            vs2 = []
+            for v in vs:
+                if "-" not in v:
+                    vs2.append(v)
+                    continue
+
+                vs2.append(v.replace("-", " "))
+                vs2.append(v.replace("-", "_"))
+                vs2.append(v.replace("-", ""))
+
+            self.tagmap[k] = vs2
+
+        self.rmap = {
+            v: [n, k] for k, vs in self.tagmap.items() for n, v in enumerate(vs)
+        }
+        # self.get = self.compare
+
+    def log(self, msg, c=0):
+        self.log_func("mtag", msg, c)
+
+    def normalize_tags(self, ret, md):
+        for k, v in dict(md).items():
+            if not v:
+                continue
+
+            k = k.lower().split("::")[0].strip()
+            mk = self.rmap.get(k)
+            if not mk:
+                continue
+
+            pref, mk = mk
+            if mk not in ret or ret[mk][0] > pref:
+                ret[mk] = [pref, v[0]]
+
+        # take first value
+        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
+
+        # track 3/7 => track 3
+        for k, v in ret.items():
+            if k[0] == ".":
+                v = v.split("/")[0].strip().lstrip("0")
+                ret[k] = v or 0
+
+        # normalize key notation to rkeobo
+        okey = ret.get("key")
+        if okey:
+            key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
+            ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
+
+        return ret
+
+    def compare(self, abspath):
+        if abspath.endswith(".au"):
+            return {}
+
+        print("\n" + abspath)
+        r1 = self.get_mutagen(abspath)
+        r2 = self.get_ffprobe(abspath)
+
+        keys = {}
+        for d in [r1, r2]:
+            for k in d.keys():
+                keys[k] = True
+
+        diffs = []
+        l1 = []
+        l2 = []
+        for k in sorted(keys.keys()):
+            if k in [".q", ".dur"]:
+                continue  # lenient
+
+            v1 = r1.get(k)
+            v2 = r2.get(k)
+            if v1 == v2:
+                print("  ", k, v1)
+            elif v1 != "0000":  # ffprobe date=0
+                diffs.append(k)
+                print(" 1", k, v1)
+                print(" 2", k, v2)
+                if v1:
+                    l1.append(k)
+                if v2:
+                    l2.append(k)
+
+        if diffs:
+            raise Exception()
+
+        return r1
+
+    def get_mutagen(self, abspath):
+        import mutagen
+
+        try:
+            md = mutagen.File(fsenc(abspath), easy=True)
+            x = md.info.length
+        except Exception as ex:
+            return {}
+
+        ret = {}
+        try:
+            dur = int(md.info.length)
+            try:
+                q = int(md.info.bitrate / 1024)
+            except:
+                q = int((os.path.getsize(fsenc(abspath)) / dur) / 128)
+
+            ret[".dur"] = [0, dur]
+            ret[".q"] = [0, q]
+        except:
+            pass
+
+        return self.normalize_tags(ret, md)
+
+    def get_ffprobe(self, abspath):
+        ret, md = ffprobe(abspath)
+        return self.normalize_tags(ret, md)
+
+    def get_bin(self, parsers, abspath):
+        pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
+        pypath = str(os.pathsep.join(pypath))
+        env = os.environ.copy()
+        env["PYTHONPATH"] = pypath
+
+        ret = {}
+        for tagname, mp in parsers.items():
+            try:
+                cmd = [sys.executable, mp.bin, abspath]
+                args = {"env": env, "timeout": mp.timeout}
+
+                if WINDOWS:
+                    args["creationflags"] = 0x4000
+                else:
+                    cmd = ["nice"] + cmd
+
+                cmd = [fsenc(x) for x in cmd]
+                v = sp.check_output(cmd, **args).strip()
+                if not v:
+                    continue
+
+                if "," not in tagname:
+                    ret[tagname] = v.decode("utf-8")
+                else:
+                    v = json.loads(v)
+                    for tag in tagname.split(","):
+                        if tag and tag in v:
+                            ret[tag] = v[tag]
+            except:
+                pass
+
+        return ret

copyparty/star.py

@@ -0,0 +1,98 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen, name="star-gen")
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/sutil.py

@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/svchub.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import re
+import os
 import sys
 import time
 import threading
@@ -9,9 +10,11 @@ from datetime import datetime, timedelta
 import calendar
 
 from .__init__ import PY2, WINDOWS, MACOS, VT100
+from .util import mp
+from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
-from .util import mp
+from .th_srv import ThumbSrv, HAVE_PIL, HAVE_WEBP
 
 
 class SvcHub(object):
@@ -35,9 +38,28 @@ class SvcHub(object):
         self.log = self._log_disabled if args.q else self._log_enabled
 
         # initiate all services to manage
+        self.asrv = AuthSrv(self.args, self.log, False)
+        if args.ls:
+            self.asrv.dbg_ls()
+
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
+        self.thumbsrv = None
+        if not args.no_thumb:
+            if HAVE_PIL:
+                if not HAVE_WEBP:
+                    args.th_no_webp = True
+                    msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
+                    self.log("thumb", msg, c=3)
+
+                self.thumbsrv = ThumbSrv(self)
+            else:
+                msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
+                self.log(
+                    "thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3
+                )
+
         # decide which worker impl to use
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker
@@ -48,7 +70,7 @@ class SvcHub(object):
         self.broker = Broker(self)
 
     def run(self):
-        thr = threading.Thread(target=self.tcpsrv.run)
+        thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
         thr.daemon = True
         thr.start()
 
@@ -63,18 +85,31 @@ class SvcHub(object):
 
             self.tcpsrv.shutdown()
             self.broker.shutdown()
-            print("nailed it")
+            if self.thumbsrv:
+                self.thumbsrv.shutdown()
 
-    def _log_disabled(self, src, msg):
+                for n in range(200):  # 10s
+                    time.sleep(0.05)
+                    if self.thumbsrv.stopped():
+                        break
+
+                    if n == 3:
+                        print("waiting for thumbsrv (10sec)...")
+
+            print("nailed it", end="")
+        finally:
+            print("\033[0m")
+
+    def _log_disabled(self, src, msg, c=0):
         pass
 
-    def _log_enabled(self, src, msg):
+    def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
         with self.log_mutex:
             now = time.time()
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
-                print("\033[36m{}\033[0m".format(dt.strftime("%Y-%m-%d")))
+                print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
 
                 # unix timestamp of next 00:00:00 (leap-seconds safe)
                 day_now = dt.day
@@ -84,23 +119,30 @@ class SvcHub(object):
                 dt = dt.replace(hour=0, minute=0, second=0)
                 self.next_day = calendar.timegm(dt.utctimetuple())
 
-            fmt = "\033[36m{} \033[33m{:21} \033[0m{}"
+            fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
-                fmt = "{} {:21} {}"
+                fmt = "{} {:21} {}\n"
                 if "\033" in msg:
                     msg = self.ansi_re.sub("", msg)
                 if "\033" in src:
                     src = self.ansi_re.sub("", src)
+            elif c:
+                if isinstance(c, int):
+                    msg = "\033[3{}m{}".format(c, msg)
+                elif "\033" not in c:
+                    msg = "\033[{}m{}\033[0m".format(c, msg)
+                else:
+                    msg = "{}{}\033[0m".format(c, msg)
 
             ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             msg = fmt.format(ts, src, msg)
             try:
-                print(msg)
+                print(msg, end="")
             except UnicodeEncodeError:
                 try:
-                    print(msg.encode("utf-8", "replace").decode())
+                    print(msg.encode("utf-8", "replace").decode(), end="")
                 except:
-                    print(msg.encode("ascii", "replace").decode())
+                    print(msg.encode("ascii", "replace").decode(), end="")
 
     def check_mp_support(self):
         vmin = sys.version_info[1]

copyparty/szip.py

@@ -0,0 +1,274 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, ok="/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import time
 import socket
+import select
 
 from .util import chkcmd, Counter
 
@@ -20,58 +21,101 @@ class TcpSrv(object):
         self.log = hub.log
 
         self.num_clients = Counter()
+        self.stopping = False
 
         ip = "127.0.0.1"
         eps = {ip: "local only"}
-        if self.args.i != ip:
-            eps = self.detect_interfaces(self.args.i) or {self.args.i: "external"}
+        nonlocals = [x for x in self.args.i if x != ip]
+        if nonlocals:
+            eps = self.detect_interfaces(self.args.i)
+            if not eps:
+                for x in nonlocals:
+                    eps[x] = "external"
 
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
-            self.log(
-                "tcpsrv",
-                "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
-                    ip, self.args.p, desc
-                ),
-            )
+            for port in sorted(self.args.p):
+                self.log(
+                    "tcpsrv",
+                    "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
+                        ip, port, desc
+                    ),
+                )
 
-        self.srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.srv = []
+        for ip in self.args.i:
+            for port in self.args.p:
+                self.srv.append(self._listen(ip, port))
+
+    def _listen(self, ip, port):
+        srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         try:
-            self.srv.bind((self.args.i, self.args.p))
+            srv.bind((ip, port))
+            return srv
         except (OSError, socket.error) as ex:
-            if ex.errno == 98:
-                raise Exception(
-                    "\033[1;31mport {} is busy on interface {}\033[0m".format(
-                        self.args.p, self.args.i
-                    )
-                )
-
-            if ex.errno == 99:
-                raise Exception(
-                    "\033[1;31minterface {} does not exist\033[0m".format(self.args.i)
-                )
+            if ex.errno in [98, 48]:
+                e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
+            elif ex.errno in [99, 49]:
+                e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
+            else:
+                raise
+            raise Exception(e)
 
     def run(self):
-        self.srv.listen(self.args.nc)
+        for srv in self.srv:
+            srv.listen(self.args.nc)
+            ip, port = srv.getsockname()
+            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
 
-        self.log("tcpsrv", "listening @ {0}:{1}".format(self.args.i, self.args.p))
+        while not self.stopping:
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
 
-        while True:
-            self.log("tcpsrv", "-" * 1 + "C-ncli")
             if self.num_clients.v >= self.args.nc:
                 time.sleep(0.1)
                 continue
 
-            self.log("tcpsrv", "-" * 2 + "C-acc1")
-            sck, addr = self.srv.accept()
-            self.log("%s %s" % addr, "-" * 3 + "C-acc2")
-            self.num_clients.add()
-            self.hub.broker.put(False, "httpconn", sck, addr)
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
+
+            try:
+                # macos throws bad-fd
+                ready, _, _ = select.select(self.srv, [], [])
+            except:
+                ready = []
+                if not self.stopping:
+                    raise
+
+            for srv in ready:
+                if self.stopping:
+                    break
+
+                sck, addr = srv.accept()
+                sip, sport = srv.getsockname()
+                if self.args.log_conn:
+                    self.log(
+                        "%s %s" % addr,
+                        "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                            "-" * 3, sip, sport % 8, sport
+                        ),
+                        c="1;30",
+                    )
+
+                self.num_clients.add()
+                self.hub.broker.put(False, "httpconn", sck, addr)
 
     def shutdown(self):
+        self.stopping = True
+        try:
+            for srv in self.srv:
+                srv.close()
+        except:
+            pass
+
         self.log("tcpsrv", "ok bye")
 
-    def detect_interfaces(self, listen_ip):
+    def detect_interfaces(self, listen_ips):
         eps = {}
 
         # get all ips and their interfaces
@@ -85,8 +129,9 @@ class TcpSrv(object):
             for ln in ip_addr.split("\n"):
                 try:
                     ip, dev = r.match(ln.rstrip()).groups()
-                    if listen_ip in ["0.0.0.0", ip]:
-                        eps[ip] = dev
+                    for lip in listen_ips:
+                        if lip in ["0.0.0.0", ip]:
+                            eps[ip] = dev
                 except:
                     pass
 
@@ -113,11 +158,12 @@ class TcpSrv(object):
 
         s.close()
 
-        if default_route and listen_ip in ["0.0.0.0", default_route]:
-            desc = "\033[32mexternal"
-            try:
-                eps[default_route] += ", " + desc
-            except:
-                eps[default_route] = desc
+        for lip in listen_ips:
+            if default_route and lip in ["0.0.0.0", default_route]:
+                desc = "\033[32mexternal"
+                try:
+                    eps[default_route] += ", " + desc
+                except:
+                    eps[default_route] = desc
 
         return eps

copyparty/th_cli.py

@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+
+from .util import Cooldown
+from .th_srv import thumb_path, THUMBABLE, FMT_FF
+
+
+class ThumbCli(object):
+    def __init__(self, broker):
+        self.broker = broker
+        self.args = broker.args
+        self.asrv = broker.asrv
+
+        # cache on both sides for less broker spam
+        self.cooldown = Cooldown(self.args.th_poke)
+
+    def get(self, ptop, rem, mtime, fmt):
+        ext = rem.rsplit(".")[-1].lower()
+        if ext not in THUMBABLE:
+            return None
+
+        is_vid = ext in FMT_FF
+        if is_vid and self.args.no_vthumb:
+            return None
+
+        if fmt == "j" and self.args.th_no_jpg:
+            fmt = "w"
+
+        if fmt == "w":
+            if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
+                fmt = "j"
+
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
+        ret = None
+        try:
+            st = os.stat(tpath)
+            if st.st_size:
+                ret = tpath
+            else:
+                return None
+        except:
+            pass
+
+        if ret:
+            tdir = os.path.dirname(tpath)
+            if self.cooldown.poke(tdir):
+                self.broker.put(False, "thumbsrv.poke", tdir)
+
+            return ret
+
+        x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
+        return x.get()

copyparty/th_srv.py

@@ -0,0 +1,409 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import time
+import shutil
+import base64
+import hashlib
+import threading
+import subprocess as sp
+
+from .__init__ import PY2
+from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+
+
+if not PY2:
+    unicode = str
+
+
+HAVE_PIL = False
+HAVE_HEIF = False
+HAVE_AVIF = False
+HAVE_WEBP = False
+
+try:
+    from PIL import Image, ImageOps
+
+    HAVE_PIL = True
+    try:
+        Image.new("RGB", (2, 2)).save(BytesIO(), format="webp")
+        HAVE_WEBP = True
+    except:
+        pass
+
+    try:
+        from pyheif_pillow_opener import register_heif_opener
+
+        register_heif_opener()
+        HAVE_HEIF = True
+    except:
+        pass
+
+    try:
+        import pillow_avif
+
+        HAVE_AVIF = True
+    except:
+        pass
+except:
+    pass
+
+# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
+# ffmpeg -formats
+FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
+FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
+
+if HAVE_HEIF:
+    FMT_PIL += " heif heifs heic heics"
+
+if HAVE_AVIF:
+    FMT_PIL += " avif avifs"
+
+FMT_PIL, FMT_FF = [{x: True for x in y.split(" ") if x} for y in [FMT_PIL, FMT_FF]]
+
+
+THUMBABLE = {}
+
+if HAVE_PIL:
+    THUMBABLE.update(FMT_PIL)
+
+if HAVE_FFMPEG and HAVE_FFPROBE:
+    THUMBABLE.update(FMT_FF)
+
+
+def thumb_path(histpath, rem, mtime, fmt):
+    # base16 = 16 = 256
+    # b64-lc = 38 = 1444
+    # base64 = 64 = 4096
+    try:
+        rd, fn = rem.rsplit("/", 1)
+    except:
+        rd = ""
+        fn = rem
+
+    if rd:
+        h = hashlib.sha512(fsenc(rd)).digest()
+        b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+        rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
+    else:
+        rd = "top"
+
+    # could keep original filenames but this is safer re pathlen
+    h = hashlib.sha512(fsenc(fn)).digest()
+    fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+
+    return "{}/th/{}/{}.{:x}.{}".format(
+        histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
+    )
+
+
+class ThumbSrv(object):
+    def __init__(self, hub):
+        self.hub = hub
+        self.asrv = hub.asrv
+        self.args = hub.args
+        self.log_func = hub.log
+
+        res = hub.args.th_size.split("x")
+        self.res = tuple([int(x) for x in res])
+        self.poke_cd = Cooldown(self.args.th_poke)
+
+        self.mutex = threading.Lock()
+        self.busy = {}
+        self.stopping = False
+        self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4
+        self.q = Queue(self.nthr * 4)
+        for n in range(self.nthr):
+            t = threading.Thread(
+                target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
+            )
+            t.daemon = True
+            t.start()
+
+        if not self.args.no_vthumb and (not HAVE_FFMPEG or not HAVE_FFPROBE):
+            missing = []
+            if not HAVE_FFMPEG:
+                missing.append("ffmpeg")
+
+            if not HAVE_FFPROBE:
+                missing.append("ffprobe")
+
+            msg = "cannot create video thumbnails because some of the required programs are not available: "
+            msg += ", ".join(missing)
+            self.log(msg, c=3)
+
+        t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
+        t.daemon = True
+        t.start()
+
+    def log(self, msg, c=0):
+        self.log_func("thumb", msg, c)
+
+    def shutdown(self):
+        self.stopping = True
+        for _ in range(self.nthr):
+            self.q.put(None)
+
+    def stopped(self):
+        with self.mutex:
+            return not self.nthr
+
+    def get(self, ptop, rem, mtime, fmt):
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
+        abspath = os.path.join(ptop, rem)
+        cond = threading.Condition(self.mutex)
+        do_conv = False
+        with self.mutex:
+            try:
+                self.busy[tpath].append(cond)
+                self.log("wait {}".format(tpath))
+            except:
+                thdir = os.path.dirname(tpath)
+                try:
+                    os.makedirs(thdir)
+                except:
+                    pass
+
+                inf_path = os.path.join(thdir, "dir.txt")
+                if not os.path.exists(inf_path):
+                    with open(inf_path, "wb") as f:
+                        f.write(fsenc(os.path.dirname(abspath)))
+
+                self.busy[tpath] = [cond]
+                do_conv = True
+
+        if do_conv:
+            self.q.put([abspath, tpath])
+            self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
+
+        while not self.stopping:
+            with self.mutex:
+                if tpath not in self.busy:
+                    break
+
+            with cond:
+                cond.wait(3)
+
+        try:
+            st = os.stat(tpath)
+            if st.st_size:
+                return tpath
+        except:
+            pass
+
+        return None
+
+    def worker(self):
+        while not self.stopping:
+            task = self.q.get()
+            if not task:
+                break
+
+            abspath, tpath = task
+            ext = abspath.split(".")[-1].lower()
+            fun = None
+            if not os.path.exists(tpath):
+                if ext in FMT_PIL:
+                    fun = self.conv_pil
+                elif ext in FMT_FF:
+                    fun = self.conv_ffmpeg
+
+            if fun:
+                try:
+                    fun(abspath, tpath)
+                except:
+                    msg = "{} failed on {}\n{}"
+                    self.log(msg.format(fun.__name__, abspath, min_ex()), 3)
+                    with open(tpath, "wb") as _:
+                        pass
+
+            with self.mutex:
+                subs = self.busy[tpath]
+                del self.busy[tpath]
+
+            for x in subs:
+                with x:
+                    x.notify_all()
+
+        with self.mutex:
+            self.nthr -= 1
+
+    def conv_pil(self, abspath, tpath):
+        with Image.open(fsenc(abspath)) as im:
+            crop = not self.args.th_no_crop
+            res2 = self.res
+            if crop:
+                res2 = (res2[0] * 2, res2[1] * 2)
+
+            try:
+                im.thumbnail(res2, resample=Image.LANCZOS)
+                if crop:
+                    iw, ih = im.size
+                    dw, dh = self.res
+                    res = (min(iw, dw), min(ih, dh))
+                    im = ImageOps.fit(im, res, method=Image.LANCZOS)
+            except:
+                im.thumbnail(self.res)
+
+            fmts = ["RGB", "L"]
+            args = {"quality": 40}
+
+            if tpath.endswith(".webp"):
+                # quality 80 = pillow-default
+                # quality 75 = ffmpeg-default
+                # method 0 = pillow-default, fast
+                # method 4 = ffmpeg-default
+                # method 6 = max, slow
+                fmts += ["RGBA", "LA"]
+                args["method"] = 6
+            else:
+                pass  # default q = 75
+
+            if im.mode not in fmts:
+                print("conv {}".format(im.mode))
+                im = im.convert("RGB")
+
+            im.save(tpath, quality=40, method=6)
+
+    def conv_ffmpeg(self, abspath, tpath):
+        ret, _ = ffprobe(abspath)
+
+        ext = abspath.rsplit(".")[-1]
+        if ext in ["h264", "h265"]:
+            seek = []
+        else:
+            dur = ret[".dur"][1] if ".dur" in ret else 4
+            seek = "{:.0f}".format(dur / 3)
+            seek = [b"-ss", seek.encode("utf-8")]
+
+        scale = "scale={0}:{1}:force_original_aspect_ratio="
+        if self.args.th_no_crop:
+            scale += "decrease,setsar=1:1"
+        else:
+            scale += "increase,crop={0}:{1},setsar=1:1"
+
+        scale = scale.format(*list(self.res)).encode("utf-8")
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner"
+        ]
+        cmd += seek
+        cmd += [
+            b"-i", fsenc(abspath),
+            b"-vf", scale,
+            b"-vframes", b"1",
+        ]
+        # fmt: on
+
+        if tpath.endswith(".jpg"):
+            cmd += [
+                b"-q:v",
+                b"6",  # default=??
+            ]
+        else:
+            cmd += [
+                b"-q:v",
+                b"50",  # default=75
+                b"-compression_level:v",
+                b"6",  # default=4, 0=fast, 6=max
+            ]
+
+        cmd += [fsenc(tpath)]
+
+        ret, sout, serr = runcmd(*cmd)
+        if ret != 0:
+            msg = ["ff: {}".format(x) for x in serr.split("\n")]
+            self.log("FFmpeg failed:\n" + "\n".join(msg), c="1;30")
+            raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
+
+    def poke(self, tdir):
+        if not self.poke_cd.poke(tdir):
+            return
+
+        ts = int(time.time())
+        try:
+            p1 = os.path.dirname(tdir)
+            p2 = os.path.dirname(p1)
+            for dp in [tdir, p1, p2]:
+                os.utime(fsenc(dp), (ts, ts))
+        except:
+            pass
+
+    def cleaner(self):
+        interval = self.args.th_clean
+        while True:
+            time.sleep(interval)
+            ndirs = 0
+            for vol, histpath in self.asrv.vfs.histtab.items():
+                if histpath.startswith(vol):
+                    self.log("\033[Jcln {}/\033[A".format(histpath))
+                else:
+                    self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
+
+                ndirs += self.clean(histpath)
+
+            self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
+
+    def clean(self, histpath):
+        thumbpath = os.path.join(histpath, "th")
+        # self.log("cln {}".format(thumbpath))
+        maxage = self.args.th_maxage
+        now = time.time()
+        prev_b64 = None
+        prev_fp = None
+        try:
+            ents = os.listdir(thumbpath)
+        except:
+            return 0
+
+        ndirs = 0
+        for f in sorted(ents):
+            fp = os.path.join(thumbpath, f)
+            cmp = fp.lower().replace("\\", "/")
+
+            # "top" or b64 prefix/full (a folder)
+            if len(f) <= 3 or len(f) == 24:
+                age = now - os.path.getmtime(fp)
+                if age > maxage:
+                    with self.mutex:
+                        safe = True
+                        for k in self.busy.keys():
+                            if k.lower().replace("\\", "/").startswith(cmp):
+                                safe = False
+                                break
+
+                        if safe:
+                            ndirs += 1
+                            self.log("rm -rf [{}]".format(fp))
+                            shutil.rmtree(fp, ignore_errors=True)
+                else:
+                    ndirs += self.clean(fp)
+                continue
+
+            # thumb file
+            try:
+                b64, ts, ext = f.split(".")
+                if len(b64) != 24 or len(ts) != 8 or ext not in ["jpg", "webp"]:
+                    raise Exception()
+
+                ts = int(ts, 16)
+            except:
+                if f != "dir.txt":
+                    self.log("foreign file in thumbs dir: [{}]".format(fp), 1)
+
+                continue
+
+            if b64 == prev_b64:
+                self.log("rm replaced [{}]".format(fp))
+                os.unlink(prev_fp)
+
+            prev_b64 = b64
+            prev_fp = fp
+
+        return ndirs

copyparty/u2idx.py

@@ -0,0 +1,271 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import re
+import os
+import time
+import threading
+from datetime import datetime
+
+from .util import s3dec, Pebkac, min_ex
+from .up2k import up2k_wark_from_hashlist
+
+
+try:
+    HAVE_SQLITE3 = True
+    import sqlite3
+except:
+    HAVE_SQLITE3 = False
+
+
+class U2idx(object):
+    def __init__(self, conn):
+        self.log_func = conn.log_func
+        self.asrv = conn.asrv
+        self.args = conn.args
+        self.timeout = self.args.srch_time
+
+        if not HAVE_SQLITE3:
+            self.log("your python does not have sqlite3; searching will be disabled")
+            return
+
+        self.cur = {}
+        self.mem_cur = sqlite3.connect(":memory:")
+        self.mem_cur.execute(r"create table a (b text)")
+
+        self.p_end = None
+        self.p_dur = 0
+
+    def log(self, msg, c=0):
+        self.log_func("u2idx", msg, c)
+
+    def fsearch(self, vols, body):
+        """search by up2k hashlist"""
+        if not HAVE_SQLITE3:
+            return []
+
+        fsize = body["size"]
+        fhash = body["hash"]
+        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
+
+        uq = "where substr(w,1,16) = ? and w = ?"
+        uv = [wark[:16], wark]
+
+        try:
+            return self.run_query(vols, uq, uv)[0]
+        except:
+            raise Pebkac(500, min_ex())
+
+    def get_cur(self, ptop):
+        if not HAVE_SQLITE3:
+            return None
+
+        cur = self.cur.get(ptop)
+        if cur:
+            return cur
+
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
+        if not os.path.exists(db_path):
+            return None
+
+        cur = sqlite3.connect(db_path, 2).cursor()
+        self.cur[ptop] = cur
+        return cur
+
+    def search(self, vols, uq):
+        """search by query params"""
+        if not HAVE_SQLITE3:
+            return []
+
+        q = ""
+        va = []
+        joins = ""
+        is_key = True
+        is_size = False
+        is_date = False
+        kw_key = ["(", ")", "and ", "or ", "not "]
+        kw_val = ["==", "=", "!=", ">", ">=", "<", "<=", "like "]
+        ptn_mt = re.compile(r"^\.?[a-z]+$")
+        mt_ctr = 0
+        mt_keycmp = "substr(up.w,1,16)"
+        mt_keycmp2 = None
+
+        while True:
+            uq = uq.strip()
+            if not uq:
+                break
+
+            ok = False
+            for kw in kw_key + kw_val:
+                if uq.startswith(kw):
+                    is_key = kw in kw_key
+                    uq = uq[len(kw) :]
+                    ok = True
+                    q += kw
+                    break
+
+            if ok:
+                continue
+
+            v, uq = (uq + " ").split(" ", 1)
+            if is_key:
+                is_key = False
+
+                if v == "size":
+                    v = "up.sz"
+                    is_size = True
+
+                elif v == "date":
+                    v = "up.mt"
+                    is_date = True
+
+                elif v == "path":
+                    v = "up.rd"
+
+                elif v == "name":
+                    v = "up.fn"
+
+                elif v == "tags" or ptn_mt.match(v):
+                    mt_ctr += 1
+                    mt_keycmp2 = "mt{}.w".format(mt_ctr)
+                    joins += "inner join mt mt{} on {} = {} ".format(
+                        mt_ctr, mt_keycmp, mt_keycmp2
+                    )
+                    mt_keycmp = mt_keycmp2
+                    if v == "tags":
+                        v = "mt{0}.v".format(mt_ctr)
+                    else:
+                        v = "+mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
+
+                else:
+                    raise Pebkac(400, "invalid key [" + v + "]")
+
+                q += v + " "
+                continue
+
+            head = ""
+            tail = ""
+
+            if is_date:
+                is_date = False
+                v = v.upper().rstrip("Z").replace(",", " ").replace("T", " ")
+                while "  " in v:
+                    v = v.replace("  ", " ")
+
+                for fmt in [
+                    "%Y-%m-%d %H:%M:%S",
+                    "%Y-%m-%d %H:%M",
+                    "%Y-%m-%d %H",
+                    "%Y-%m-%d",
+                ]:
+                    try:
+                        v = datetime.strptime(v, fmt).timestamp()
+                        break
+                    except:
+                        pass
+
+            elif is_size:
+                is_size = False
+                v = int(float(v) * 1024 * 1024)
+
+            else:
+                if v.startswith("*"):
+                    head = "'%'||"
+                    v = v[1:]
+
+                if v.endswith("*"):
+                    tail = "||'%'"
+                    v = v[:-1]
+
+            q += " {}?{} ".format(head, tail)
+            va.append(v)
+            is_key = True
+
+        try:
+            return self.run_query(vols, joins + "where " + q, va)
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
+
+    def run_query(self, vols, uq, uv):
+        done_flag = []
+        self.active_id = "{:.6f}_{}".format(
+            time.time(), threading.current_thread().ident
+        )
+        thr = threading.Thread(
+            target=self.terminator,
+            args=(
+                self.active_id,
+                done_flag,
+            ),
+            name="u2idx-terminator",
+        )
+        thr.daemon = True
+        thr.start()
+
+        if not uq or not uv:
+            q = "select * from up"
+            v = ()
+        else:
+            q = "select up.* from up " + uq
+            v = tuple(uv)
+
+        self.log("qs: {!r} {!r}".format(q, v))
+
+        ret = []
+        lim = 1000
+        taglist = {}
+        for (vtop, ptop, flags) in vols:
+            cur = self.get_cur(ptop)
+            if not cur:
+                continue
+
+            self.active_cur = cur
+
+            sret = []
+            c = cur.execute(q, v)
+            for hit in c:
+                w, ts, sz, rd, fn = hit
+                lim -= 1
+                if lim <= 0:
+                    break
+
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
+                rp = "/".join([x for x in [vtop, rd, fn] if x])
+                sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
+
+            for hit in sret:
+                w = hit["w"]
+                del hit["w"]
+                tags = {}
+                q2 = "select k, v from mt where w = ? and k != 'x'"
+                for k, v2 in cur.execute(q2, (w,)):
+                    taglist[k] = True
+                    tags[k] = v2
+
+                hit["tags"] = tags
+
+            ret.extend(sret)
+            # print("[{}] {}".format(ptop, sret))
+
+        done_flag.append(True)
+        self.active_id = None
+
+        # undupe hits from multiple metadata keys
+        if len(ret) > 1:
+            ret = [ret[0]] + [
+                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+            ]
+
+        return ret, list(taglist.keys())
+
+    def terminator(self, identifier, done_flag):
+        for _ in range(self.timeout):
+            time.sleep(1)
+            if done_flag:
+                return
+
+        if identifier == self.active_id:
+            self.active_cur.connection.interrupt()

copyparty/util.py

@@ -2,17 +2,22 @@
 from __future__ import print_function, unicode_literals
 
 import re
+import os
 import sys
 import time
 import base64
+import select
 import struct
 import hashlib
 import platform
+import traceback
 import threading
 import mimetypes
+import contextlib
 import subprocess as sp  # nosec
+from datetime import datetime
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -30,10 +35,12 @@ if not PY2:
     from urllib.parse import unquote_to_bytes as unquote
     from urllib.parse import quote_from_bytes as quote
     from queue import Queue
+    from io import BytesIO
 else:
     from urllib import unquote  # pylint: disable=no-name-in-module
     from urllib import quote  # pylint: disable=no-name-in-module
     from Queue import Queue  # pylint: disable=import-error,no-name-in-module
+    from StringIO import StringIO as BytesIO
 
 surrogateescape.register_surrogateescape()
 FS_ENCODING = sys.getfilesystemencoding()
@@ -41,10 +48,14 @@ if WINDOWS and PY2:
     FS_ENCODING = "utf-8"
 
 
+HTTP_TS_FMT = "%a, %d %b %Y %H:%M:%S GMT"
+
+
 HTTPCODE = {
     200: "OK",
     204: "No Content",
     206: "Partial Content",
+    302: "Found",
     304: "Not Modified",
     400: "Bad Request",
     403: "Forbidden",
@@ -53,11 +64,65 @@ HTTPCODE = {
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
 }
 
 
+IMPLICATIONS = [
+    ["e2dsa", "e2ds"],
+    ["e2ds", "e2d"],
+    ["e2tsr", "e2ts"],
+    ["e2ts", "e2t"],
+    ["e2t", "e2d"],
+]
+
+
+MIMES = {
+    "md": "text/plain; charset=UTF-8",
+    "opus": "audio/ogg; codecs=opus",
+    "webp": "image/webp",
+}
+
+
+REKOBO_KEY = {
+    v: ln.split(" ", 1)[0]
+    for ln in """
+1B 6d B
+2B 7d Gb F#
+3B 8d Db C#
+4B 9d Ab G#
+5B 10d Eb D#
+6B 11d Bb A#
+7B 12d F
+8B 1d C
+9B 2d G
+10B 3d D
+11B 4d A
+12B 5d E
+1A 6m Abm G#m
+2A 7m Ebm D#m
+3A 8m Bbm A#m
+4A 9m Fm
+5A 10m Cm
+6A 11m Gm
+7A 12m Dm
+8A 1m Am
+9A 2m Em
+10A 3m Bm
+11A 4m Gbm F#m
+12A 5m Dbm C#m
+""".strip().split(
+        "\n"
+    )
+    for v in ln.strip().split(" ")[1:]
+    if v
+}
+
+REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
+
+
 class Counter(object):
     def __init__(self, v=0):
         self.v = v
@@ -72,6 +137,32 @@ class Counter(object):
             self.v = absval
 
 
+class Cooldown(object):
+    def __init__(self, maxage):
+        self.maxage = maxage
+        self.mutex = threading.Lock()
+        self.hist = {}
+        self.oldest = 0
+
+    def poke(self, key):
+        with self.mutex:
+            now = time.time()
+
+            ret = False
+            v = self.hist.get(key, 0)
+            if now - v > self.maxage:
+                self.hist[key] = now
+                ret = True
+
+            if self.oldest - now > self.maxage * 2:
+                self.hist = {
+                    k: v for k, v in self.hist.items() if now - v < self.maxage
+                }
+                self.oldest = sorted(self.hist.values())[0]
+
+            return ret
+
+
 class Unrecv(object):
     """
     undo any number of socket recv ops
@@ -96,6 +187,191 @@ class Unrecv(object):
         self.buf = buf + self.buf
 
 
+class ProgressPrinter(threading.Thread):
+    """
+    periodically print progress info without linefeeds
+    """
+
+    def __init__(self):
+        threading.Thread.__init__(self, name="pp")
+        self.daemon = True
+        self.msg = None
+        self.end = False
+        self.start()
+
+    def run(self):
+        msg = None
+        while not self.end:
+            time.sleep(0.1)
+            if msg == self.msg or self.end:
+                continue
+
+            msg = self.msg
+            uprint(" {}\033[K\r".format(msg))
+            if PY2:
+                sys.stdout.flush()
+
+        print("\033[K", end="")
+        sys.stdout.flush()  # necessary on win10 even w/ stderr btw
+
+
+def uprint(msg):
+    try:
+        print(msg, end="")
+    except UnicodeEncodeError:
+        try:
+            print(msg.encode("utf-8", "replace").decode(), end="")
+        except:
+            print(msg.encode("ascii", "replace").decode(), end="")
+
+
+def nuprint(msg):
+    uprint("{}\n".format(msg))
+
+
+def rice_tid():
+    tid = threading.current_thread().ident
+    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
+
+
+def trace(*args, **kwargs):
+    t = time.time()
+    stack = "".join(
+        "\033[36m{}\033[33m{}".format(x[0].split(os.sep)[-1][:-3], x[1])
+        for x in traceback.extract_stack()[3:-1]
+    )
+    parts = ["{:.6f}".format(t), rice_tid(), stack]
+
+    if args:
+        parts.append(repr(args))
+
+    if kwargs:
+        parts.append(repr(kwargs))
+
+    msg = "\033[0m ".join(parts)
+    # _tracebuf.append(msg)
+    nuprint(msg)
+
+
+def alltrace():
+    threads = {}
+    names = dict([(t.ident, t.name) for t in threading.enumerate()])
+    for tid, stack in sys._current_frames().items():
+        name = "{} ({:x})".format(names.get(tid), tid)
+        threads[name] = stack
+
+    rret = []
+    bret = []
+    for name, stack in sorted(threads.items()):
+        ret = ["\n\n# {}".format(name)]
+        pad = None
+        for fn, lno, name, line in traceback.extract_stack(stack):
+            fn = os.sep.join(fn.split(os.sep)[-3:])
+            ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
+            if line:
+                ret.append("  " + str(line.strip()))
+                if "self.not_empty.wait()" in line:
+                    pad = " " * 4
+
+        if pad:
+            bret += [ret[0]] + [pad + x for x in ret[1:]]
+        else:
+            rret += ret
+
+    return "\n".join(rret + bret)
+
+
+def min_ex():
+    et, ev, tb = sys.exc_info()
+    tb = traceback.extract_tb(tb, 2)
+    ex = [
+        "{} @ {} <{}>: {}".format(fp.split(os.sep)[-1], ln, fun, txt)
+        for fp, ln, fun, txt in tb
+    ]
+    ex.append("{}: {}".format(et.__name__, ev))
+    return "\n".join(ex)
+
+
+@contextlib.contextmanager
+def ren_open(fname, *args, **kwargs):
+    fdir = kwargs.pop("fdir", None)
+    suffix = kwargs.pop("suffix", None)
+
+    if fname == os.devnull:
+        with open(fname, *args, **kwargs) as f:
+            yield {"orz": [f, fname]}
+            return
+
+    if suffix:
+        ext = fname.split(".")[-1]
+        if len(ext) < 7:
+            suffix += "." + ext
+
+    orig_name = fname
+    bname = fname
+    ext = ""
+    while True:
+        ofs = bname.rfind(".")
+        if ofs < 0 or ofs < len(bname) - 7:
+            # doesn't look like an extension anymore
+            break
+
+        ext = bname[ofs:] + ext
+        bname = bname[:ofs]
+
+    b64 = ""
+    while True:
+        try:
+            if fdir:
+                fpath = os.path.join(fdir, fname)
+            else:
+                fpath = fname
+
+            if suffix and os.path.exists(fsenc(fpath)):
+                fpath += suffix
+                fname += suffix
+                ext += suffix
+
+            with open(fsenc(fpath), *args, **kwargs) as f:
+                if b64:
+                    fp2 = "fn-trunc.{}.txt".format(b64)
+                    fp2 = os.path.join(fdir, fp2)
+                    with open(fsenc(fp2), "wb") as f2:
+                        f2.write(orig_name.encode("utf-8"))
+
+                yield {"orz": [f, fname]}
+                return
+
+        except OSError as ex_:
+            ex = ex_
+            if ex.errno not in [36, 63] and (not WINDOWS or ex.errno != 22):
+                raise
+
+        if not b64:
+            b64 = (bname + ext).encode("utf-8", "replace")
+            b64 = hashlib.sha512(b64).digest()[:12]
+            b64 = base64.urlsafe_b64encode(b64).decode("utf-8")
+
+        badlen = len(fname)
+        while len(fname) >= badlen:
+            if len(bname) < 8:
+                raise ex
+
+            if len(bname) > len(ext):
+                # drop the last letter of the filename
+                bname = bname[:-1]
+            else:
+                try:
+                    # drop the leftmost sub-extension
+                    _, ext = ext.split(".", 1)
+                except:
+                    # okay do the first letter then
+                    ext = "." + ext[2:]
+
+            fname = "{}~{}{}".format(bname, b64, ext)
+
+
 class MultipartParser(object):
     def __init__(self, log_func, sr, http_headers):
         self.sr = sr
@@ -331,8 +607,10 @@ def read_header(sr):
             else:
                 continue
 
-        sr.unrecv(ret[ofs + 4 :])
-        return ret[:ofs].decode("utf-8", "surrogateescape").split("\r\n")
+        if len(ret) > ofs + 4:
+            sr.unrecv(ret[ofs + 4 :])
+
+        return ret[:ofs].decode("utf-8", "surrogateescape").lstrip("\r\n").split("\r\n")
 
 
 def humansize(sz, terse=False):
@@ -360,6 +638,26 @@ def get_spd(nbyte, t0, t=None):
     return "{} \033[0m{}/s\033[0m".format(s1, s2)
 
 
+def s2hms(s, optional_h=False):
+    s = int(s)
+    h, s = divmod(s, 3600)
+    m, s = divmod(s, 60)
+    if not h and optional_h:
+        return "{}:{:02}".format(m, s)
+
+    return "{}:{:02}:{:02}".format(h, m, s)
+
+
+def uncyg(path):
+    if len(path) < 2 or not path.startswith("/"):
+        return path
+
+    if len(path) > 2 and path[2] != "/":
+        return path
+
+    return "{}:\\{}".format(path[1], path[3:])
+
+
 def undot(path):
     ret = []
     for node in path.split("/"):
@@ -376,11 +674,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
-    fn = fn.replace("\\", "/").split("/")[-1]
+def sanitize_fn(fn, ok="", bad=[]):
+    if "/" not in ok:
+        fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -390,36 +689,54 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
-            fn = fn.replace(bad, good)
+        ]
+        for a, b in [x for x in remap if x[0] not in ok]:
+            fn = fn.replace(a, b)
 
-        bad = ["con", "prn", "aux", "nul"]
+        bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):
             bad += "com{0} lpt{0}".format(n).split(" ")
 
-        if fn.lower() in bad:
-            fn = "_" + fn
+    if fn.lower() in bad:
+        fn = "_" + fn
 
     return fn.strip()
 
 
+def u8safe(txt):
+    try:
+        return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
+    except:
+        return txt.encode("utf-8", "replace").decode("utf-8", "replace")
+
+
 def exclude_dotfiles(filepaths):
-    for fpath in filepaths:
-        if not fpath.split("/")[-1].startswith("."):
-            yield fpath
+    return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def http_ts(ts):
+    file_dt = datetime.utcfromtimestamp(ts)
+    return file_dt.strftime(HTTP_TS_FMT)
+
+
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 
@@ -459,6 +776,16 @@ def w8enc(txt):
     return txt.encode(FS_ENCODING, "surrogateescape")
 
 
+def w8b64dec(txt):
+    """decodes base64(filesystem-bytes) to wtf8"""
+    return w8dec(base64.urlsafe_b64decode(txt.encode("ascii")))
+
+
+def w8b64enc(txt):
+    """encodes wtf8 to base64(filesystem-bytes)"""
+    return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii")
+
+
 if PY2 and WINDOWS:
     # moonrunes become \x3f with bytestrings,
     # losing mojibake support is worth
@@ -472,6 +799,43 @@ else:
     fsdec = w8dec
 
 
+def s3enc(mem_cur, rd, fn):
+    ret = []
+    for v in [rd, fn]:
+        try:
+            mem_cur.execute("select * from a where b = ?", (v,))
+            ret.append(v)
+        except:
+            ret.append("//" + w8b64enc(v))
+            # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+    return tuple(ret)
+
+
+def s3dec(rd, fn):
+    ret = []
+    for k, v in [["d", rd], ["f", fn]]:
+        if v.startswith("//"):
+            ret.append(w8b64dec(v[2:]))
+            # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+        else:
+            ret.append(v)
+
+    return tuple(ret)
+
+
+def atomic_move(src, dst):
+    src = fsenc(src)
+    dst = fsenc(dst)
+    if not PY2:
+        os.replace(src, dst)
+    else:
+        if os.path.exists(dst):
+            os.unlink(dst)
+
+        os.rename(src, dst)
+
+
 def read_socket(sr, total_size):
     remains = total_size
     while remains > 0:
@@ -496,25 +860,142 @@ def read_socket_unbounded(sr):
         yield buf
 
 
+def read_socket_chunked(sr, log=None):
+    err = "expected chunk length, got [{}] |{}| instead"
+    while True:
+        buf = b""
+        while b"\r" not in buf:
+            rbuf = sr.recv(2)
+            if not rbuf or len(buf) > 16:
+                err = err.format(buf.decode("utf-8", "replace"), len(buf))
+                raise Pebkac(400, err)
+
+            buf += rbuf
+
+        if not buf.endswith(b"\n"):
+            sr.recv(1)
+
+        try:
+            chunklen = int(buf.rstrip(b"\r\n"), 16)
+        except:
+            err = err.format(buf.decode("utf-8", "replace"), len(buf))
+            raise Pebkac(400, err)
+
+        if chunklen == 0:
+            sr.recv(2)  # \r\n after final chunk
+            return
+
+        if log:
+            log("receiving {} byte chunk".format(chunklen))
+
+        for chunk in read_socket(sr, chunklen):
+            yield chunk
+
+        sr.recv(2)  # \r\n after each chunk too
+
+
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
-    u32_lim = int((2 ** 31) * 0.9)
+    is_mp = actor.is_mp
     hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
-        actor.workload += 1
-        if actor.workload > u32_lim:
-            actor.workload = 100  # prevent overflow
+        if is_mp:
+            actor.workload += 1
+            if actor.workload > 2 ** 31:
+                actor.workload = 100
 
         tlen += len(buf)
         hashobj.update(buf)
         fout.write(buf)
 
-    digest32 = hashobj.digest()[:32]
-    digest_b64 = base64.urlsafe_b64encode(digest32).decode("utf-8").rstrip("=")
+    digest = hashobj.digest()[:33]
+    digest_b64 = base64.urlsafe_b64encode(digest).decode("utf-8")
 
     return tlen, hashobj.hexdigest(), digest_b64
 
 
+def sendfile_py(lower, upper, f, s, actor=None):
+    remains = upper - lower
+    f.seek(lower)
+    while remains > 0:
+        if actor:
+            actor.workload += 1
+            if actor.workload > 2 ** 31:
+                actor.workload = 100
+
+        # time.sleep(0.01)
+        buf = f.read(min(1024 * 32, remains))
+        if not buf:
+            return remains
+
+        try:
+            s.sendall(buf)
+            remains -= len(buf)
+        except:
+            return remains
+
+    return 0
+
+
+def sendfile_kern(lower, upper, f, s):
+    out_fd = s.fileno()
+    in_fd = f.fileno()
+    ofs = lower
+    while ofs < upper:
+        try:
+            req = min(2 ** 30, upper - ofs)
+            select.select([], [out_fd], [], 10)
+            n = os.sendfile(out_fd, in_fd, ofs, req)
+        except Exception as ex:
+            # print("sendfile: " + repr(ex))
+            n = 0
+
+        if n <= 0:
+            return upper - ofs
+
+        ofs += n
+        # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
+
+    return 0
+
+
+def statdir(logger, scandir, lstat, top):
+    try:
+        btop = fsenc(top)
+        if scandir and hasattr(os, "scandir"):
+            src = "scandir"
+            with os.scandir(btop) as dh:
+                for fh in dh:
+                    try:
+                        yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
+                    except Exception as ex:
+                        msg = "scan-stat: \033[36m{} @ {}"
+                        logger(msg.format(repr(ex), fsdec(fh.path)))
+        else:
+            src = "listdir"
+            fun = os.lstat if lstat else os.stat
+            for name in os.listdir(btop):
+                abspath = os.path.join(btop, name)
+                try:
+                    yield [fsdec(name), fun(abspath)]
+                except Exception as ex:
+                    msg = "list-stat: \033[36m{} @ {}"
+                    logger(msg.format(repr(ex), fsdec(abspath)))
+
+    except Exception as ex:
+        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
+
+
 def unescape_cookie(orig):
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""
@@ -543,18 +1024,20 @@ def unescape_cookie(orig):
     return ret
 
 
-def guess_mime(url):
-    if url.endswith(".md"):
-        return ["text/plain; charset=UTF-8"]
+def guess_mime(url, fallback="application/octet-stream"):
+    try:
+        _, ext = url.rsplit(".", 1)
+    except:
+        return fallback
 
-    return mimetypes.guess_type(url)
+    return MIMES.get(ext) or mimetypes.guess_type(url)[0] or fallback
 
 
 def runcmd(*argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
-    stdout = stdout.decode("utf-8")
-    stderr = stderr.decode("utf-8")
+    stdout = stdout.decode("utf-8", "replace")
+    stderr = stderr.decode("utf-8", "replace")
     return [p.returncode, stdout, stderr]
 
 
@@ -566,10 +1049,25 @@ def chkcmd(*argv):
     return sout, serr
 
 
+def mchkcmd(argv, timeout=10):
+    if PY2:
+        with open(os.devnull, "wb") as f:
+            rv = sp.call(argv, stdout=f, stderr=f)
+    else:
+        rv = sp.call(argv, stdout=sp.DEVNULL, stderr=sp.DEVNULL, timeout=timeout)
+
+    if rv:
+        raise sp.CalledProcessError(rv, (argv[0], b"...", argv[-1]))
+
+
 def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
-        return struct.unpack(b"I", f.read(4))[0]
+        rv = f.read(4)
+        try:
+            return struct.unpack(b"I", rv)[0]
+        except:
+            return struct.unpack("I", rv)[0]
 
 
 def py_desc():
@@ -579,7 +1077,11 @@ def py_desc():
     if ofs > 0:
         py_ver = py_ver[:ofs]
 
-    bitness = struct.calcsize(b"P") * 8
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
     host_os = platform.system()
     compiler = platform.python_compiler()
 
@@ -591,7 +1093,26 @@ def py_desc():
     )
 
 
+def align_tab(lines):
+    rows = []
+    ncols = 0
+    for ln in lines:
+        row = [x for x in ln.split(" ") if x]
+        ncols = max(ncols, len(row))
+        rows.append(row)
+
+    lens = [0] * ncols
+    for row in rows:
+        for n, col in enumerate(row):
+            lens[n] = max(lens[n], len(col))
+
+    return ["".join(x.ljust(y + 2) for x, y in zip(row, lens)) for row in rows]
+
+
 class Pebkac(Exception):
     def __init__(self, code, msg=None):
         super(Pebkac, self).__init__(msg or HTTPCODE[code])
         self.code = code
+
+    def __repr__(self):
+        return "Pebkac({}, {})".format(self.code, repr(self.args))

copyparty/web/Makefile

@@ -0,0 +1,12 @@
+# run me to zopfli all the static files
+# which should help on really slow connections
+# but then why are you using copyparty in the first place
+
+pk: $(addsuffix .gz, $(wildcard *.js *.css))
+un: $(addsuffix .un, $(wildcard *.gz))
+
+%.gz: %
+	pigz -11 -J 34 -I 5730 $<
+
+%.un: %
+	pigz -d $<

copyparty/web/baguettebox.js

@@ -0,0 +1,583 @@
+/*!
+ * baguetteBox.js
+ * @author  feimosi
+ * @version 1.11.1-mod
+ * @url https://github.com/feimosi/baguetteBox.js
+ */
+
+window.baguetteBox = (function () {
+    'use strict';
+
+    var options = {},
+        defaults = {
+            captions: true,
+            buttons: 'auto',
+            noScrollbars: false,
+            bodyClass: 'baguetteBox-open',
+            titleTag: false,
+            async: false,
+            preload: 2,
+            animation: 'slideIn',
+            afterShow: null,
+            afterHide: null,
+            onChange: null,
+        },
+        overlay, slider, previousButton, nextButton, closeButton,
+        currentGallery = [],
+        currentIndex = 0,
+        isOverlayVisible = false,
+        touch = {},  // start-pos
+        touchFlag = false,  // busy
+        regex = /.+\.(gif|jpe?g|png|webp)/i,
+        data = {},  // all galleries
+        imagesElements = [],
+        documentLastFocus = null;
+
+    var overlayClickHandler = function (event) {
+        if (event.target.id.indexOf('baguette-img') !== -1) {
+            hideOverlay();
+        }
+    };
+
+    var touchstartHandler = function (event) {
+        touch.count++;
+        if (touch.count > 1) {
+            touch.multitouch = true;
+        }
+        touch.startX = event.changedTouches[0].pageX;
+        touch.startY = event.changedTouches[0].pageY;
+    };
+    var touchmoveHandler = function (event) {
+        if (touchFlag || touch.multitouch) {
+            return;
+        }
+        event.preventDefault ? event.preventDefault() : event.returnValue = false;
+        var touchEvent = event.touches[0] || event.changedTouches[0];
+        if (touchEvent.pageX - touch.startX > 40) {
+            touchFlag = true;
+            showPreviousImage();
+        } else if (touchEvent.pageX - touch.startX < -40) {
+            touchFlag = true;
+            showNextImage();
+        } else if (touch.startY - touchEvent.pageY > 100) {
+            hideOverlay();
+        }
+    };
+    var touchendHandler = function () {
+        touch.count--;
+        if (touch.count <= 0) {
+            touch.multitouch = false;
+        }
+        touchFlag = false;
+    };
+    var contextmenuHandler = function () {
+        touchendHandler();
+    };
+
+    var trapFocusInsideOverlay = function (event) {
+        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(event.target))) {
+            event.stopPropagation();
+            initFocus();
+        }
+    };
+
+    function run(selector, userOptions) {
+        buildOverlay();
+        removeFromCache(selector);
+        return bindImageClickListeners(selector, userOptions);
+    }
+
+    function bindImageClickListeners(selector, userOptions) {
+        var galleryNodeList = document.querySelectorAll(selector);
+        var selectorData = {
+            galleries: [],
+            nodeList: galleryNodeList
+        };
+        data[selector] = selectorData;
+
+        [].forEach.call(galleryNodeList, function (galleryElement) {
+            if (userOptions && userOptions.filter) {
+                regex = userOptions.filter;
+            }
+
+            var tagsNodeList = [];
+            if (galleryElement.tagName === 'A') {
+                tagsNodeList = [galleryElement];
+            } else {
+                tagsNodeList = galleryElement.getElementsByTagName('a');
+            }
+
+            tagsNodeList = [].filter.call(tagsNodeList, function (element) {
+                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1) {
+                    return regex.test(element.href);
+                }
+            });
+            if (tagsNodeList.length === 0) {
+                return;
+            }
+
+            var gallery = [];
+            [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
+                var imageElementClickHandler = function (event) {
+                    if (event && event.ctrlKey)
+                        return true;
+
+                    event.preventDefault ? event.preventDefault() : event.returnValue = false;
+                    prepareOverlay(gallery, userOptions);
+                    showOverlay(imageIndex);
+                };
+                var imageItem = {
+                    eventHandler: imageElementClickHandler,
+                    imageElement: imageElement
+                };
+                bind(imageElement, 'click', imageElementClickHandler);
+                gallery.push(imageItem);
+            });
+            selectorData.galleries.push(gallery);
+        });
+
+        return selectorData.galleries;
+    }
+
+    function clearCachedData() {
+        for (var selector in data) {
+            if (data.hasOwnProperty(selector)) {
+                removeFromCache(selector);
+            }
+        }
+    }
+
+    function removeFromCache(selector) {
+        if (!data.hasOwnProperty(selector)) {
+            return;
+        }
+        var galleries = data[selector].galleries;
+        [].forEach.call(galleries, function (gallery) {
+            [].forEach.call(gallery, function (imageItem) {
+                unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
+            });
+
+            if (currentGallery === gallery) {
+                currentGallery = [];
+            }
+        });
+
+        delete data[selector];
+    }
+
+    function buildOverlay() {
+        overlay = ebi('baguetteBox-overlay');
+        if (overlay) {
+            slider = ebi('baguetteBox-slider');
+            previousButton = ebi('previous-button');
+            nextButton = ebi('next-button');
+            closeButton = ebi('close-button');
+            return;
+        }
+        overlay = mknod('div');
+        overlay.setAttribute('role', 'dialog');
+        overlay.id = 'baguetteBox-overlay';
+        document.getElementsByTagName('body')[0].appendChild(overlay);
+
+        slider = mknod('div');
+        slider.id = 'baguetteBox-slider';
+        overlay.appendChild(slider);
+
+        previousButton = mknod('button');
+        previousButton.setAttribute('type', 'button');
+        previousButton.id = 'previous-button';
+        previousButton.setAttribute('aria-label', 'Previous');
+        previousButton.innerHTML = '&lt;';
+        overlay.appendChild(previousButton);
+
+        nextButton = mknod('button');
+        nextButton.setAttribute('type', 'button');
+        nextButton.id = 'next-button';
+        nextButton.setAttribute('aria-label', 'Next');
+        nextButton.innerHTML = '&gt;';
+        overlay.appendChild(nextButton);
+
+        closeButton = mknod('button');
+        closeButton.setAttribute('type', 'button');
+        closeButton.id = 'close-button';
+        closeButton.setAttribute('aria-label', 'Close');
+        closeButton.innerHTML = '&times;';
+        overlay.appendChild(closeButton);
+
+        previousButton.className = nextButton.className = closeButton.className = 'baguetteBox-button';
+
+        bindEvents();
+    }
+
+    function keyDownHandler(event) {
+        switch (event.keyCode) {
+            case 37: // Left
+                showPreviousImage();
+                break;
+            case 39: // Right
+                showNextImage();
+                break;
+            case 27: // Esc
+                hideOverlay();
+                break;
+            case 36: // Home
+                showFirstImage(event);
+                break;
+            case 35: // End
+                showLastImage(event);
+                break;
+        }
+    }
+
+    var passiveSupp = false;
+    try {
+        var opts = {
+            get passive() {
+                passiveSupp = true;
+                return false;
+            }
+        };
+        window.addEventListener('test', null, opts);
+        window.removeEventListener('test', null, opts);
+    }
+    catch (ex) {
+        passiveSupp = false;
+    }
+    var passiveEvent = passiveSupp ? { passive: false } : null;
+    var nonPassiveEvent = passiveSupp ? { passive: true } : null;
+
+    function bindEvents() {
+        bind(overlay, 'click', overlayClickHandler);
+        bind(previousButton, 'click', showPreviousImage);
+        bind(nextButton, 'click', showNextImage);
+        bind(closeButton, 'click', hideOverlay);
+        bind(slider, 'contextmenu', contextmenuHandler);
+        bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        bind(overlay, 'touchend', touchendHandler);
+        bind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function unbindEvents() {
+        unbind(overlay, 'click', overlayClickHandler);
+        unbind(previousButton, 'click', showPreviousImage);
+        unbind(nextButton, 'click', showNextImage);
+        unbind(closeButton, 'click', hideOverlay);
+        unbind(slider, 'contextmenu', contextmenuHandler);
+        unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        unbind(overlay, 'touchend', touchendHandler);
+        unbind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function prepareOverlay(gallery, userOptions) {
+        if (currentGallery === gallery) {
+            return;
+        }
+        currentGallery = gallery;
+        setOptions(userOptions);
+        slider.innerHTML = '';
+        imagesElements.length = 0;
+
+        var imagesFiguresIds = [];
+        var imagesCaptionsIds = [];
+        for (var i = 0, fullImage; i < gallery.length; i++) {
+            fullImage = mknod('div');
+            fullImage.className = 'full-image';
+            fullImage.id = 'baguette-img-' + i;
+            imagesElements.push(fullImage);
+
+            imagesFiguresIds.push('baguetteBox-figure-' + i);
+            imagesCaptionsIds.push('baguetteBox-figcaption-' + i);
+            slider.appendChild(imagesElements[i]);
+        }
+        overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
+        overlay.setAttribute('aria-describedby', imagesCaptionsIds.join(' '));
+    }
+
+    function setOptions(newOptions) {
+        if (!newOptions) {
+            newOptions = {};
+        }
+        for (var item in defaults) {
+            options[item] = defaults[item];
+            if (typeof newOptions[item] !== 'undefined') {
+                options[item] = newOptions[item];
+            }
+        }
+        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
+            options.animation === 'slideIn' ? '' : 'none');
+
+        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1)) {
+            options.buttons = false;
+        }
+
+        previousButton.style.display = nextButton.style.display = (options.buttons ? '' : 'none');
+    }
+
+    function showOverlay(chosenImageIndex) {
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'hidden';
+            document.body.style.overflowY = 'scroll';
+        }
+        if (overlay.style.display === 'block') {
+            return;
+        }
+
+        bind(document, 'keydown', keyDownHandler);
+        currentIndex = chosenImageIndex;
+        touch = {
+            count: 0,
+            startX: null,
+            startY: null
+        };
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+
+        updateOffset();
+        overlay.style.display = 'block';
+        // Fade in overlay
+        setTimeout(function () {
+            overlay.className = 'visible';
+            if (options.bodyClass && document.body.classList) {
+                document.body.classList.add(options.bodyClass);
+            }
+            if (options.afterShow) {
+                options.afterShow();
+            }
+        }, 50);
+        if (options.onChange) {
+            options.onChange(currentIndex, imagesElements.length);
+        }
+        documentLastFocus = document.activeElement;
+        initFocus();
+        isOverlayVisible = true;
+    }
+
+    function initFocus() {
+        if (options.buttons) {
+            previousButton.focus();
+        } else {
+            closeButton.focus();
+        }
+    }
+
+    function hideOverlay(e) {
+        ev(e);
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'auto';
+            document.body.style.overflowY = 'auto';
+        }
+        if (overlay.style.display === 'none') {
+            return;
+        }
+
+        unbind(document, 'keydown', keyDownHandler);
+        // Fade out and hide the overlay
+        overlay.className = '';
+        setTimeout(function () {
+            overlay.style.display = 'none';
+            if (options.bodyClass && document.body.classList) {
+                document.body.classList.remove(options.bodyClass);
+            }
+            if (options.afterHide) {
+                options.afterHide();
+            }
+            documentLastFocus && documentLastFocus.focus();
+            isOverlayVisible = false;
+        }, 500);
+    }
+
+    function loadImage(index, callback) {
+        var imageContainer = imagesElements[index];
+        var galleryItem = currentGallery[index];
+
+        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined') {
+            return;  // out-of-bounds or gallery dirty
+        }
+
+        if (imageContainer.getElementsByTagName('img')[0]) {
+            // image is loaded, cb and bail
+            if (callback) {
+                callback();
+            }
+            return;
+        }
+
+        var imageElement = galleryItem.imageElement,
+            imageSrc = imageElement.href,
+            thumbnailElement = imageElement.getElementsByTagName('img')[0],
+            imageCaption = typeof options.captions === 'function' ?
+                options.captions.call(currentGallery, imageElement) :
+                imageElement.getAttribute('data-caption') || imageElement.title;
+
+        var figure = mknod('figure');
+        figure.id = 'baguetteBox-figure-' + index;
+        figure.innerHTML = '<div class="baguetteBox-spinner">' +
+            '<div class="baguetteBox-double-bounce1"></div>' +
+            '<div class="baguetteBox-double-bounce2"></div>' +
+            '</div>';
+
+        if (options.captions && imageCaption) {
+            var figcaption = mknod('figcaption');
+            figcaption.id = 'baguetteBox-figcaption-' + index;
+            figcaption.innerHTML = imageCaption;
+            figure.appendChild(figcaption);
+        }
+        imageContainer.appendChild(figure);
+
+        var image = mknod('img');
+        image.onload = function () {
+            // Remove loader element
+            var spinner = document.querySelector('#baguette-img-' + index + ' .baguetteBox-spinner');
+            figure.removeChild(spinner);
+            if (!options.async && callback) {
+                callback();
+            }
+        };
+        image.setAttribute('src', imageSrc);
+        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (options.titleTag && imageCaption) {
+            image.title = imageCaption;
+        }
+        figure.appendChild(image);
+
+        if (options.async && callback) {
+            callback();
+        }
+    }
+
+    function showNextImage(e) {
+        ev(e);
+        return show(currentIndex + 1);
+    }
+
+    function showPreviousImage(e) {
+        ev(e);
+        return show(currentIndex - 1);
+    }
+
+    function showFirstImage(event) {
+        if (event) {
+            event.preventDefault();
+        }
+        return show(0);
+    }
+
+    function showLastImage(event) {
+        if (event) {
+            event.preventDefault();
+        }
+        return show(currentGallery.length - 1);
+    }
+
+    /**
+     * Move the gallery to a specific index
+     * @param `index` {number} - the position of the image
+     * @param `gallery` {array} - gallery which should be opened, if omitted assumes the currently opened one
+     * @return {boolean} - true on success or false if the index is invalid
+     */
+    function show(index, gallery) {
+        if (!isOverlayVisible && index >= 0 && index < gallery.length) {
+            prepareOverlay(gallery, options);
+            showOverlay(index);
+            return true;
+        }
+        if (index < 0) {
+            if (options.animation) {
+                bounceAnimation('left');
+            }
+            return false;
+        }
+        if (index >= imagesElements.length) {
+            if (options.animation) {
+                bounceAnimation('right');
+            }
+            return false;
+        }
+
+        currentIndex = index;
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+        updateOffset();
+
+        if (options.onChange) {
+            options.onChange(currentIndex, imagesElements.length);
+        }
+
+        return true;
+    }
+
+    /**
+     * Triggers the bounce animation
+     * @param {('left'|'right')} direction - Direction of the movement
+     */
+    function bounceAnimation(direction) {
+        slider.className = 'bounce-from-' + direction;
+        setTimeout(function () {
+            slider.className = '';
+        }, 400);
+    }
+
+    function updateOffset() {
+        var offset = -currentIndex * 100 + '%';
+        if (options.animation === 'fadeIn') {
+            slider.style.opacity = 0;
+            setTimeout(function () {
+                slider.style.transform = 'translate3d(' + offset + ',0,0)';
+                slider.style.opacity = 1;
+            }, 400);
+        } else {
+            slider.style.transform = 'translate3d(' + offset + ',0,0)';
+        }
+    }
+
+    function preloadNext(index) {
+        if (index - currentIndex >= options.preload) {
+            return;
+        }
+        loadImage(index + 1, function () {
+            preloadNext(index + 1);
+        });
+    }
+
+    function preloadPrev(index) {
+        if (currentIndex - index >= options.preload) {
+            return;
+        }
+        loadImage(index - 1, function () {
+            preloadPrev(index - 1);
+        });
+    }
+
+    function bind(element, event, callback, options) {
+        element.addEventListener(event, callback, options);
+    }
+
+    function unbind(element, event, callback, options) {
+        element.removeEventListener(event, callback, options);
+    }
+
+    function destroyPlugin() {
+        unbindEvents();
+        clearCachedData();
+        unbind(document, 'keydown', keyDownHandler);
+        document.getElementsByTagName('body')[0].removeChild(ebi('baguetteBox-overlay'));
+        data = {};
+        currentGallery = [];
+        currentIndex = 0;
+    }
+
+    return {
+        run: run,
+        show: show,
+        showNext: showNextImage,
+        showPrevious: showPreviousImage,
+        hide: hideOverlay,
+        destroy: destroyPlugin
+    };
+})();

copyparty/web/browser.html

@@ -2,78 +2,134 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <title>⇆🎉 {{ title }}</title>
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=0.8">
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
-    {%- if can_upload %}
-    <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
-    {%- endif %}
+	<meta charset="utf-8">
+	<title>⇆🎉 {{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
+	<link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
+	{%- if css %}
+	<link rel="stylesheet" type="text/css" media="screen" href="{{ css }}{{ ts }}">
+	{%- endif %}
 </head>
 
 <body>
-    {%- if can_upload %}
-    {%- include 'upload.html' %}
-    {%- endif %}
-    
-    <h1 id="path">
-        {%- for n in vpnodes %}
-        <a href="/{{ n[0] }}">{{ n[1] }}</a>
-        {%- endfor %}
-    </h1>
-    
-    {%- if can_read %}
-    {%- if prologue %}
-    <div id="pro" class="logue">{{ prologue }}</div>
-    {%- endif %}
+	<div id="ops"></div>
 
-    <table id="files">
-        <thead>
-            <tr>
-                <th></th>
-                <th>File Name</th>
-                <th sort="int">File Size</th>
-                <th>Date</th>
-            </tr>
-        </thead>
-        <tbody>
+	<div id="op_search" class="opview">
+		{%- if have_tags_idx %}
+		<div id="srch_form" class="tags"></div>
+		{%- else %}
+		<div id="srch_form"></div>
+		{%- endif %}
+		<div id="srch_q"></div>
+	</div>
+
+	<div id="op_player" class="opview opbox opwide"></div>
+
+	<div id="op_bup" class="opview opbox act">
+		<div id="u2err"></div>
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="bput" />
+			<input type="file" name="f" multiple><br />
+			<input type="submit" value="start upload">
+		</form>
+	</div>
+
+	<div id="op_mkdir" class="opview opbox act">
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="mkdir" />
+			<input type="text" name="name" size="30">
+			<input type="submit" value="mkdir">
+		</form>
+	</div>
+
+	<div id="op_new_md" class="opview opbox">
+		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="hidden" name="act" value="new_md" />
+			<input type="text" name="name" size="30">
+			<input type="submit" value="create doc">
+		</form>
+	</div>
+
+	<div id="op_msg" class="opview opbox act">
+		<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
+			<input type="text" name="msg" size="30">
+			<input type="submit" value="send msg">
+		</form>
+	</div>
+
+	<div id="op_up2k" class="opview"></div>
+
+	<div id="op_cfg" class="opview opbox opwide"></div>
+	
+	<h1 id="path">
+		<a href="#" id="entree" tt="show directory tree$NHotkey: B">🌲</a>
+		{%- for n in vpnodes %}
+		<a href="/{{ n[0] }}">{{ n[1] }}</a>
+		{%- endfor %}
+	</h1>
+	
+	<div id="tree"></div>
+
+<div id="wrap">
+
+	<div id="pro" class="logue">{{ logues[0] }}</div>
+
+	<table id="files">
+		<thead>
+			<tr>
+				<th name="lead"><span>c</span></th>
+				<th name="href"><span>File Name</span></th>
+				<th name="sz" sort="int"><span>Size</span></th>
+				{%- for k in taglist %}
+					{%- if k.startswith('.') %}
+				<th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
+					{%- else %}
+				<th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
+					{%- endif %}
+				{%- endfor %}
+				<th name="ext"><span>T</span></th>
+				<th name="ts"><span>Date</span></th>
+			</tr>
+		</thead>
+<tbody>
 
 {%- for f in files %}
-<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td>
+	{%- if f.tags is defined %}
+		{%- for k in taglist %}
+<td>{{ f.tags[k] }}</td>
+		{%- endfor %}
+	{%- endif %}
+<td>{{ f.ext }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
-        </tbody>
-    </table>
-    
-    {%- if epilogue %}
-    <div id="epi" class="logue">{{ epilogue }}</div>
-    {%- endif %}
-    {%- endif %}
+		</tbody>
+	</table>
+	
+	<div id="epi" class="logue">{{ logues[1] }}</div>
 
-    <h2><a href="?h">control-panel</a></h2>
+	<h2><a href="?h">control-panel</a></h2>
 
-    {%- if srv_info %}
-    <div id="srv_info"><span>{{ srv_info }}</span></div>
-    {%- endif %}
+</div>
 
-    <div id="widget">
-        <div id="wtoggle">♫</div>
-        <div id="widgeti">
-            <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
-            <canvas id="pvol" width="288" height="38"></canvas>
-            <canvas id="barpos"></canvas>
-            <canvas id="barbuf"></canvas>
-        </div>
-    </div>
-    
-    {%- if can_read %}
-    <script src="/.cpr/browser.js{{ ts }}"></script>
-    {%- endif %}
-    
-    {%- if can_upload %}
-    <script src="/.cpr/up2k.js{{ ts }}"></script>
-    {%- endif %}
+	{%- if srv_info %}
+	<div id="srv_info"><span>{{ srv_info }}</span></div>
+	{%- endif %}
+
+	<div id="widget"></div>
+
+	<script>
+		var perms = {{ perms }},
+			tag_order_cfg = {{ tag_order }},
+			have_up2k_idx = {{ have_up2k_idx|tojson }},
+			have_tags_idx = {{ have_tags_idx|tojson }},
+			have_zip = {{ have_zip|tojson }};
+	</script>
+	<script src="/.cpr/util.js{{ ts }}"></script>
+	<script src="/.cpr/browser.js{{ ts }}"></script>
+	<script src="/.cpr/up2k.js{{ ts }}"></script>
 </body>
 
 </html>

copyparty/web/browser2.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>{{ title }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<style>
+		html{font-family:sans-serif}
+		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
+		a{display:block}
+	</style>
+</head>
+
+<body>
+	{%- if srv_info %}
+	<p><span>{{ srv_info }}</span></p>
+	{%- endif %}
+
+	{%- if have_b_u %}
+	<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+		<input type="hidden" name="act" value="bput" />
+		<input type="file" name="f" multiple /><br />
+		<input type="submit" value="start upload" />
+	</form>
+	<br />
+	{%- endif %}
+
+	{%- if logues[0] %}
+	<div>{{ logues[0] }}</div><br />
+	{%- endif %}
+
+	<table id="files">
+		<thead>
+			<tr>
+				<th name="lead"><span>c</span></th>
+				<th name="href"><span>File Name</span></th>
+				<th name="sz" sort="int"><span>Size</span></th>
+				<th name="ts"><span>Date</span></th>
+			</tr>
+		</thead>
+		<tbody>
+<tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+
+{%- for f in files %}
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+{%- endfor %}
+
+		</tbody>
+	</table>
+	
+	{%- if logues[1] %}
+	<div>{{ logues[1] }}</div><br />
+	{%- endif %}
+	
+	<h2><a href="{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
+
+</body>
+</html>

copyparty/web/dbg-audio.js

@@ -0,0 +1,61 @@
+var ofun = audio_eq.apply.bind(audio_eq);
+audio_eq.apply = function () {
+    var ac1 = mp.ac;
+    ofun();
+    var ac = mp.ac,
+        w = 2048,
+        h = 256;
+
+    if (!audio_eq.filters.length) {
+        audio_eq.ana = null;
+        return;
+    }
+
+    var can = ebi('fft_can');
+    if (!can) {
+        can = mknod('canvas');
+        can.setAttribute('id', 'fft_can');
+        can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
+        document.body.appendChild(can);
+        can.width = w;
+        can.height = h;
+    }
+    var cc = can.getContext('2d');
+    if (!ac)
+        return;
+
+    var ana = ac.createAnalyser();
+    ana.smoothingTimeConstant = 0;
+    ana.fftSize = 8192;
+
+    audio_eq.filters[0].connect(ana);
+    audio_eq.ana = ana;
+
+    var buf = new Uint8Array(ana.frequencyBinCount),
+        colw = can.width / buf.length;
+
+    cc.fillStyle = '#fc0';
+    function draw() {
+        if (ana == audio_eq.ana)
+            requestAnimationFrame(draw);
+
+        ana.getByteFrequencyData(buf);
+
+        cc.clearRect(0, 0, can.width, can.height);
+
+        /*var x = 0, w = 1;
+        for (var a = 0; a < buf.length; a++) {
+            cc.fillRect(x, h - buf[a], w, h);
+            x += w;
+        }*/
+        var mul = Math.pow(w, 4) / buf.length;
+        for (var x = 0; x < w; x++) {
+            var a = Math.floor(Math.pow(x, 4) / mul),
+                v = buf[a];
+
+            cc.fillRect(x, h - v, 1, v);
+        }
+    }
+    draw();
+};
+audio_eq.apply();

copyparty/web/md.css

@@ -50,6 +50,9 @@ pre code:last-child {
 pre code::before {
 	content: counter(precode);
 	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
 	display: inline-block;
 	text-align: right;
 	font-size: .75em;
@@ -591,12 +594,3 @@ blink {
 		color: #940;
 	}
 }
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-*/

copyparty/web/md.html

@@ -123,8 +123,12 @@ write markdown (most html is 🙆 too)
 	
 	<script>
 
-var link_md_as_html = false;  // TODO (does nothing)
 var last_modified = {{ lastmod }};
+var md_opt = {
+	link_md_as_html: false,
+	allow_plugins: {{ md_plug }},
+	modpoll_freq: {{ md_chk_rate }}
+};
 
 (function () {
     var btn = document.getElementById("lightswitch");
@@ -134,24 +138,18 @@ var last_modified = {{ lastmod }};
         document.documentElement.setAttribute("class", dark ? "dark" : "");
         btn.innerHTML = "go " + (dark ? "light" : "dark");
         if (window.localStorage)
-            localStorage.setItem('darkmode', dark ? 1 : 0);
+            localStorage.setItem('lightmode', dark ? 0 : 1);
     };
     btn.onclick = toggle;
-    if (window.localStorage && localStorage.getItem('darkmode') == 1)
+    if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		toggle();
 })();
 
-if (!String.startsWith) {
-	String.prototype.startsWith = function(s, i) {
-		i = i>0 ? i|0 : 0;
-		return this.substring(i, i + s.length) === s;
-	};
-}
-
 	</script>
-	<script src="/.cpr/deps/marked.full.js"></script>
+    <script src="/.cpr/util.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}
-		<script src="/.cpr/md2.js"></script>
+	<script src="/.cpr/md2.js"></script>
 	{%- endif %}
 </body></html>

copyparty/web/md.js

@@ -1,10 +1,12 @@
-var dom_toc = document.getElementById('toc');
-var dom_wrap = document.getElementById('mw');
-var dom_hbar = document.getElementById('mh');
-var dom_nav = document.getElementById('mn');
-var dom_pre = document.getElementById('mp');
-var dom_src = document.getElementById('mt');
-var dom_navtgl = document.getElementById('navtoggle');
+"use strict";
+
+var dom_toc = ebi('toc');
+var dom_wrap = ebi('mw');
+var dom_hbar = ebi('mh');
+var dom_nav = ebi('mn');
+var dom_pre = ebi('mp');
+var dom_src = ebi('mt');
+var dom_navtgl = ebi('navtoggle');
 
 
 // chrome 49 needs this
@@ -18,6 +20,10 @@ var dbg = function () { };
 // dbg = console.log
 
 
+// plugins
+var md_plug = {};
+
+
 function hesc(txt) {
     return txt.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 }
@@ -30,7 +36,7 @@ function cls(dom, name, add) {
 }
 
 
-function static(obj) {
+function statify(obj) {
     return JSON.parse(JSON.stringify(obj));
 }
 
@@ -40,7 +46,7 @@ function static(obj) {
     var ua = navigator.userAgent;
     if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
         // necessary on ff-68.7 at least
-        var s = document.createElement('style');
+        var s = mknod('style');
         s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
         console.log(s.innerHTML);
         document.head.appendChild(s);
@@ -59,7 +65,7 @@ function static(obj) {
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = hesc(decodeURIComponent(n[a]));
+        var dec = hesc(uricom_dec(n[a])[0]);
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -154,13 +160,110 @@ function copydom(src, dst, lv) {
 }
 
 
+function md_plug_err(ex, js) {
+    var errbox = ebi('md_errbox');
+    if (errbox)
+        errbox.parentNode.removeChild(errbox);
+
+    if (!ex)
+        return;
+
+    var msg = (ex + '').split('\n')[0];
+    var ln = ex.lineNumber;
+    var o = null;
+    if (ln) {
+        msg = "Line " + ln + ", " + msg;
+        var lns = js.split('\n');
+        if (ln < lns.length) {
+            o = mknod('span');
+            o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
+            o.textContent = lns[ln - 1];
+        }
+    }
+    errbox = mknod('div');
+    errbox.setAttribute('id', 'md_errbox');
+    errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
+    errbox.textContent = msg;
+    errbox.onclick = function () {
+        alert('' + ex.stack);
+    };
+    if (o) {
+        errbox.appendChild(o);
+        errbox.style.padding = '.25em .5em';
+    }
+    dom_nav.appendChild(errbox);
+
+    try {
+        console.trace();
+    }
+    catch (ex2) { }
+}
+
+
+function load_plug(md_text, plug_type) {
+    if (!md_opt.allow_plugins)
+        return md_text;
+
+    var find = '\n```copyparty_' + plug_type + '\n';
+    var ofs = md_text.indexOf(find);
+    if (ofs === -1)
+        return md_text;
+
+    var ofs2 = md_text.indexOf('\n```', ofs + 1);
+    if (ofs2 == -1)
+        return md_text;
+
+    var js = md_text.slice(ofs + find.length, ofs2 + 1);
+    var md = md_text.slice(0, ofs + 1) + md_text.slice(ofs2 + 4);
+
+    var old_plug = md_plug[plug_type];
+    if (!old_plug || old_plug[1] != js) {
+        js = 'const x = { ' + js + ' }; x;';
+        try {
+            var x = eval(js);
+        }
+        catch (ex) {
+            md_plug[plug_type] = null;
+            md_plug_err(ex, js);
+            return md;
+        }
+        if (x['ctor']) {
+            x['ctor']();
+            delete x['ctor'];
+        }
+        md_plug[plug_type] = [x, js];
+    }
+
+    return md;
+}
+
+
 function convert_markdown(md_text, dest_dom) {
-    marked.setOptions({
+    md_text = md_text.replace(/\r/g, '');
+
+    md_plug_err(null);
+    md_text = load_plug(md_text, 'pre');
+    md_text = load_plug(md_text, 'post');
+
+    var marked_opts = {
         //headerPrefix: 'h-',
         breaks: true,
         gfm: true
-    });
-    var md_html = marked(md_text);
+    };
+
+    var ext = md_plug['pre'];
+    if (ext)
+        Object.assign(marked_opts, ext[0]);
+
+    try {
+        var md_html = marked(md_text, marked_opts);
+    }
+    catch (ex) {
+        if (ext)
+            md_plug_err(ex, ext[1]);
+
+        throw ex;
+    }
     var md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
 
     var nodes = md_dom.getElementsByTagName('a');
@@ -196,7 +299,7 @@ function convert_markdown(md_text, dest_dom) {
     }
 
     // separate <code> for each line in <pre>
-    var nodes = md_dom.getElementsByTagName('pre');
+    nodes = md_dom.getElementsByTagName('pre');
     for (var a = nodes.length - 1; a >= 0; a--) {
         var el = nodes[a];
 
@@ -209,7 +312,7 @@ function convert_markdown(md_text, dest_dom) {
             continue;
 
         var nline = parseInt(el.getAttribute('data-ln')) + 1;
-        var lines = el.innerHTML.replace(/\r?\n<\/code>$/i, '</code>').split(/\r?\n/g);
+        var lines = el.innerHTML.replace(/\n<\/code>$/i, '</code>').split(/\n/g);
         for (var b = 0; b < lines.length - 1; b++)
             lines[b] += '</code>\n<code data-ln="' + (nline + b) + '">';
 
@@ -242,12 +345,29 @@ function convert_markdown(md_text, dest_dom) {
         el.innerHTML = '<a href="#' + id + '">' + el.innerHTML + '</a>';
     }
 
+    ext = md_plug['post'];
+    if (ext && ext[0].render)
+        try {
+            ext[0].render(md_dom);
+        }
+        catch (ex) {
+            md_plug_err(ex, ext[1]);
+        }
+
     copydom(md_dom, dest_dom, 0);
+
+    if (ext && ext[0].render2)
+        try {
+            ext[0].render2(dest_dom);
+        }
+        catch (ex) {
+            md_plug_err(ex, ext[1]);
+        }
 }
 
 
 function init_toc() {
-    var loader = document.getElementById('ml');
+    var loader = ebi('ml');
     loader.parentNode.removeChild(loader);
 
     var anchors = [];  // list of toc entries, complex objects
@@ -281,7 +401,12 @@ function init_toc() {
 
             elm.childNodes[0].setAttribute('ctr', ctr.slice(0, lv).join('.'));
 
-            html.push('<li>' + elm.innerHTML + '</li>');
+            var elm2 = elm.cloneNode(true);
+            elm2.childNodes[0].textContent = elm.textContent;
+            while (elm2.childNodes.length > 1)
+                elm2.removeChild(elm2.childNodes[1]);
+
+            html.push('<li>' + elm2.innerHTML + '</li>');
 
             if (anchor != null)
                 anchors.push(anchor);
@@ -399,11 +524,9 @@ dom_navtgl.onclick = function () {
     dom_navtgl.innerHTML = hidden ? 'show nav' : 'hide nav';
     dom_nav.style.display = hidden ? 'none' : 'block';
 
-    if (window.localStorage)
-        localStorage.setItem('hidenav', hidden ? 1 : 0);
-
+    swrite('hidenav', hidden ? 1 : 0);
     redraw();
 };
 
-if (window.localStorage && localStorage.getItem('hidenav') == 1)
+if (sread('hidenav') == 1)
     dom_navtgl.onclick();

copyparty/web/md2.css

@@ -1,108 +1,125 @@
 #toc {
-    display: none;
+	display: none;
 }
 #mtw {
-    display: block;
-    position: fixed;
-    left: .5em;
-    bottom: 0;
-    width: calc(100% - 56em);
+	display: block;
+	position: fixed;
+	left: .5em;
+	bottom: 0;
+	width: calc(100% - 56em);
 }
 #mw {
-    left: calc(100% - 55em);
-    overflow-y: auto;
-    position: fixed;
-    bottom: 0;
+	left: calc(100% - 55em);
+	overflow-y: auto;
+	position: fixed;
+	bottom: 0;
 }
 
 
 /* single-screen */
 #mtw.preview,
 #mw.editor {
-    opacity: 0;
-    z-index: 1;
+	opacity: 0;
+	z-index: 1;
 }
 #mw.preview,
 #mtw.editor {
-    z-index: 5;
+	z-index: 5;
 }
 #mtw.single,
 #mw.single {
-    margin: 0;
-    left: 1em;
-    left: max(1em, calc((100% - 56em) / 2));
+	margin: 0;
+	left: 1em;
+	left: max(1em, calc((100% - 56em) / 2));
 }
 #mtw.single {
-    width: 55em;
-    width: min(55em, calc(100% - 2em));
+	width: 55em;
+	width: min(55em, calc(100% - 2em));
 }
 
 
 #mp {
-    position: relative;
+	position: relative;
 }
 #mt, #mtr {
-    width: 100%;
-    height: calc(100% - 1px);
-    color: #444;
-    background: #f7f7f7;
-    border: 1px solid #999;
-    outline: none;
-    padding: 0;
-    margin: 0;
-    font-family: 'consolas', monospace, monospace;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    word-wrap: break-word; /*ie*/
-    overflow-y: scroll;
-    line-height: 1.3em;
-    font-size: .9em;
-    position: relative;
-    scrollbar-color: #eb0 #f7f7f7;
+	width: 100%;
+	height: calc(100% - 1px);
+	color: #444;
+	background: #f7f7f7;
+	border: 1px solid #999;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	font-family: 'consolas', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-word;
+	overflow-wrap: break-word;
+	word-wrap: break-word; /*ie*/
+	overflow-y: scroll;
+	line-height: 1.3em;
+	font-size: .9em;
+	position: relative;
+	scrollbar-color: #eb0 #f7f7f7;
 }
 html.dark #mt {
-    color: #eee;
-    background: #222;
-    border: 1px solid #777;
-    scrollbar-color: #b80 #282828;
+	color: #eee;
+	background: #222;
+	border: 1px solid #777;
+	scrollbar-color: #b80 #282828;
 }
 #mtr {
-    position: absolute;
-    top: 0;
-    left: 0;
+	position: absolute;
+	top: 0;
+	left: 0;
 }
 #save.force-save {
-    color: #400;
-    background: #f97;
-    border-radius: .15em;
+	color: #400;
+	background: #f97;
+	border-radius: .15em;
+}
+html.dark #save.force-save {
+	color: #fca;
+	background: #720;
 }
 #save.disabled {
-    opacity: .4;
+	opacity: .4;
+}
+#helpbox,
+#toast {
+	background: #f7f7f7;
+	border-radius: .4em;
+	z-index: 9001;
 }
 #helpbox {
-    display: none;
-    position: fixed;
-    background: #f7f7f7;
-    box-shadow: 0 .5em 2em #777;
-    border-radius: .4em;
-    padding: 2em;
-    top: 4em;
-    overflow-y: auto;
-    height: calc(100% - 12em);
-    left: calc(50% - 15em);
-    right: 0;
-    width: 30em;
-    z-index: 9001;
+	display: none;
+	position: fixed;
+	padding: 2em;
+	top: 4em;
+	overflow-y: auto;
+	box-shadow: 0 .5em 2em #777;
+	height: calc(100% - 12em);
+	left: calc(50% - 15em);
+	right: 0;
+	width: 30em;
 }
 #helpclose {
-    display: block;
+	display: block;
 }
 html.dark #helpbox {
-    background: #222;
-    box-shadow: 0 .5em 2em #444;
-    border: 1px solid #079;
-    border-width: 1px 0;
+	box-shadow: 0 .5em 2em #444;
+}
+html.dark #helpbox,
+html.dark #toast {
+	background: #222;
+	border: 1px solid #079;
+	border-width: 1px 0;
+}
+#toast {
+	font-weight: bold;
+	text-align: center;
+	padding: .6em 0;
+	position: fixed;
+	top: 30%;
+	transition: opacity 0.2s ease-in-out;
+	opacity: 1;
 }
-
-# mt {opacity: .5;top:1px}

copyparty/web/md2.js

@@ -1,3 +1,6 @@
+"use strict";
+
+
 // server state
 var server_md = dom_src.value;
 
@@ -8,15 +11,15 @@ var js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
 
 
 // dom nodes
-var dom_swrap = document.getElementById('mtw');
-var dom_sbs = document.getElementById('sbs');
-var dom_nsbs = document.getElementById('nsbs');
-var dom_tbox = document.getElementById('toolsbox');
+var dom_swrap = ebi('mtw');
+var dom_sbs = ebi('sbs');
+var dom_nsbs = ebi('nsbs');
+var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = document.createElement('div');
+    var d = mknod('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
-    d = document.getElementById('mtr');
+    d = ebi('mtr');
     // hide behind the textarea (offsetTop is not computed if display:none)
     dom_src.style.zIndex = '4';
     d.style.zIndex = '3';
@@ -68,7 +71,7 @@ var map_src = [];
 var map_pre = [];
 function genmap(dom, oldmap) {
     var find = nlines;
-    while (oldmap && find --> 0) {
+    while (oldmap && find-- > 0) {
         var tmap = genmapq(dom, '*[data-ln="' + find + '"]');
         if (!tmap || !tmap.length)
             continue;
@@ -91,7 +94,7 @@ var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
     function draw_md() {
-        var t0 = new Date().getTime();
+        var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
 
@@ -105,9 +108,9 @@ var draw_md = (function () {
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
 
-        cls(document.getElementById('save'), 'disabled', src == server_md);
+        cls(ebi('save'), 'disabled', src == server_md);
 
-        var t1 = new Date().getTime();
+        var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
     }
 
@@ -141,7 +144,7 @@ redraw = (function () {
         onresize();
     }
     function modetoggle() {
-        mode = dom_nsbs.innerHTML;
+        var mode = dom_nsbs.innerHTML;
         dom_nsbs.innerHTML = mode == 'editor' ? 'preview' : 'editor';
         mode += ' single';
         dom_wrap.setAttribute('class', mode);
@@ -177,7 +180,7 @@ redraw = (function () {
         y += src.clientHeight / 2;
         var sy1 = -1, sy2 = -1, dy1 = -1, dy2 = -1;
         for (var a = 1; a < nlines + 1; a++) {
-            if (srcmap[a] === null || dstmap[a] === null)
+            if (srcmap[a] == null || dstmap[a] == null)
                 continue;
 
             if (srcmap[a] > y) {
@@ -220,14 +223,108 @@ redraw = (function () {
 })();
 
 
+// modification checker
+function Modpoll() {
+    this.skip_one = true;
+    this.disabled = false;
+
+    this.periodic = function () {
+        var that = this;
+        setTimeout(function () {
+            that.periodic();
+        }, 1000 * md_opt.modpoll_freq);
+
+        var skip = null;
+
+        if (ebi('toast'))
+            skip = 'toast';
+
+        else if (this.skip_one)
+            skip = 'saved';
+
+        else if (this.disabled)
+            skip = 'disabled';
+
+        if (skip) {
+            console.log('modpoll skip, ' + skip);
+            this.skip_one = false;
+            return;
+        }
+
+        console.log('modpoll...');
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
+        var xhr = new XMLHttpRequest();
+        xhr.modpoll = this;
+        xhr.open('GET', url, true);
+        xhr.responseType = 'text';
+        xhr.onreadystatechange = this.cb;
+        xhr.send();
+    }
+
+    this.cb = function () {
+        if (this.modpoll.disabled || this.modpoll.skip_one) {
+            console.log('modpoll abort');
+            return;
+        }
+
+        if (this.readyState != XMLHttpRequest.DONE)
+            return;
+
+        if (this.status !== 200) {
+            console.log('modpoll err ' + this.status + ": " + this.responseText);
+            return;
+        }
+
+        if (!this.responseText)
+            return;
+
+        var server_ref = server_md.replace(/\r/g, '');
+        var server_now = this.responseText.replace(/\r/g, '');
+
+        if (server_ref != server_now) {
+            console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
+            this.modpoll.disabled = true;
+            var msg = [
+                "The document has changed on the server.<br />" +
+                "The changes will NOT be loaded into your editor automatically.",
+
+                "Press F5 or CTRL-R to refresh the page,<br />" +
+                "replacing your document with the server copy.",
+
+                "You can click this message to ignore and contnue."
+            ];
+            return toast(false, "box-shadow:0 1em 2em rgba(64,64,64,0.8);font-weight:normal",
+                36, "<p>" + msg.join('</p>\n<p>') + '</p>');
+        }
+
+        console.log('modpoll eq');
+    }
+
+    if (md_opt.modpoll_freq > 0)
+        this.periodic();
+
+    return this;
+}
+var modpoll = new Modpoll();
+
+
+window.onbeforeunload = function (e) {
+    if ((ebi("save").getAttribute('class') + '').indexOf('disabled') >= 0)
+        return; //nice (todo)
+
+    e.preventDefault(); //ff
+    e.returnValue = ''; //chrome
+};
+
+
 // save handler
 function save(e) {
     if (e) e.preventDefault();
-    var save_btn = document.getElementById("save"),
+    var save_btn = ebi("save"),
         save_cls = save_btn.getAttribute('class') + '';
 
     if (save_cls.indexOf('disabled') >= 0) {
-        toast('font-size:2em;color:#fc6;width:9em;', 'no changes');
+        toast(true, ";font-size:2em;color:#c90", 9, "no changes");
         return;
     }
 
@@ -251,6 +348,8 @@ function save(e) {
     xhr.onreadystatechange = save_cb;
     xhr.btn = save_btn;
     xhr.txt = txt;
+
+    modpoll.skip_one = true;  // skip one iteration while we save
     xhr.send(fd);
 }
 
@@ -300,7 +399,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
@@ -344,23 +443,44 @@ function savechk_cb() {
     last_modified = this.lastmod;
     server_md = this.txt;
     draw_md();
-    toast('font-size:6em;font-family:serif;color:#cf6;width:4em;',
+    toast(true, ";font-size:6em;font-family:serif;color:#9b4", 4,
         'OK✔️<span style="font-size:.2em;color:#999;position:absolute">' + this.ntry + '</span>');
+
+    modpoll.disabled = false;
 }
 
-function toast(style, msg) {
-    var ok = document.createElement('div');
-    style += 'font-weight:bold;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1';
+function toast(autoclose, style, width, msg) {
+    var ok = ebi("toast");
+    if (ok)
+        ok.parentNode.removeChild(ok);
+
+    style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
+    ok = mknod('div');
+    ok.setAttribute('id', 'toast');
     ok.setAttribute('style', style);
     ok.innerHTML = msg;
-    var parent = document.getElementById('m');
+    var parent = ebi('m');
     document.documentElement.appendChild(ok);
-    setTimeout(function () {
-        ok.style.opacity = 0;
-    }, 500);
-    setTimeout(function () {
-        ok.parentNode.removeChild(ok);
-    }, 750);
+
+    var hide = function (delay) {
+        delay = delay || 0;
+
+        setTimeout(function () {
+            ok.style.opacity = 0;
+        }, delay);
+
+        setTimeout(function () {
+            if (ok.parentNode)
+                ok.parentNode.removeChild(ok);
+        }, delay + 250);
+    }
+
+    ok.onclick = function () {
+        hide(0);
+    };
+
+    if (autoclose)
+        hide(500);
 }
 
 
@@ -540,6 +660,10 @@ function md_backspace() {
     if (/^\s*$/.test(left))
         return true;
 
+    // same if selection
+    if (o0 != dom_src.selectionEnd)
+        return true;
+
     // same if line is all-whitespace or non-markup
     var v = m[0].replace(/[^ ]/g, " ");
     if (v === m[0] || v.length !== left.length)
@@ -602,7 +726,7 @@ function fmt_table(e) {
         //o0 = txt.lastIndexOf('\n\n', ofs),
         //o1 = txt.indexOf('\n\n', ofs);
         o0 = reLastIndexOf(txt, /\n\s*\n/m, ofs),
-        o1 = txt.slice(ofs).search(/\n\s*\n/m);
+        o1 = txt.slice(ofs).search(/\n\s*\n|\n\s*$/m);
     // note \s contains \n but its fine
 
     if (o0 < 0)
@@ -623,12 +747,21 @@ function fmt_table(e) {
         lpipe = tab[1].indexOf('|') < tab[1].indexOf('-'),
         rpipe = tab[1].lastIndexOf('|') > tab[1].lastIndexOf('-'),
         re_lpipe = lpipe ? /^\s*\|\s*/ : /^\s*/,
-        re_rpipe = rpipe ? /\s*\|\s*$/ : /\s*$/;
+        re_rpipe = rpipe ? /\s*\|\s*$/ : /\s*$/,
+        ncols;
+
+    // the second row defines the table,
+    // need to process that first
+    var tmp = tab[0];
+    tab[0] = tab[1];
+    tab[1] = tmp;
 
     for (var a = 0; a < tab.length; a++) {
+        var row_name = (a == 1) ? 'header' : 'row#' + (a + 1);
+
         var ind2 = tab[a].match(re_ind)[0];
-        if (ind != ind2 && a > 0)  // the table can be a list entry or something, ignore [0]
-            return alert(err + 'indentation mismatch on row 2 and ' + (a + 1) + ',\n' + tab[a]);
+        if (ind != ind2 && a != 1)  // the table can be a list entry or something, ignore [0]
+            return alert(err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
 
         var t = tab[a].slice(ind.length);
         t = t.replace(re_lpipe, "");
@@ -637,17 +770,25 @@ function fmt_table(e) {
 
         if (a == 0)
             ncols = tab[a].length;
+        else if (ncols < tab[a].length)
+            return alert(err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
 
-        if (ncols != tab[a].length)
-            return alert(err + 'num.columns mismatch on row 2 and ' + (a + 1) + '; ' + ncols + ' != ' + tab[a].length);
+        // if row has less columns than row2, fill them in
+        while (tab[a].length < ncols)
+            tab[a].push('');
     }
 
+    // aight now swap em back
+    tmp = tab[0];
+    tab[0] = tab[1];
+    tab[1] = tmp;
+
     var re_align = /^ *(:?)-+(:?) *$/;
     var align = [];
     for (var col = 0; col < tab[1].length; col++) {
         var m = tab[1][col].match(re_align);
         if (!m)
-            return alert(err + 'invalid column specification, row 2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
+            return alert(err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
 
         if (m[2]) {
             if (m[1])
@@ -664,7 +805,8 @@ function fmt_table(e) {
     for (var col = 0; col < ncols; col++) {
         var max = 0;
         for (var row = 0; row < tab.length; row++)
-            max = Math.max(max, tab[row][col].length);
+            if (row != 1)
+                max = Math.max(max, tab[row][col].length);
 
         var s = '';
         for (var n = 0; n < max; n++)
@@ -731,9 +873,8 @@ function mark_uni(e) {
     dom_tbox.setAttribute('class', '');
 
     var txt = dom_src.value,
-        ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g');
-
-    mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
+        ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g'),
+        mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
 
     if (txt == mod) {
         alert('no results;  no modifications were made');
@@ -769,7 +910,12 @@ function iter_uni(e) {
 // configure whitelist
 function cfg_uni(e) {
     if (e) e.preventDefault();
-    esc_uni_whitelist = prompt("unicode whitelist", esc_uni_whitelist);
+
+    var reply = prompt("unicode whitelist", esc_uni_whitelist);
+    if (reply === null)
+        return;
+
+    esc_uni_whitelist = reply;
     js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
 }
 
@@ -786,7 +932,7 @@ function cfg_uni(e) {
             return false;
         }
         if (ev.code == "Escape" || kc == 27) {
-            var d = document.getElementById('helpclose');
+            var d = ebi('helpclose');
             if (d)
                 d.click();
         }
@@ -843,22 +989,22 @@ function cfg_uni(e) {
         }
     }
     document.onkeydown = keydown;
-    document.getElementById('save').onclick = save;
+    ebi('save').onclick = save;
 })();
 
 
-document.getElementById('tools').onclick = function (e) {
+ebi('tools').onclick = function (e) {
     if (e) e.preventDefault();
     var is_open = dom_tbox.getAttribute('class') != 'open';
     dom_tbox.setAttribute('class', is_open ? 'open' : '');
 };
 
 
-document.getElementById('help').onclick = function (e) {
+ebi('help').onclick = function (e) {
     if (e) e.preventDefault();
     dom_tbox.setAttribute('class', '');
 
-    var dom = document.getElementById('helpbox');
+    var dom = ebi('helpbox');
     var dtxt = dom.getElementsByTagName('textarea');
     if (dtxt.length > 0) {
         convert_markdown(dtxt[0].value, dom);
@@ -866,16 +1012,16 @@ document.getElementById('help').onclick = function (e) {
     }
 
     dom.style.display = 'block';
-    document.getElementById('helpclose').onclick = function () {
+    ebi('helpclose').onclick = function () {
         dom.style.display = 'none';
     };
 };
 
 
-document.getElementById('fmt_table').onclick = fmt_table;
-document.getElementById('mark_uni').onclick = mark_uni;
-document.getElementById('iter_uni').onclick = iter_uni;
-document.getElementById('cfg_uni').onclick = cfg_uni;
+ebi('fmt_table').onclick = fmt_table;
+ebi('mark_uni').onclick = mark_uni;
+ebi('iter_uni').onclick = iter_uni;
+ebi('cfg_uni').onclick = cfg_uni;
 
 
 // blame steen
@@ -903,7 +1049,7 @@ action_stack = (function () {
         var p1 = from.length,
             p2 = to.length;
 
-        while (p1 --> 0 && p2 --> 0)
+        while (p1-- > 0 && p2-- > 0)
             if (from[p1] != to[p2])
                 break;
 
@@ -983,13 +1129,12 @@ action_stack = (function () {
         ref = newtxt;
         dbg('undos(%d) redos(%d)', hist.un.length, hist.re.length);
         if (hist.un.length > 0)
-            dbg(static(hist.un.slice(-1)[0]));
+            dbg(statify(hist.un.slice(-1)[0]));
         if (hist.re.length > 0)
-            dbg(static(hist.re.slice(-1)[0]));
+            dbg(statify(hist.re.slice(-1)[0]));
     }
 
     return {
-        push: push,
         undo: undo,
         redo: redo,
         push: schedule_push,
@@ -997,14 +1142,3 @@ action_stack = (function () {
         _ref: ref
     }
 })();
-
-/*
-document.getElementById('help').onclick = function () {
-    var c1 = getComputedStyle(dom_src).cssText.split(';');
-    var c2 = getComputedStyle(dom_ref).cssText.split(';');
-    var max = Math.min(c1.length, c2.length);
-    for (var a = 0; a < max; a++)
-        if (c1[a] !== c2[a])
-            console.log(c1[a] + '\n' + c2[a]);
-}
-*/

copyparty/web/mde.css

@@ -8,68 +8,58 @@ html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
 html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
 
 html {
-    line-height: 1.5em;
+	line-height: 1.5em;
 }
 html, body {
-    margin: 0;
-    padding: 0;
-    min-height: 100%;
-    font-family: sans-serif;
-    background: #f7f7f7;
-    color: #333;
+	margin: 0;
+	padding: 0;
+	min-height: 100%;
+	font-family: sans-serif;
+	background: #f7f7f7;
+	color: #333;
 }
 #mn {
-    font-weight: normal;
-    margin: 1.3em 0 .7em 1em;
+	font-weight: normal;
+	margin: 1.3em 0 .7em 1em;
 }
 #mn a {
-    color: #444;
-    margin: 0 0 0 -.2em;
-    padding: 0 0 0 .4em;
-    text-decoration: none;
-    /* ie: */
-    border-bottom: .1em solid #777\9;
-    margin-right: 1em\9;
+	color: #444;
+	margin: 0 0 0 -.2em;
+	padding: 0 0 0 .4em;
+	text-decoration: none;
+	/* ie: */
+	border-bottom: .1em solid #777\9;
+	margin-right: 1em\9;
 }
 #mn a:first-child {
-    padding-left: .5em;
+	padding-left: .5em;
 }
 #mn a:last-child {
-    padding-right: .5em;
+	padding-right: .5em;
 }
 #mn a:not(:last-child):after {
-    content: '';
-    width: 1.05em;
-    height: 1.05em;
-    margin: -.2em .3em -.2em -.4em;
-    display: inline-block;
-    border: 1px solid rgba(0,0,0,0.2);
-    border-width: .2em .2em 0 0;
-    transform: rotate(45deg);
+	content: '';
+	width: 1.05em;
+	height: 1.05em;
+	margin: -.2em .3em -.2em -.4em;
+	display: inline-block;
+	border: 1px solid rgba(0,0,0,0.2);
+	border-width: .2em .2em 0 0;
+	transform: rotate(45deg);
 }
 #mn a:hover {
-    color: #000;
-    text-decoration: underline;
+	color: #000;
+	text-decoration: underline;
 }
 
 html .editor-toolbar>button.disabled {
-    opacity: .35;
-    pointer-events: none;
+	opacity: .35;
+	pointer-events: none;
 }
 html .editor-toolbar>button.save.force-save {
-    background: #f97;
+	background: #f97;
 }
 
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-.cm-header { font-size: .4em !important }
-*/
-
 
 
 
@@ -101,29 +91,29 @@ html .editor-toolbar>button.save.force-save {
 	line-height: 1.1em;
 }
 .mdo a {
-    color: #fff;
-    background: #39b;
-    text-decoration: none;
-    padding: 0 .3em;
-    border: none;
-    border-bottom: .07em solid #079;
+	color: #fff;
+	background: #39b;
+	text-decoration: none;
+	padding: 0 .3em;
+	border: none;
+	border-bottom: .07em solid #079;
 }
 .mdo h2 {
-    color: #fff;
-    background: #555;
-    margin-top: 2em;
-    border-bottom: .22em solid #999;
-    border-top: none;
+	color: #fff;
+	background: #555;
+	margin-top: 2em;
+	border-bottom: .22em solid #999;
+	border-top: none;
 }
 .mdo h1 {
-    color: #fff;
-    background: #444;
-    font-weight: normal;
-    border-top: .4em solid #fb0;
-    border-bottom: .4em solid #777;
-    border-radius: 0 1em 0 1em;
-    margin: 3em 0 1em 0;
-    padding: .5em 0;
+	color: #fff;
+	background: #444;
+	font-weight: normal;
+	border-top: .4em solid #fb0;
+	border-bottom: .4em solid #777;
+	border-radius: 0 1em 0 1em;
+	margin: 3em 0 1em 0;
+	padding: .5em 0;
 }
 h1, h2 {
 	line-height: 1.5em;
@@ -197,14 +187,14 @@ th {
 
 /* mde support */
 .mdo {
-    padding: 1em;
-    background: #f7f7f7;
+	padding: 1em;
+	background: #f7f7f7;
 }
 html.dark .mdo {
-    background: #1c1c1c;
+	background: #1c1c1c;
 }
 .CodeMirror {
-    background: #f7f7f7;
+	background: #f7f7f7;
 }
 
 
@@ -214,108 +204,108 @@ html.dark .mdo {
 /* darkmode */
 html.dark .mdo,
 html.dark .CodeMirror {
-    border-color: #222;
+	border-color: #222;
 }
 html.dark,
 html.dark body,
 html.dark .CodeMirror {
-    background: #222;
-    color: #ccc;
+	background: #222;
+	color: #ccc;
 }
 html.dark .CodeMirror-cursor {
-    border-color: #fff;
+	border-color: #fff;
 }
 html.dark .CodeMirror-selected {
-    box-shadow: 0 0 1px #0cf inset;
+	box-shadow: 0 0 1px #0cf inset;
 }
 html.dark .CodeMirror-selected,
 html.dark .CodeMirror-selectedtext {
-    border-radius: .1em;
-    background: #246;
-    color: #fff;
+	border-radius: .1em;
+	background: #246;
+	color: #fff;
 }
 html.dark .mdo a {
-    background: #057;
+	background: #057;
 }
 html.dark .mdo h1 a, html.dark .mdo h4 a,
 html.dark .mdo h2 a, html.dark .mdo h5 a,
 html.dark .mdo h3 a, html.dark .mdo h6 a {
-    color: inherit;
-    background: none;
+	color: inherit;
+	background: none;
 }
 html.dark pre,
 html.dark code {
-    color: #8c0;
-    background: #1a1a1a;
-    border: .07em solid #333;
+	color: #8c0;
+	background: #1a1a1a;
+	border: .07em solid #333;
 }
 html.dark .mdo ul,
 html.dark .mdo ol {
-    border-color: #444;
+	border-color: #444;
 }
 html.dark .mdo>ul,
 html.dark .mdo>ol {
-    border-color: #555;
+	border-color: #555;
 }
 html.dark strong {
-    color: #fff;
+	color: #fff;
 }
 html.dark p>em,
 html.dark li>em,
 html.dark td>em {
-    color: #f94;
-    border-color: #666;
+	color: #f94;
+	border-color: #666;
 }
 html.dark h1 {
-    background: #383838;
-    border-top: .4em solid #b80;
-    border-bottom: .4em solid #4c4c4c;
+	background: #383838;
+	border-top: .4em solid #b80;
+	border-bottom: .4em solid #4c4c4c;
 }
 html.dark h2 {
-    background: #444;
-    border-bottom: .22em solid #555;
+	background: #444;
+	border-bottom: .22em solid #555;
 }
 html.dark td,
 html.dark th {
-    border-color: #444;
+	border-color: #444;
 }
 html.dark blockquote {
-    background: #282828;
-    border: .07em dashed #444;
+	background: #282828;
+	border: .07em dashed #444;
 }
 
 
 
 html.dark #mn a {
-    color: #ccc;
+	color: #ccc;
 }
 html.dark #mn a:not(:last-child):after {
-    border-color: rgba(255,255,255,0.3);
+	border-color: rgba(255,255,255,0.3);
 }
 html.dark .editor-toolbar {
-    border-color: #2c2c2c;
-    background: #1c1c1c;
+	border-color: #2c2c2c;
+	background: #1c1c1c;
 }
 html.dark .editor-toolbar>i.separator {
-    border-left: 1px solid #444;
-    border-right: 1px solid #111;
+	border-left: 1px solid #444;
+	border-right: 1px solid #111;
 }
 html.dark .editor-toolbar>button {
-    margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
-    color: #aaa;
+	margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
+	color: #aaa;
 }
 
 
 
 html.dark .editor-toolbar>button:hover {
-    color: #333;
+	color: #333;
 }
 html.dark .editor-toolbar>button.active {
-    color: #333;
-    border-color: #ec1;
-    background: #c90;
+	color: #333;
+	border-color: #ec1;
+	background: #c90;
 }
 html.dark .editor-toolbar::after,
 html.dark .editor-toolbar::before {
-    background: none;
+	background: none;
 }

copyparty/web/mde.html

@@ -22,23 +22,28 @@
 	</div>
 	<script>
 
-var link_md_as_html = false;  // TODO (does nothing)
 var last_modified = {{ lastmod }};
+var md_opt = {
+	link_md_as_html: false,
+	allow_plugins: {{ md_plug }},
+	modpoll_freq: {{ md_chk_rate }}
+};
 
 var lightswitch = (function () {
 	var fun = function () {
-		var dark = !!!document.documentElement.getAttribute("class");
+		var dark = !document.documentElement.getAttribute("class");
 		document.documentElement.setAttribute("class", dark ? "dark" : "");
 		if (window.localStorage)
-			localStorage.setItem('darkmode', dark ? 1 : 0);
+			localStorage.setItem('lightmode', dark ? 0 : 1);
 	};
-	if (window.localStorage && localStorage.getItem('darkmode') == 1)
+	if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		fun();
 	
 	return fun;
 })();
 
 	</script>
+    <script src="/.cpr/util.js"></script>
 	<script src="/.cpr/deps/easymde.js"></script>
 	<script src="/.cpr/mde.js"></script>
 </body></html>

copyparty/web/mde.js

@@ -1,7 +1,9 @@
-var dom_wrap = document.getElementById('mw');
-var dom_nav = document.getElementById('mn');
-var dom_doc = document.getElementById('m');
-var dom_md = document.getElementById('mt');
+"use strict";
+
+var dom_wrap = ebi('mw');
+var dom_nav = ebi('mn');
+var dom_doc = ebi('m');
+var dom_md = ebi('mt');
 
 (function () {
     var n = document.location + '';
@@ -13,7 +15,7 @@ var dom_md = document.getElementById('mt');
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = decodeURIComponent(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+        var dec = uricom_dec(n[a])[0].replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -63,13 +65,13 @@ var mde = (function () {
     mde.codemirror.on("change", function () {
         md_changed(mde);
     });
-    var loader = document.getElementById('ml');
+    var loader = ebi('ml');
     loader.parentNode.removeChild(loader);
     return mde;
 })();
 
 function set_jumpto() {
-    document.querySelector('.editor-preview-side').onclick = jumpto;
+    QS('.editor-preview-side').onclick = jumpto;
 }
 
 function jumpto(ev) {
@@ -92,7 +94,7 @@ function md_changed(mde, on_srv) {
         window.md_saved = mde.value();
 
     var md_now = mde.value();
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
 
     if (md_now == window.md_saved)
         save_btn.classList.add('disabled');
@@ -103,7 +105,7 @@ function md_changed(mde, on_srv) {
 }
 
 function save(mde) {
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
     if (save_btn.classList.contains('disabled')) {
         alert('there is nothing to save');
         return;
@@ -210,10 +212,10 @@ function save_chk() {
     last_modified = this.lastmod;
     md_changed(this.mde, true);
 
-    var ok = document.createElement('div');
+    var ok = mknod('div');
     ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
     ok.innerHTML = 'OK✔️';
-    var parent = document.getElementById('m');
+    var parent = ebi('m');
     document.documentElement.appendChild(ok);
     setTimeout(function () {
         ok.style.opacity = 0;

copyparty/web/msg.css

@@ -3,7 +3,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -20,8 +20,8 @@ body {
 	padding-bottom: 5em;
 }
 #box {
-    padding: .5em 1em;
-    background: #2c2c2c;
+	padding: .5em 1em;
+	background: #2c2c2c;
 }
 pre {
 	font-family: monospace, monospace;

copyparty/web/splash.css

@@ -26,6 +26,26 @@ a {
 	border-radius: .2em;
 	padding: .2em .8em;
 }
+table {
+	border-collapse: collapse;
+}
+.vols td,
+.vols th {
+	padding: .3em .6em;
+	text-align: left;
+}
+.num {
+	border-right: 1px solid #bbb;
+}
+.num td {
+	padding: .1em .7em .1em 0;
+}
+.num td:first-child {
+	text-align: right;
+}
+.btns {
+	margin: 1em 0;
+}
 
 
 html.dark,
@@ -50,4 +70,7 @@ html.dark input {
 	border-radius: .5em;
 	padding: .5em .7em;
 	margin: 0 .5em 0 0;
+}
+html.dark .num {
+	border-color: #777;
 }

copyparty/web/splash.html

@@ -13,19 +13,49 @@
     <div id="wrap">
         <p>hello {{ this.uname }}</p>
 
+        {%- if avol %}
+        <h1>admin panel:</h1>
+        <table><tr><td> <!-- hehehe -->
+            <table class="num">
+                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
+                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
+                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
+                <tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+            </table>
+        </td><td>
+            <table class="vols">
+                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+                <tbody>
+                    {% for mp in avol %}
+                    {%- if mp in vstate and vstate[mp] %}
+                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                    {%- endif %}
+                    {% endfor %}
+                </tbody>
+            </table>
+        </td></tr></table>
+        <div class="btns">
+            <a href="{{ avol[0] }}?stack">dump stack</a>
+        </div>
+        {%- endif %}
+
+        {%- if rvol %}
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
+        {%- if wvol %}
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
         <h1>login for more:</h1>
         <ul>
@@ -38,7 +68,7 @@
     </div>
     <script>
 
-if (window.localStorage && localStorage.getItem('darkmode') == 1)
+if (window.localStorage && localStorage.getItem('lightmode') != 1)
     document.documentElement.setAttribute("class", "dark");
 
 </script>

copyparty/web/upload.css

@@ -1,92 +1,4 @@
-.opview {
-	display: none;
-}
-.opview.act {
-	display: block;
-}
-#ops a {
-	color: #fc5;
-	font-size: 1.5em;
-	padding: 0 .3em;
-	margin: 0;
-	outline: none;
-}
-#ops a.act {
-	text-decoration: underline;
-}
-/*
-#ops a+a:after,
-#ops a:first-child:after {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #01a7e1;
-	margin-left: .3em;
-	position: relative;
-}
-#ops a+a:before {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #ff3f1a;
-	margin-right: .3em;
-	margin-left: -.3em;
-}
-#ops a:last-child:after {
-	content: '';
-}
-#ops a.act:before,
-#ops a.act:after {
-	text-decoration: none !important;
-}
-*/
-#ops i {
-	font-size: 1.5em;
-}
-#ops i:before {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #01a7e1;
-	position: relative;
-}
-#ops i:after {
-	content: 'x';
-	color: #282828;
-	text-shadow: 0 0 .08em #ff3f1a;
-	margin-left: -.35em;
-	font-size: 1.05em;
-}
-#ops,
-.opbox {
-	border: 1px solid #3a3a3a;
-	box-shadow: 0 0 1em #222 inset;
-}
-#ops {
-	display: none;
-	background: #333;
-	margin: 1.7em 1.5em 0 1.5em;
-	padding: .3em .6em;
-	border-radius: .3em;
-	border-width: .15em 0;
-}
-.opbox {
-	background: #2d2d2d;
-	margin: 1.5em 0 0 0;
-	padding: .5em;
-	border-radius: 0 1em 1em 0;
-	border-width: .15em .3em .3em 0;
-	max-width: 40em;
-}
-.opbox input {
-	margin: .5em;
-}
-.opbox input[type=text] {
-	color: #fff;
-	background: #383838;
-	border: none;
-	box-shadow: 0 0 .3em #222;
-	border-bottom: 1px solid #fc5;
-	border-radius: .2em;
-	padding: .2em .3em;
-}
+
 #op_up2k {
 	padding: 0 1em 1em 1em;
 }
@@ -94,6 +6,9 @@
 	position: absolute;
 	top: 0;
 	left: 0;
+	width: 2px;
+	height: 2px;
+	overflow: hidden;
 }
 #u2form input {
 	background: #444;
@@ -104,10 +19,10 @@
 	color: #f87;
 	padding: .5em;
 }
-#u2form {
-	width: 2px;
-	height: 2px;
-	overflow: hidden;
+#u2err.msg {
+	color: #999;
+	padding: .5em;
+	font-size: .9em;
 }
 #u2btn {
 	color: #eee;
@@ -117,17 +32,32 @@
 	background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%);
 	filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0);
 	text-decoration: none;
-	line-height: 1.5em;
+	line-height: 1.3em;
 	border: 1px solid #222;
 	border-radius: .4em;
 	text-align: center;
-	font-size: 2em;
-	margin: 1em auto;
-	padding: 1em 0;
-	width: 12em;
+	font-size: 1.5em;
+	margin: .5em auto;
+	padding: .8em 0;
+	width: 16em;
 	cursor: pointer;
 	box-shadow: .4em .4em 0 #111;
 }
+#op_up2k.srch #u2btn {
+	background: linear-gradient(to bottom, #ca3 0%, #fd8 50%, #fc6 51%, #b92 100%);
+	text-shadow: 1px 1px 1px #fc6;
+	color: #333;
+}
+#u2conf #u2btn {
+	margin: -1.5em 0;
+	padding: .8em 0;
+	width: 100%;
+	max-width: 12em;
+	display: inline-block;
+}
+#u2conf #u2btn_cw {
+	text-align: right;
+}
 #u2notbtn {
 	display: none;
 	text-align: center;
@@ -142,6 +72,9 @@
 	width: calc(100% - 2em);
 	max-width: 100em;
 }
+#op_up2k.srch #u2tab {
+	max-width: none;
+}
 #u2tab td {
 	border: 1px solid #ccc;
 	border-width: 0 0px 1px 0;
@@ -149,16 +82,61 @@
 }
 #u2tab td:nth-child(2) {
 	width: 5em;
+	white-space: nowrap;
 }
 #u2tab td:nth-child(3) {
 	width: 40%;
 }
-#u2tab tr+tr:hover td {
+#op_up2k.srch #u2tab td:nth-child(3) {
+	font-family: sans-serif;
+	width: auto;
+}
+#u2tab tbody tr:hover td {
 	background: #222;
 }
+#u2cards {
+	padding: 1em 0 .3em 1em;
+	margin: 1.5em auto -2.5em auto;
+	white-space: nowrap;
+	text-align: center;
+	overflow: hidden;
+}
+#u2cards.w {
+	width: 45em;
+	text-align: left;
+}
+#u2cards a {
+	padding: .2em 1em;
+	border: 1px solid #777;
+	border-width: 0 0 1px 0;
+	background: linear-gradient(to bottom, #333, #222);
+}
+#u2cards a:first-child {
+	border-radius: .4em 0 0 0;
+}
+#u2cards a:last-child {
+	border-radius: 0 .4em 0 0;
+}
+#u2cards a.act {
+	padding-bottom: .5em;
+	border-width: 1px 1px .1em 1px;
+	border-radius: .3em .3em 0 0;
+	margin-left: -1px;
+	background: linear-gradient(to bottom, #464, #333 80%);
+	box-shadow: 0 -.17em .67em #280;
+	border-color: #7c5 #583 #333 #583;
+	position: relative;
+	color: #fd7;
+}
+#u2cards span {
+	color: #fff;
+}
 #u2conf {
 	margin: 1em auto;
-	width: 26em;
+	width: 30em;
+}
+#u2conf.has_btn {
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;
@@ -169,12 +147,16 @@
 	outline: none;
 }
 #u2conf .txtbox {
-	width: 4em;
+	width: 3em;
 	color: #fff;
 	background: #444;
 	border: 1px solid #777;
 	font-size: 1.2em;
 	padding: .15em 0;
+	height: 1.05em;
+}
+#u2conf .txtbox.err {
+	background: #922;
 }
 #u2conf a {
 	color: #fff;
@@ -183,39 +165,110 @@
 	border-radius: .1em;
 	font-size: 1.5em;
 	padding: .1em 0;
-	margin: 0 -.25em;
+	margin: 0 -1px;
 	width: 1.5em;
 	height: 1em;
 	display: inline-block;
 	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
+	bottom: -0.08em;
 }
 #u2conf input+a {
 	background: #d80;
 }
+#u2conf label {
+	font-size: 1.6em;
+	width: 2em;
+	height: 1em;
+	padding: .4em 0;
+	display: block;
+	border-radius: .25em;
+}
+#u2conf input[type="checkbox"] {
+	position: relative;
+	opacity: .02;
+	top: 2em;
+}
+#u2conf input[type="checkbox"]+label {
+	position: relative;
+	background: #603;
+	border-bottom: .2em solid #a16;
+	box-shadow: 0 .1em .3em #a00 inset;
+}
+#u2conf input[type="checkbox"]:checked+label {
+	background: #6a1;
+	border-bottom: .2em solid #efa;
+	box-shadow: 0 .1em .5em #0c0;
+}
+#u2conf input[type="checkbox"]+label:hover {
+	box-shadow: 0 .1em .3em #fb0;
+	border-color: #fb0;
+}
+#op_up2k.srch #u2conf td:nth-child(1)>*,
+#op_up2k.srch #u2conf td:nth-child(2)>*,
+#op_up2k.srch #u2conf td:nth-child(3)>* {
+	background: #777;
+	border-color: #ccc;
+	box-shadow: none;
+	opacity: .2;
+}
 #u2foot {
 	color: #fff;
 	font-style: italic;
 }
+#u2foot span {
+	color: #999;
+	font-size: .9em;
+}
+#u2footfoot {
+	margin-bottom: -1em;
+}
 .prog {
 	font-family: monospace;
 }
-.prog>div {
-	display: inline-block;
-	position: relative;
-	overflow: hidden;
-	margin: 0;
-	padding: 0;
-	height: 1.1em;
-	margin-bottom: -.15em;
-	box-shadow: -1px -1px 0 inset rgba(255,255,255,0.1);
+#u2tab a>span {
+	font-weight: bold;
+	font-style: italic;
+	color: #fff;
+	padding-left: .2em;
 }
-.prog>div>div {
-	width: 0%;
-	position: absolute;
-	left: 0;
-	top: 0;
-	bottom: 0;
-	background: #0a0;
+#u2cleanup {
+	float: right;
+	margin-bottom: -.3em;
+}
+
+
+
+
+
+html.light #u2btn {
+	box-shadow: .4em .4em 0 #ccc;
+}
+html.light #u2cards span {
+	color: #000;
+}
+html.light #u2cards a {
+	background: linear-gradient(to bottom, #eee, #fff);
+}
+html.light #u2cards a.act {
+	color: #037;
+	background: inherit;
+	box-shadow: 0 -.17em .67em #0ad;
+	border-color: #09c #05a #eee #05a;
+}
+html.light #u2conf .txtbox {
+	background: #fff;
+	color: #444;
+}
+html.light #u2conf .txtbox.err {
+	background: #f96;
+	color: #300;
+}
+html.light #op_up2k.srch #u2btn {
+	border-color: #a80;
+}
+html.light #u2foot {
+	color: #000;
+}
+html.light #u2tab tbody tr:hover td {
+	background: #fff;
 }

copyparty/web/upload.html

@@ -1,70 +0,0 @@
-    <div id="ops"><a
-        href="#" data-dest="">---</a><i></i><a
-        href="#" data-dest="up2k">up2k</a><i></i><a
-        href="#" data-dest="bup">bup</a><i></i><a
-        href="#" data-dest="mkdir">mkdir</a><i></i><a
-        href="#" data-dest="new_md">new.md</a></div>
-
-    <div id="op_bup" class="opview opbox act">
-        <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
-            <input type="hidden" name="act" value="bput" />
-            <input type="file" name="f" multiple><br />
-            <input type="submit" value="start upload">
-        </form>
-    </div>
-
-    <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
-            <input type="hidden" name="act" value="mkdir" />
-            <input type="text" name="name" size="30">
-            <input type="submit" value="mkdir">
-        </form>
-    </div>
-
-    <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
-            <input type="hidden" name="act" value="new_md" />
-            <input type="text" name="name" size="30">
-            <input type="submit" value="create doc">
-        </form>
-    </div>
-
-    <div id="op_up2k" class="opview">
-        <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
-
-            <table id="u2conf">
-                <tr>
-                    <td>parallel uploads</td>
-                </tr>
-                <tr>
-                    <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
-                    </td>
-                    <td rowspan="2">
-                        <input type="checkbox" id="multitask" />
-                        <label for="multitask">hash while<br />uploading</label>
-                    </td>
-                </tr>
-            </table>
-
-            <div id="u2notbtn"></div>
-
-            <div id="u2btn">
-                drop files here<br />
-                (or click me)
-            </div>
-
-            <table id="u2tab">
-                <tr>
-                    <td>filename</td>
-                    <td>status</td>
-                    <td>progress</td>
-                </tr>
-            </table>
-
-            <p id="u2foot"></p>
-            <p>( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
-    </div>

copyparty/web/util.js

@@ -0,0 +1,546 @@
+"use strict";
+
+if (!window['console'])
+    window['console'] = {
+        "log": function (msg) { }
+    };
+
+
+var is_touch = 'ontouchstart' in window,
+    ANDROID = /(android)/i.test(navigator.userAgent);
+
+
+// error handler for mobile devices
+function hcroak(msg) {
+    document.body.innerHTML = msg;
+    window.onerror = undefined;
+    throw 'fatal_err';
+}
+function croak(msg) {
+    document.body.textContent = msg;
+    window.onerror = undefined;
+    throw msg;
+}
+function esc(txt) {
+    return txt.replace(/[&"<>]/g, function (c) {
+        return {
+            '&': '&amp;',
+            '"': '&quot;',
+            '<': '&lt;',
+            '>': '&gt;'
+        }[c];
+    });
+}
+function vis_exh(msg, url, lineNo, columnNo, error) {
+    window.onerror = undefined;
+    window['vis_exh'] = null;
+    var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
+        esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
+
+    if (error) {
+        var find = ['desc', 'stack', 'trace'];
+        for (var a = 0; a < find.length; a++)
+            if (String(error[find[a]]) !== 'undefined')
+                html.push('<h2>' + find[a] + '</h2>' +
+                    esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
+    }
+    document.body.style.fontSize = '0.8em';
+    document.body.style.padding = '0 1em 1em 1em';
+    hcroak(html.join('\n'));
+}
+
+
+var ebi = document.getElementById.bind(document),
+    QS = document.querySelector.bind(document),
+    QSA = document.querySelectorAll.bind(document),
+    mknod = document.createElement.bind(document);
+
+
+function ev(e) {
+    e = e || window.event;
+    if (!e)
+        return;
+
+    if (e.preventDefault)
+        e.preventDefault()
+
+    if (e.stopPropagation)
+        e.stopPropagation();
+
+    e.returnValue = false;
+    return e;
+}
+
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
+if (!String.prototype.endsWith) {
+    String.prototype.endsWith = function (search, this_len) {
+        if (this_len === undefined || this_len > this.length) {
+            this_len = this.length;
+        }
+        return this.substring(this_len - search.length, this_len) === search;
+    };
+}
+if (!String.startsWith) {
+    String.prototype.startsWith = function (s, i) {
+        i = i > 0 ? i | 0 : 0;
+        return this.substring(i, i + s.length) === s;
+    };
+}
+
+
+// https://stackoverflow.com/a/950146
+function import_js(url, cb) {
+    var head = document.head || document.getElementsByTagName('head')[0];
+    var script = mknod('script');
+    script.type = 'text/javascript';
+    script.src = url;
+
+    script.onreadystatechange = cb;
+    script.onload = cb;
+
+    head.appendChild(script);
+}
+
+
+var crctab = (function () {
+    var c, tab = [];
+    for (var n = 0; n < 256; n++) {
+        c = n;
+        for (var k = 0; k < 8; k++) {
+            c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+        }
+        tab[n] = c;
+    }
+    return tab;
+})();
+
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+        crc = (crc >>> 8) ^ crctab[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return ((crc ^ (-1)) >>> 0).toString(16);
+}
+
+
+function clmod(obj, cls, add) {
+    var re = new RegExp('\\s*\\b' + cls + '\\s*\\b', 'g');
+    if (add == 't')
+        add = !re.test(obj.className);
+
+    obj.className = obj.className.replace(re, ' ') + (add ? ' ' + cls : '');
+}
+
+
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name == 'ts')
+                typ = 'int';
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v) {
+                        if (v.slice(-1) == '/')
+                            v = '\t' + v;
+
+                        v = uricom_dec(v)[0]
+                    }
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map(function (x) { return x; });
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+        console.log("resetting fsort " + sread('fsort'))
+        localStorage.removeItem('fsort');
+    }
+    return nodes;
+}
+
+
+function sortTable(table, col, cb) {
+    var tb = table.tBodies[0],
+        th = table.tHead.rows[0].cells,
+        tr = Array.prototype.slice.call(tb.rows, 0),
+        i, reverse = th[col].className.indexOf('sort1') !== -1 ? -1 : 1;
+    for (var a = 0, thl = th.length; a < thl; a++)
+        th[a].className = th[a].className.replace(/ *sort-?1 */, " ");
+    th[col].className += ' sort' + reverse;
+    var stype = th[col].getAttribute('sort');
+    try {
+        var nrules = [], rules = jread("fsort", []);
+        rules.unshift([th[col].getAttribute('name'), reverse, stype || '']);
+        for (var a = 0; a < rules.length; a++) {
+            var add = true;
+            for (var b = 0; b < a; b++)
+                if (rules[a][0] == rules[b][0])
+                    add = false;
+
+            if (add)
+                nrules.push(rules[a]);
+
+            if (nrules.length >= 10)
+                break;
+        }
+        jwrite("fsort", nrules);
+    }
+    catch (ex) {
+        console.log("failed to persist sort rules, resetting: " + ex);
+        jwrite("fsort", null);
+    }
+    var vl = [];
+    for (var a = 0; a < tr.length; a++) {
+        var cell = tr[a].cells[col];
+        if (!cell) {
+            vl.push([null, a]);
+            continue;
+        }
+        var v = cell.getAttribute('sortv') || cell.textContent.trim();
+        if (stype == 'int') {
+            v = parseInt(v.replace(/[, ]/g, '')) || 0;
+        }
+        vl.push([v, a]);
+    }
+    vl.sort(function (a, b) {
+        a = a[0];
+        b = b[0];
+        if (a === null)
+            return -1;
+        if (b === null)
+            return 1;
+
+        if (stype == 'int') {
+            return reverse * (a - b);
+        }
+        return reverse * (a.localeCompare(b));
+    });
+    for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
+    if (cb) cb();
+}
+function makeSortable(table, cb) {
+    var th = table.tHead, i;
+    th && (th = th.rows[0]) && (th = th.cells);
+    if (th) i = th.length;
+    else return; // if no `<thead>` then do nothing
+    while (--i >= 0) (function (i) {
+        th[i].onclick = function (e) {
+            ev(e);
+            sortTable(table, i, cb);
+        };
+    }(i));
+}
+
+
+function linksplit(rp) {
+    var ret = [];
+    var apath = '/';
+    if (rp && rp.charAt(0) == '/')
+        rp = rp.slice(1);
+
+    while (rp) {
+        var link = rp;
+        var ofs = rp.indexOf('/');
+        if (ofs === -1) {
+            rp = null;
+        }
+        else {
+            link = rp.slice(0, ofs + 1);
+            rp = rp.slice(ofs + 1);
+        }
+        var vlink = esc(link),
+            elink = uricom_enc(link);
+
+        if (link.indexOf('/') !== -1) {
+            vlink = vlink.slice(0, -1) + '<span>/</span>';
+            elink = elink.slice(0, -3) + '/';
+        }
+
+        ret.push('<a href="' + apath + elink + '">' + vlink + '</a>');
+        apath += elink;
+    }
+    return ret;
+}
+
+
+function uricom_enc(txt, do_fb_enc) {
+    try {
+        return encodeURIComponent(txt);
+    }
+    catch (ex) {
+        console.log("uce-err [" + txt + "]");
+        if (do_fb_enc)
+            return esc(txt);
+
+        return txt;
+    }
+}
+
+
+function uricom_dec(txt) {
+    try {
+        return [decodeURIComponent(txt), true];
+    }
+    catch (ex) {
+        console.log("ucd-err [" + txt + "]");
+        return [txt, false];
+    }
+}
+
+
+function get_evpath() {
+    var ret = document.location.pathname;
+
+    if (ret.indexOf('/') !== 0)
+        ret = '/' + ret;
+
+    if (ret.lastIndexOf('/') !== ret.length - 1)
+        ret += '/';
+
+    return ret;
+}
+
+
+function get_vpath() {
+    return uricom_dec(get_evpath())[0];
+}
+
+
+function get_pwd() {
+	var pwd = ('; ' + document.cookie).split('; cppwd=');
+	if (pwd.length < 2)
+		return null;
+
+	return pwd[1].split(';')[0];
+}
+
+
+function unix2iso(ts) {
+    return new Date(ts * 1000).toISOString().replace("T", " ").slice(0, -5);
+}
+
+
+function s2ms(s) {
+    s = Math.floor(s);
+    var m = Math.floor(s / 60);
+    return m + ":" + ("0" + (s - m * 60)).slice(-2);
+}
+
+
+function has(haystack, needle) {
+    for (var a = 0; a < haystack.length; a++)
+        if (haystack[a] == needle)
+            return true;
+
+    return false;
+}
+
+
+function sread(key) {
+    if (window.localStorage)
+        return localStorage.getItem(key);
+
+    return null;
+}
+
+function swrite(key, val) {
+    if (window.localStorage) {
+        if (val === undefined || val === null)
+            localStorage.removeItem(key);
+        else
+            localStorage.setItem(key, val);
+    }
+}
+
+function jread(key, fb) {
+    var str = sread(key);
+    if (!str)
+        return fb;
+
+    return JSON.parse(str);
+}
+
+function jwrite(key, val) {
+    if (!val)
+        swrite(key);
+    else
+        swrite(key, JSON.stringify(val));
+}
+
+function icfg_get(name, defval) {
+    return parseInt(fcfg_get(name, defval));
+}
+
+function fcfg_get(name, defval) {
+    var o = ebi(name);
+
+    var val = parseFloat(sread(name));
+    if (isNaN(val))
+        return parseFloat(o ? o.value : defval);
+
+    if (o)
+        o.value = val;
+
+    return val;
+}
+
+function bcfg_get(name, defval) {
+    var o = ebi(name);
+    if (!o)
+        return defval;
+
+    var val = sread(name);
+    if (val === null)
+        val = defval;
+    else
+        val = (val == '1');
+
+    bcfg_upd_ui(name, val);
+    return val;
+}
+
+function bcfg_set(name, val) {
+    swrite(name, val ? '1' : '0');
+    bcfg_upd_ui(name, val);
+    return val;
+}
+
+function bcfg_upd_ui(name, val) {
+    var o = ebi(name);
+    if (!o)
+        return;
+
+    if (o.getAttribute('type') == 'checkbox')
+        o.checked = val;
+    else if (o) {
+        clmod(o, 'on', val);
+    }
+}
+
+
+function hist_push(url) {
+    console.log("h-push " + url);
+    history.pushState(url, url, url);
+}
+
+function hist_replace(url) {
+    console.log("h-repl " + url);
+    history.replaceState(url, url, url);
+}
+
+
+var tt = (function () {
+    var r = {
+        "tt": mknod("div"),
+        "en": true
+    };
+
+    r.tt.setAttribute('id', 'tt');
+    document.body.appendChild(r.tt);
+
+    function show() {
+        var cfg = sread('tooltips');
+        if (cfg !== null && cfg != '1')
+            return;
+
+        var msg = this.getAttribute('tt');
+        if (!msg)
+            return;
+
+        var pos = this.getBoundingClientRect(),
+            left = pos.left < window.innerWidth / 2,
+            top = pos.top < window.innerHeight / 2;
+
+        r.tt.style.top = top ? pos.bottom + 'px' : 'auto';
+        r.tt.style.bottom = top ? 'auto' : (window.innerHeight - pos.top) + 'px';
+        r.tt.style.left = left ? pos.left + 'px' : 'auto';
+        r.tt.style.right = left ? 'auto' : (window.innerWidth - pos.right) + 'px';
+
+        r.tt.innerHTML = msg.replace(/\$N/g, "<br />");
+        clmod(r.tt, 'show', 1);
+    }
+
+    function hide() {
+        clmod(r.tt, 'show');
+    }
+
+    r.init = function () {
+        var ttb = ebi('tooltips');
+        if (ttb) {
+            ttb.onclick = function (e) {
+                ev(e);
+                r.en = !r.en;
+                bcfg_set('tooltips', r.en);
+                r.init();
+            };
+            r.en = bcfg_get('tooltips', true)
+        }
+
+        var _show = r.en ? show : null,
+            _hide = r.en ? hide : null;
+
+        var o = QSA('*[tt]');
+        for (var a = o.length - 1; a >= 0; a--) {
+            o[a].onfocus = _show;
+            o[a].onblur = _hide;
+            o[a].onmouseenter = _show;
+            o[a].onmouseleave = _hide;
+        }
+        hide();
+    };
+
+    return r;
+})();

docs/README.md

@@ -0,0 +1,22 @@
+# example `.epilogue.html`
+save one of these as `.epilogue.html` inside a folder to customize it:
+
+* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+
+
+# example browser-css
+point `--css-browser` to one of these by URL:
+
+* [`browser.css`](browser.css) changes the background
+* [`browser-icons.css`](browser-icons.css) adds filetype icons
+
+
+
+# other stuff
+
+## [`rclone.md`](rclone.md)
+* notes on using rclone as a fuse client/server
+
+## [`example.conf`](example.conf)
+* example config file for `-c` which never really happened

docs/biquad.html

@@ -0,0 +1,95 @@
+<!DOCTYPE html><html><head></head><body><script>
+
+setTimeout(location.reload.bind(location), 700);
+document.documentElement.scrollLeft = 0;
+
+var can = document.createElement('canvas'),
+    cc = can.getContext('2d'),
+    w = 2048,
+    h = 1024;
+
+w = 2048;
+
+can.width = w;
+can.height = h;
+document.body.appendChild(can);
+can.style.cssText = 'width:' + w + 'px;height:' + h + 'px';
+
+cc.fillStyle = '#000';
+cc.fillRect(0, 0, w, h);
+
+var cfg = [ // hz, q, g
+    [31.25 * 0.88, 0, 1.4],  // shelf
+    [31.25 * 1.04, 0.7, 0.96],  // peak
+    [62.5, 0.7, 1],
+    [125, 0.8, 1],
+    [250, 0.9, 1.03],
+    [500, 0.9, 1.1],
+    [1000, 0.9, 1.1],
+    [2000, 0.9, 1.105],
+    [4000, 0.88, 1.05],
+    [8000 * 1.006, 0.73, 1.24],
+    //[16000 * 1.00, 0.5, 1.75],  // peak.v1
+    //[16000 * 1.19, 0, 1.8]  // shelf.v1
+    [16000 * 0.89, 0.7, 1.26],  // peak
+    [16000 * 1.13, 0.82, 1.09],  // peak
+    [16000 * 1.205, 0, 1.9]  // shelf
+];
+
+var freqs = new Float32Array(22000),
+    sum = new Float32Array(freqs.length),
+    ac = new AudioContext(),
+    step = w / freqs.length,
+    colors = [
+        'rgba(255, 0, 0, 0.7)',
+        'rgba(0, 224, 0, 0.7)',
+        'rgba(0, 64, 255, 0.7)'
+    ];
+
+var order = [];
+
+for (var a = 0; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var a = 1; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var ia = 0; ia < order.length; ia++) {
+    var a = order[ia],
+        fi = ac.createBiquadFilter(),
+        mag = new Float32Array(freqs.length),
+        phase = new Float32Array(freqs.length);
+
+    for (var b = 0; b < freqs.length; b++)
+        freqs[b] = b;
+
+    fi.type = a == 0 ? 'lowshelf' : a == cfg.length - 1 ? 'highshelf' : 'peaking';
+    fi.frequency.value = cfg[a][0];
+    fi.Q.value = cfg[a][1];
+    fi.gain.value = 1;
+
+    fi.getFrequencyResponse(freqs, mag, phase);
+    cc.fillStyle = colors[a % colors.length];
+    for (var b = 0; b < sum.length; b++) {
+        mag[b] -= 1;
+        sum[b] += mag[b] * cfg[a][2];
+        var y = h - (mag[b] * h * 3);
+        cc.fillRect(b * step, y, step, h - y);
+        cc.fillRect(b * step - 1, y - 1, 3, 3);
+    }
+}
+
+var min = 999999, max = 0;
+for (var a = 0; a < sum.length; a++) {
+    min = Math.min(min, sum[a]);
+    max = Math.max(max, sum[a]);
+}
+cc.fillStyle = 'rgba(255,255,255,1)';
+for (var a = 0; a < sum.length; a++) {
+    var v = (sum[a] - min) / (max - min);
+    cc.fillRect(a * step, 0, step, v * h / 2);
+}
+
+cc.fillRect(0, 460, w, 1);
+
+</script></body></html>

docs/browser-icons.css

@@ -0,0 +1,68 @@
+/* put filetype icons inline with text
+#ggrid>a>span:before,
+#ggrid>a>span.dir:before {
+	display: inline;
+	line-height: 0;
+	font-size: 1.7em;
+	margin: -.7em .1em -.5em -.6em;
+}
+*/
+
+
+/* move folder icons top-left */
+#ggrid>a>span.dir:before {
+	content: initial;
+}
+#ggrid>a[href$="/"]:before {
+	content: '📂';
+    display: block;
+    position: absolute;
+    margin: -.1em -.4em;
+    text-shadow: 0 0 .1em #000;
+    font-size: 2em;
+}
+
+
+/* put filetype icons top-left */
+#ggrid>a:before {
+    display: block;
+    position: absolute;
+    margin: -.1em -.4em;
+    text-shadow: 0 0 .1em #000;
+    font-size: 2em;
+}
+
+
+/* video */
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '📺';
+}
+
+
+/* audio */
+#ggrid>a:is(
+[href$=".mp3"i],
+[href$=".ogg"i],
+[href$=".opus"i],
+[href$=".flac"i],
+[href$=".m4a"i],
+[href$=".aac"i],
+):before {
+    content: '🎵';
+}
+
+
+/* image */
+#ggrid>a:is(
+[href$=".jpg"i],
+[href$=".jpeg"i],
+[href$=".png"i],
+[href$=".gif"i],
+[href$=".webp"i],
+):before {
+    content: '🎨';
+}

docs/browser.css

@@ -0,0 +1,29 @@
+html {
+    background: #333 url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
+}
+#files th {
+    background: rgba(32, 32, 32, 0.9) !important;
+}
+#ops,
+#treeul,
+#files td {
+    background: rgba(32, 32, 32, 0.3) !important;
+}
+
+
+html.light {
+    background: #eee url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
+}
+html.light #files th {
+    background: rgba(255, 255, 255, 0.9) !important;
+}
+html.light #ops,
+html.light #treeul,
+html.light #files td {
+    background: rgba(248, 248, 248, 0.8) !important;
+}
+
+
+#files * {
+    background: transparent !important;
+}

docs/example.conf

@@ -32,9 +32,13 @@ r
 
 # and a folder where anyone can upload
 # but nobody can see the contents
+# and set the e2d flag to enable the uploads database
+# and set the nodupe flag to reject duplicate uploads
 /home/ed/inc
 /dump
 w
+c e2d
+c nodupe
 
 # this entire config file can be replaced with these arguments:
 # -u ed:123 -u k:k -v .::r:aed -v priv:priv:rk:aed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w

docs/minimal-up2k.html

@@ -0,0 +1,32 @@
+<!--
+  save this as .epilogue.html inside a write-only folder to declutter the UI,  makes it look like
+  https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
+-->
+
+<style>
+
+    /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
+
+    #ops, #tree, #path, #wrap>h2:last-child,  /* main tabs and navigators (tree/breadcrumbs) */
+
+    #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
+
+    #u2cards  /* and the upload progress tabs */
+
+    {display: none !important}  /* do it! */
+
+
+
+    /* add some margins because now it's weird */
+    .opview {margin-top: 2.5em}
+    #op_up2k {margin-top: 3em}
+
+    /* and embiggen the upload button */
+    #u2conf #u2btn, #u2btn {padding:1.5em 0}
+
+    /* adjust the button area a bit */
+    #u2conf.has_btn {width: 35em !important; margin: 5em auto}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

docs/music-analysis.sh

@@ -0,0 +1,242 @@
+#!/bin/bash
+echo please dont actually run this as a scriopt
+exit 1
+
+
+# dependency-heavy, not particularly good fit
+pacman -S llvm10
+python3 -m pip install --user librosa
+git clone https://github.com/librosa/librosa.git
+
+
+# correct bpm for tracks with bad tags
+br='
+/Trip Trip Trip\(Hardcore Edit\).mp3/ {v=176}
+/World!!.BIG_SOS/ {v=175}
+/\/08\..*\(BIG_SOS Bootleg\)\.mp3/ {v=175}
+/もってけ！セーラ服.Asterisk DnB/ {v=175}
+/Rondo\(Asterisk DnB Re.mp3/ {v=175}
+/Ray Nautica 175 Edit/ {v=175;x="thunk"}
+/TOKIMEKI Language.Jauz/ {v=174}
+/YUPPUN Hardcore Remix\).mp3/ {v=174;x="keeps drifting"}
+/(èâAâï.î╧ûδ|バーチャリアル.狐耶)J-Core Remix\).mp3/ {v=172;x="hard"}
+/lucky train..Freezer/ {v=170}
+/Alf zero Bootleg ReMix/ {v=170}
+/Prisoner of Love.Kacky/ {v=170}
+/火炎 .Qota/ {v=170}
+/\(hu-zin Bootleg\)\.mp3/ {v=170}
+/15. STRAIGHT BET\(Milynn Bootleg\)\.mp3/ {v=170}
+/\/13.*\(Milynn Bootleg\)\.mp3/ {v=167;x="way hard"}
+/COLOR PLANET .10SAI . nijikon Remix\)\.mp3/ {v=165}
+/11\. (朝はご飯派|Æ⌐é═é▓ö╤öh)\.mp3/ {v=162}
+/09\. Where.s the core/ {v=160}
+/PLANET\(Koushif Jersey Club Bootleg\)remaster.mp3/ {v=160;x="starts ez turns bs"}
+/kened Soul - Madeon x Angel Beats!.mp3/ {v=160}
+/Dear Moments\(Mother Harlot Bootleg\)\.mp3/ {v=150}
+/POWER.Ringos UKG/ {v=140}
+/ブルー・フィールド\(Ringos UKG Remix\).mp3/ {v=135}
+/プラチナジェット.Ringo Remix..mp3/ {v=131.2}
+/Mirrorball Love \(TKM Bootleg Mix\).mp3/ {v=130}
+/Photon Melodies \(TKM Bootleg Mix\).mp3/ {v=128}
+/Trap of Love \(TKM Bootleg Mix\).mp3/ {v=128}
+/One Step \(TKM Bootleg Mix\)\.mp3/ {v=126}
+/04 (トリカムイ岩|âgâèâJâÇâCèΓ).mp3/ {v=125}
+/Get your Wish \(NAWN REMIX\)\.mp3/ {v=95}
+/Flicker .Nitro Fun/ {v=92}
+/\/14\..*suicat Remix/ {v=85.5;x="tricky"}
+/Yanagi Nagi - Harumodoki \(EO Remix\)\.mp3/ {v=150}
+/Azure - Nicology\.mp3/ {v=128;x="off by 5 how"}
+'
+
+
+# afun host, collects/grades the results
+runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | awk '{v=$1;sub(/[^ ]+ /,"")} '"$br"' {printf "%s %s\n",v,$0}' | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk 'function prt(c) {printf "\033[3%sm%s\033[0m\n",c,$0} $8!="div,"{next} $5!~/^[0-9\.]+/{next} {meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;prt(6);next} idiff==1{n1++;prt(3);next} idiff>10{nx++;prt(1);next} {n10++;prt(5)} END {printf "ok: %d   1off: %2s   (%3s)   10off: %2s   (%3s)   fail: %2s\n",n0,n1,n0+n1,n10,n0+n1+n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; }
+
+
+# ok:   8   1off: 62   ( 70)   10off: 86   (156)   fail: 25   # 105 sec,  librosa @ 8c archvm on 3700x w10
+# ok:   4   1off: 59   ( 63)   10off: 65   (128)   fail: 53   # using original tags (bad)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } runfun
+
+
+# ok: 119   1off:  5   (124)   10off:  8   (132)   fail: 49   # 51 sec,  vamp-example-fixedtempo
+# ok: 109   1off:  4   (113)   10off:  9   (122)   fail: 59   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; }; runfun
+
+
+# ok: 102   1off: 61   (163)   10off: 12   (175)   fail:  6   # 61 sec,  vamp-qm-tempotracker
+# ok:  80   1off: 48   (128)   10off: 11   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; }; runfun
+
+
+# ok: 133   1off: 32   (165)   10off: 12   (177)   fail:  3   # 51 sec,  vamp-beatroot
+# ok: 101   1off: 22   (123)   10off: 16   (139)   fail: 39   # bad-tags
+# note: some tracks fully fail to analyze (unlike the others which always provide a guess)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; }; runfun
+
+
+# ok: 124   1off:  9   (133)   10off: 40   (173)   fail:  8   # 231 sec,  essentia/full
+# ok: 109   1off:  8   (117)   10off: 22   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'import essentia; import essentia.standard as es; fe, fef = es.MusicExtractor(lowlevelStats=["mean", "stdev"], rhythmStats=["mean", "stdev"], tonalStats=["mean", "stdev"])("/dev/shm/'$core'.wav"); print("{:.2f}".format(fe["rhythm.bpm"]))')"; }; runfun
+
+
+# ok: 113   1off: 18   (131)   10off: 46   (177)   fail:  4   # 134 sec,  essentia/re2013
+# ok: 101   1off: 15   (116)   10off: 26   (142)   fail: 39   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'from essentia.standard import *; a=MonoLoader(filename="/dev/shm/'$core'.wav")(); bpm,beats,confidence,_,intervals=RhythmExtractor2013(method="multifeature")(a); print("{:.2f}".format(bpm))')"; }; runfun
+
+
+
+########################################################################
+##
+##  key detectyion
+##
+########################################################################
+
+
+
+# console scriptlet reusing keytabs from browser.js
+var m=''; for (var a=0; a<24; a++) m += 's/\\|(' + maps["traktor_sharps"][a].trim() + "|" + maps["rekobo_classic"][a].trim() + "|" + maps["traktor_musical"][a].trim() + "|" + maps["traktor_open"][a].trim() + ')$/|' + maps["rekobo_alnum"][a].trim() + '/;'; console.log(m);
+
+
+# translate to camelot
+re='s/\|(B|B|B|6d)$/|1B/;s/\|(F#|F#|Gb|7d)$/|2B/;s/\|(C#|Db|Db|8d)$/|3B/;s/\|(G#|Ab|Ab|9d)$/|4B/;s/\|(D#|Eb|Eb|10d)$/|5B/;s/\|(A#|Bb|Bb|11d)$/|6B/;s/\|(F|F|F|12d)$/|7B/;s/\|(C|C|C|1d)$/|8B/;s/\|(G|G|G|2d)$/|9B/;s/\|(D|D|D|3d)$/|10B/;s/\|(A|A|A|4d)$/|11B/;s/\|(E|E|E|5d)$/|12B/;s/\|(G#m|Abm|Abm|6m)$/|1A/;s/\|(D#m|Ebm|Ebm|7m)$/|2A/;s/\|(A#m|Bbm|Bbm|8m)$/|3A/;s/\|(Fm|Fm|Fm|9m)$/|4A/;s/\|(Cm|Cm|Cm|10m)$/|5A/;s/\|(Gm|Gm|Gm|11m)$/|6A/;s/\|(Dm|Dm|Dm|12m)$/|7A/;s/\|(Am|Am|Am|1m)$/|8A/;s/\|(Em|Em|Em|2m)$/|9A/;s/\|(Bm|Bm|Bm|3m)$/|10A/;s/\|(F#m|F#m|Gbm|4m)$/|11A/;s/\|(C#m|Dbm|Dbm|5m)$/|12A/;'
+
+
+# runner/wrapper
+runfun() { cores=8; touch run; tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, key.v from mt key join mt dur on key.w = dur.w where key.k = "key" and dur.k = ".dur" order by dur.w' | uniq -w16 | grep -vE '(Off-Key|None)$' | sed -r "s/ //g;$re" | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done| grep mir/cr | tr \| / | while read key fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); [ "$key" = "$py" ] && c=2 || c=5; printf '%4s sec, %4s orig, \033[3%dm%4s py,\033[0m %s\n' $td "$key" $c "$py" "$fn") || break; done & done; time wait 2>/dev/null; }
+
+
+# ok: 26   1off: 10   2off: 1   fail: 3   #  15 sec, keyfinder
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 -c 'import sys; import keyfinder; print(keyfinder.key(sys.argv[1]).camelot())' "/dev/shm/$core.wav")"; }; runfun
+
+
+# https://github.com/MTG/essentia/raw/master/src/examples/tutorial/example_key_by_steps_streaming.py
+# https://essentia.upf.edu/reference/std_Key.html  # edma edmm braw bgate
+sed -ri 's/^(key = Key\().*/\1profileType="bgate")/' example_key_by_steps_streaming.py
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 example_key_by_steps_streaming.py /dev/shm/$core.{wav,yml} 2>/dev/null | sed -r "s/ major//;s/ minor/m/;s/^/|/;$re;s/.//")"; }; runfun
+
+
+
+########################################################################
+##
+##  misc
+##
+########################################################################
+
+
+
+python3 -m pip install --user vamp
+
+import librosa
+d, r = librosa.load('/dev/shm/0.wav')
+d.dtype
+# dtype('float32')
+d.shape
+# (1323000,)
+d
+# array([-1.9614939e-08,  1.8037968e-08, -1.4106059e-08, ...,
+#         1.2024145e-01,  2.7462116e-01,  1.6202132e-01], dtype=float32)
+
+
+
+import vamp
+c = vamp.collect(d, r, "vamp-example-plugins:fixedtempo")
+c
+# {'list': [{'timestamp':  0.005804988, 'duration':  9.999092971, 'label': '110.0 bpm', 'values': array([109.98116], dtype=float32)}]}
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+d
+array([-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+       -0.0267825 , -0.03564296], dtype=float32)
+
+d = np.reshape(d, [1, -1])
+d
+array([[-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+        -0.0267825 , -0.03564296]], dtype=float32)
+
+
+
+import vampyhost
+print("\n".join(vampyhost.list_plugins()))
+
+mvamp:marsyas_bextract_centroid
+mvamp:marsyas_bextract_lpcc
+mvamp:marsyas_bextract_lsp
+mvamp:marsyas_bextract_mfcc
+mvamp:marsyas_bextract_rolloff
+mvamp:marsyas_bextract_scf
+mvamp:marsyas_bextract_sfm
+mvamp:marsyas_bextract_zero_crossings
+mvamp:marsyas_ibt
+mvamp:zerocrossing
+qm-vamp-plugins:qm-adaptivespectrogram
+qm-vamp-plugins:qm-barbeattracker
+qm-vamp-plugins:qm-chromagram
+qm-vamp-plugins:qm-constantq
+qm-vamp-plugins:qm-dwt
+qm-vamp-plugins:qm-keydetector
+qm-vamp-plugins:qm-mfcc
+qm-vamp-plugins:qm-onsetdetector
+qm-vamp-plugins:qm-segmenter
+qm-vamp-plugins:qm-similarity
+qm-vamp-plugins:qm-tempotracker
+qm-vamp-plugins:qm-tonalchange
+qm-vamp-plugins:qm-transcription
+vamp-aubio:aubiomelenergy
+vamp-aubio:aubiomfcc
+vamp-aubio:aubionotes
+vamp-aubio:aubioonset
+vamp-aubio:aubiopitch
+vamp-aubio:aubiosilence
+vamp-aubio:aubiospecdesc
+vamp-aubio:aubiotempo
+vamp-example-plugins:amplitudefollower
+vamp-example-plugins:fixedtempo
+vamp-example-plugins:percussiononsets
+vamp-example-plugins:powerspectrum
+vamp-example-plugins:spectralcentroid
+vamp-example-plugins:zerocrossing
+vamp-rubberband:rubberband
+
+
+
+plug = vampyhost.load_plugin("vamp-example-plugins:fixedtempo", 22050, 0)
+plug.info
+{'apiVersion': 2, 'pluginVersion': 1, 'identifier': 'fixedtempo', 'name': 'Simple Fixed Tempo Estimator', 'description': 'Study a short section of audio and estimate its tempo, assuming the tempo is constant', 'maker': 'Vamp SDK Example Plugins', 'copyright': 'Code copyright 2008 Queen Mary, University of London.  Freely redistributable (BSD license)'}
+plug = vampyhost.load_plugin("qm-vamp-plugins:qm-tempotracker", 22050, 0)
+from pprint import pprint; pprint(plug.parameters)
+
+
+
+for c in plug.parameters: print("{} \033[36m{}  [\033[33m{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], "\033[36m, \033[33m".join(c["valueNames"]), c["valueNames"][int(c["defaultValue"])])) if "valueNames" in c else print("{} \033[36m{}  [\033[33m{}..{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], c["minValue"], c["maxValue"], c["defaultValue"]))
+
+
+
+beatroot-vamp:beatroot
+cl=c["list"]; 60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"])))
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+# 128 bpm, key 5A Cm
+
+import vamp
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40})
+c["list"][0]["label"]
+# 127.6 bpm
+
+c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150})
+print("\n".join([v["label"] for v in c["list"] if v["label"]]))
+v = [float(x["label"].split(' ')[0]) for x in c["list"] if x["label"]]
+v = list(sorted(v))[len(v)//4:-len(v)//4]
+v = sum(v) / len(v)
+# 128.1 bpm
+

docs/notes.sh

@@ -3,6 +3,21 @@ echo not a script
 exit 1
 
 
+##
+## delete all partial uploads
+##  (supports linux/macos, probably windows+msys2)
+
+gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f -- "$f"; done
+gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done
+
+
+##
+## detect partial uploads based on file contents
+##  (in case of context loss or old copyparties)
+
+echo; find -type f | while IFS= read -r x; do printf '\033[A\033[36m%s\033[K\033[0m\n' "$x"; tail -c$((1024*1024)) <"$x" | xxd -a | awk 'NR==1&&/^[0: ]+.{16}$/{next} NR==2&&/^\*$/{next} NR==3&&/^[0f]+: [0 ]+65 +.{16}$/{next} {e=1} END {exit e}' || continue; printf '\033[A\033[31msus:\033[33m %s \033[0m\n\n' "$x"; done
+
+
 ##
 ## create a test payload
 
@@ -52,6 +67,56 @@ wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:392
 shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*1024*1024)); printf $(tail -c +$((v+1)) "$f" | head -c $((w-v)) | sha512sum | cut -c-64 | sed -r 's/ .*//;s/(..)/\\x\1/g') | base64 -w0 | cut -c-43 | tr '+/' '-_'; v=$w; [ $v -lt $sz ] || break; done; }
 
 
+##
+## poll url for performance issues
+
+command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
+
+
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+
+##
+## bash oneliners
+
+# get the size and video-id of all youtube vids in folder, assuming filename ends with -id.ext, and create a copyparty search query
+find -maxdepth 1 -printf '%s %p\n' | sort -n | awk '!/-([0-9a-zA-Z_-]{11})\.(mkv|mp4|webm)$/{next} {sub(/\.[^\.]+$/,"");n=length($0);v=substr($0,n-10);print $1, v}' | tee /dev/stderr | awk 'BEGIN {p="("} {printf("%s name like *-%s.* ",p,$2);p="or"} END {print ")\n"}' | cat >&2
+
+# unique stacks in a stackdump
+f=a; rm -rf stacks; mkdir stacks; grep -E '^#' $f | while IFS= read -r n; do awk -v n="$n" '!$0{o=0} o; $0==n{o=1}' <$f >stacks/f; h=$(sha1sum <stacks/f | cut -c-16); mv stacks/f stacks/$h-"$n"; done ; find stacks/ | sort | uniq -cw24
+
+
+##
+## sqlite3 stuff
+
+# find dupe metadata keys
+sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = mt2.k and mt1.rowid != mt2.rowid'
+
+# partial reindex by deleting all tags for a list of files
+time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
+cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
+
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
+# unschedule mtp scan for all files somewhere under "enc/"
+sqlite3 -readonly up2k.db 'select substr(up.w,1,16) from up inner join mt on mt.w = substr(up.w,1,16) where rd like "enc/%" and +mt.k = "t:mtp"' > keys; awk '{printf "delete from mt where w = \"%s\" and +k = \"t:mtp\";\n", $0}' <keys | tee /dev/stderr | sqlite3 up2k.db
+
+
+##
+## media
+
+# split track into test files
+e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
+
+-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
+sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
+
+
 ##
 ## vscode
 
@@ -81,6 +146,22 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 brew install python@2
 pip install virtualenv
 
+# readme toc
+cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
+
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
+# determine server version
+git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+
 
 ##
 ## http 206
@@ -106,7 +187,7 @@ Range: bytes=26-         Content-Range: bytes */26
 
 var tsh = [];
 function convert_markdown(md_text, dest_dom) {
-    tsh.push(new Date().getTime());
+    tsh.push(Date.now());
     while (tsh.length > 10)
         tsh.shift();
     if (tsh.length > 1) {
@@ -122,3 +203,4 @@ mk() { rm -rf /tmp/foo; sudo -u ed bash -c 'mkdir /tmp/foo; echo hi > /tmp/foo/b
 mk && t0="$(date)" && while true; do date -s "$(date '+ 1 hour')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
 mk && sudo -u ed flock /tmp/foo sleep 40 & sleep 1; ps aux | grep -E 'sleep 40$' && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; done; echo "$t0"
 mk && t0="$(date)" && for n in {1..40}; do date -s "$(date '+ 1 day')"; systemd-tmpfiles --clean; ls -1 /tmp | grep foo || break; tar -cf/dev/null /tmp/foo; done; echo "$t0"
+

docs/nuitka.txt

@@ -0,0 +1,82 @@
+# recipe for building an exe with nuitka (extreme jank edition)
+#
+# NOTE: win7 and win10 builds both work on win10 but
+#   on win7 they immediately c0000005 in kernelbase.dll
+#
+# first install python-3.6.8-amd64.exe
+#   [x] add to path
+#
+# copypaste the rest of this file into cmd
+
+rem from pypi
+cd \users\ed\downloads
+python -m pip install --user Nuitka-0.6.14.7.tar.gz
+
+rem https://github.com/brechtsanders/winlibs_mingw/releases/download/10.2.0-11.0.0-8.0.0-r5/winlibs-x86_64-posix-seh-gcc-10.2.0-llvm-11.0.0-mingw-w64-8.0.0-r5.zip
+mkdir C:\Users\ed\AppData\Local\Nuitka\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\x86_64\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\x86_64\10.2.0-11.0.0-8.0.0-r5\
+copy c:\users\ed\downloads\winlibs-x86_64-posix-seh-gcc-10.2.0-llvm-11.0.0-mingw-w64-8.0.0-r5.zip C:\Users\ed\AppData\Local\Nuitka\Nuitka\gcc\x86_64\10.2.0-11.0.0-8.0.0-r5\winlibs-x86_64-posix-seh-gcc-10.2.0-llvm-11.0.0-mingw-w64-8.0.0-r5.zip
+
+rem https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12-windows-32.zip
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\ccache\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\ccache\v3.7.12\
+copy c:\users\ed\downloads\ccache-3.7.12-windows-32.zip C:\Users\ed\AppData\Local\Nuitka\Nuitka\ccache\v3.7.12\ccache-3.7.12-windows-32.zip
+
+rem https://dependencywalker.com/depends22_x64.zip
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\depends\
+mkdir C:\Users\ed\AppData\Local\Nuitka\Nuitka\depends\x86_64\
+copy c:\users\ed\downloads\depends22_x64.zip C:\Users\ed\AppData\Local\Nuitka\Nuitka\depends\x86_64\depends22_x64.zip
+
+cd \
+rd /s /q %appdata%\..\local\temp\pe-copyparty
+cd \users\ed\downloads
+python copyparty-sfx.py -h
+cd %appdata%\..\local\temp\pe-copyparty\copyparty
+
+python
+import os, re
+os.rename('../dep-j2/jinja2', '../jinja2')
+os.rename('../dep-j2/markupsafe', '../markupsafe')
+
+print("# nuitka dies if .__init__.stuff is imported")
+with open('__init__.py','r',encoding='utf-8') as f:
+ t1 = f.read()
+
+with open('util.py','r',encoding='utf-8') as f:
+ t2 = f.read().split('\n')[3:]
+
+t2 = [x for x in t2 if 'from .__init__' not in x]
+t = t1 + '\n'.join(t2)
+with open('__init__.py','w',encoding='utf-8') as f:
+ f.write('\n')
+
+with open('util.py','w',encoding='utf-8') as f:
+ f.write(t)
+
+print("# local-imports fail, prefix module names")
+ptn = re.compile(r'^( *from )(\.[^ ]+ import .*)')
+for d, _, fs in os.walk('.'):
+ for f in fs:
+  fp = os.path.join(d, f)
+  if not fp.endswith('.py'):
+   continue
+  t = ''
+  with open(fp,'r',encoding='utf-8') as f:
+   for ln in [x.rstrip('\r\n') for x in f]:
+    m = ptn.match(ln)
+    if not m:
+     t += ln + '\n'
+     continue
+    p1, p2 = m.groups()
+    t += "{}copyparty{}\n".format(p1, p2).replace("__init__", "util")
+  with open(fp,'w',encoding='utf-8') as f:
+   f.write(t)
+
+exit()
+
+cd ..
+
+rd /s /q bout & python -m nuitka --standalone --onefile --windows-onefile-tempdir --python-flag=no_site --assume-yes-for-downloads --include-data-dir=copyparty\web=copyparty\web --include-data-dir=copyparty\res=copyparty\res --run --output-dir=bout --mingw64 --include-package=markupsafe --include-package=jinja2 copyparty

docs/tcp-debug.sh

@@ -0,0 +1,32 @@
+(cd ~/dev/copyparty && strace -Tttyyvfs 256 -o strace.strace python3 -um copyparty -i 127.0.0.1 --http-only --stackmon /dev/shm/cpps,10 ) 2>&1 | tee /dev/stderr > ~/log-copyparty-$(date +%Y-%m%d-%H%M%S).txt
+
+14/Jun/2021:16:34:02 1623688447.212405 death
+14/Jun/2021:16:35:02 1623688502.420860 back
+
+tcpdump -nni lo -w /home/ed/lo.pcap
+
+# 16:35:25.324662 IP 127.0.0.1.48632 > 127.0.0.1.3920: Flags [F.], seq 849, ack 544, win 359, options [nop,nop,TS val 809396796 ecr 809396796], length 0
+
+tcpdump -nnr /home/ed/lo.pcap | awk '/ > 127.0.0.1.3920: /{sub(/ > .*/,"");sub(/.*\./,"");print}' | sort -n | uniq | while IFS= read -r port; do echo; tcpdump -nnr /home/ed/lo.pcap 2>/dev/null | grep -E "\.$port( > |: F)" | sed -r 's/ > .*, /, /'; done | grep -E '^16:35:0.*length [^0]' -C50
+
+16:34:02.441732 IP 127.0.0.1.48638, length 0
+16:34:02.441738 IP 127.0.0.1.3920, length 0
+16:34:02.441744 IP 127.0.0.1.48638, length 0
+16:34:02.441756 IP 127.0.0.1.48638, length 791
+16:34:02.441759 IP 127.0.0.1.3920, length 0
+16:35:02.445529 IP 127.0.0.1.48638, length 0
+16:35:02.489194 IP 127.0.0.1.3920, length 0
+16:35:02.515595 IP 127.0.0.1.3920, length 216
+16:35:02.515600 IP 127.0.0.1.48638, length 0
+
+grep 48638 "$(find ~ -maxdepth 1 -name log-copyparty-\*.txt | sort | tail -n 1)"
+
+1623688502.510380 48638 rh
+1623688502.511291 48638 Unrecv direct ...
+1623688502.511827 48638 rh = 791
+16:35:02.518 127.0.0.1 48638       shut(8): [Errno 107] Socket not connected
+Exception in thread httpsrv-0.1-48638:
+
+grep 48638 ~/dev/copyparty/strace.strace
+14561 16:35:02.506310 <... accept4 resumed> {sa_family=AF_INET, sin_port=htons(48638), sin_addr=inet_addr("127.0.0.1")}, [16], SOCK_CLOEXEC) = 8<TCP:[127.0.0.1:3920->127.0.0.1:48638]> <0.000012>
+15230 16:35:02.510725 write(1<pipe:[256639555]>, "1623688502.510380 48638 rh\n", 27 <unfinished ...>

scripts/copyparty-repack.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+repacker=1
+set -e
+
+# -- download latest copyparty (source.tgz and sfx),
+# -- build minimal sfx versions,
+# -- create a .tar.gz bundle
+#
+# convenient for deploying updates to inconvenient locations
+#  (and those are usually linux so bash is good inaff)
+#   (but that said this even has macos support)
+#
+# bundle will look like:
+# -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty
+# -rw-r--r--  0 ed ed  491318 Nov 19 00:40 copyparty-extras/copyparty-0.5.4.tar.gz
+# -rwxr-xr-x  0 ed ed   30254 Nov 17 23:58 copyparty-extras/copyparty-fuse.py
+# -rwxr-xr-x  0 ed ed  481403 Nov 19 00:40 copyparty-extras/sfx-full/copyparty-sfx.sh
+# -rwxr-xr-x  0 ed ed  506043 Nov 19 00:40 copyparty-extras/sfx-full/copyparty-sfx.py
+# -rwxr-xr-x  0 ed ed  167699 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.sh
+# -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+td="$(mktemp -d)"
+od="$(pwd)"
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	command -v curl >/dev/null && exec curl -L --remote-name-all "$@"
+	exec wget "$@"
+}
+export -f dl_files
+
+
+# if cache exists, use that instead of bothering github
+cache="$od/.copyparty-repack.cache"
+[ -e "$cache" ] &&
+	tar -xf "$cache" ||
+{
+	# get download links from github
+	dl_text https://api.github.com/repos/9001/copyparty/releases/latest |
+	(
+		# prefer jq if available
+		jq -r '.assets[]|select(.name|test("-sfx|tar.gz")).browser_download_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"browser_download_url".*(\.tar\.gz|-sfx\.)/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+
+	tar -czf "$cache" *
+}
+
+
+# move src into copyparty-extras/,
+# move sfx into copyparty-extras/sfx-full/
+mkdir -p copyparty-extras/sfx-{full,lite}
+mv copyparty-sfx.* copyparty-extras/sfx-full/
+mv copyparty-*.tar.gz copyparty-extras/
+
+
+# unpack the source code
+( cd copyparty-extras/
+tar -xf *.tar.gz
+)
+
+
+# use repacker from release if that is newer
+p_other=copyparty-extras/copyparty-*/scripts/copyparty-repack.sh
+other=$(awk -F= 'BEGIN{v=-1} NR<10&&/^repacker=/{v=$NF} END{print v}' <$p_other) 
+[ $repacker -lt $other ] &&
+  cat $p_other >"$od/$0" && cd "$od" && rm -rf "$td" && exec "$0" "$@"
+
+
+# now drop the cache
+rm -f "$cache"
+
+
+# fix permissions
+chmod 755 \
+  copyparty-extras/sfx-full/* \
+  copyparty-extras/copyparty-*/{scripts,bin}/*
+
+
+# extract the sfx
+( cd copyparty-extras/sfx-full/
+./copyparty-sfx.py -h
+)
+
+
+repack() {
+
+	# do the repack
+	(cd copyparty-extras/copyparty-*/
+	./scripts/make-sfx.sh $2
+	)
+
+	# put new sfx into copyparty-extras/$name/,
+	( cd copyparty-extras/
+	mv copyparty-*/dist/* $1/
+	)
+}
+
+repack sfx-full "re gz no-sh"
+repack sfx-lite "re no-ogv no-cm"
+repack sfx-lite "re no-ogv no-cm gz no-sh"
+
+
+# move fuse client into copyparty-extras/,
+# copy lite-sfx.py to ./copyparty,
+# delete extracted source code
+( cd copyparty-extras/
+mv copyparty-*/bin/copyparty-fuse.py .
+cp -pv sfx-lite/copyparty-sfx.py ../copyparty
+rm -rf copyparty-{0..9}*.*.*{0..9}
+)
+
+
+# and include the repacker itself too
+cp -av "$od/$0" copyparty-extras/ ||
+cp -av "$0" copyparty-extras/ ||
+true
+
+
+# create the bundle
+printf '\n\n'
+fn=copyparty-$(date +%Y-%m%d-%H%M%S).tgz
+tar -czvf "$od/$fn" *
+cd "$od"
+rm -rf "$td"
+
+
+echo
+echo "done, here's your bundle:"
+ls -al "$fn"

scripts/deps-docker/Dockerfile

@@ -1,12 +1,11 @@
-FROM    alpine:3.11
+FROM    alpine:3.13
 WORKDIR /z
-ENV     ver_asmcrypto=2821dd1dedd1196c378f5854037dda5c869313f3 \
-        ver_markdownit=10.0.0 \
-        ver_showdown=1.9.1 \
+ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
+        ver_hashwasm=4.7.0 \
         ver_marked=1.1.0 \
-        ver_ogvjs=1.6.1 \
-        ver_mde=2.10.1 \
-        ver_codemirror=5.53.2 \
+        ver_ogvjs=1.8.0 \
+        ver_mde=2.14.0 \
+        ver_codemirror=5.59.3 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 
@@ -17,13 +16,17 @@ RUN     mkdir -p /z/dist/no-pk \
         && wget https://fonts.gstatic.com/s/sourcecodepro/v11/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevW.woff2 -O scp.woff2 \
         && apk add cmake make g++ git bash npm patch wget tar pigz brotli gzip unzip python3 python3-dev brotli py3-brotli \
         && wget https://github.com/brion/ogv.js/releases/download/$ver_ogvjs/ogvjs-$ver_ogvjs.zip -O ogvjs.zip \
-        && wget https://github.com/asmcrypto/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
+        && wget https://github.com/openpgpjs/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
         && wget https://github.com/markedjs/marked/archive/v$ver_marked.tar.gz -O marked.tgz \
         && wget https://github.com/Ionaru/easy-markdown-editor/archive/$ver_mde.tar.gz -O mde.tgz \
         && wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
         && wget https://github.com/FortAwesome/Font-Awesome/releases/download/$ver_fontawesome/fontawesome-free-$ver_fontawesome-web.zip -O fontawesome.zip \
         && wget https://github.com/google/zopfli/archive/zopfli-$ver_zopfli.tar.gz -O zopfli.tgz \
+        && wget https://github.com/Daninet/hash-wasm/releases/download/v$ver_hashwasm/hash-wasm@$ver_hashwasm.zip -O hash-wasm.zip \
         && unzip ogvjs.zip \
+        && (mkdir hash-wasm \
+            && cd hash-wasm \
+            && unzip ../hash-wasm.zip) \
         && (tar -xf asmcrypto.tgz \
             && cd asmcrypto.js-$ver_asmcrypto \
             && npm install ) \
@@ -52,6 +55,7 @@ RUN     tar -xf zopfli.tgz \
             -S . \
         && make -C build \
         && make -C build install \
+        && python3 -m ensurepip \
         && python3 -m pip install fonttools zopfli
 
 
@@ -59,7 +63,12 @@ RUN     tar -xf zopfli.tgz \
 RUN     cd asmcrypto.js-$ver_asmcrypto \
         && echo "export { Sha512 } from './hash/sha512/sha512';" > src/entry-export_all.ts \
         && node -r esm build.js \
-        && mv asmcrypto.all.es5.js /z/dist/sha512.js
+        && awk '/HMAC state/{o=1}  /var HEAP/{o=0}  /function hmac_reset/{o=1}  /return \{/{o=0}  /var __extends =/{o=1}  /var Hash =/{o=0}  /hmac_|pbkdf2_/{next}  o{next}  {gsub(/IllegalStateError/,"Exception")}  {sub(/^ +/,"");sub(/^\/\/ .*/,"");sub(/;$/," ;")}  1' < asmcrypto.all.es5.js > /z/dist/sha512.ac.js
+
+
+# build hash-wasm
+RUN     cd hash-wasm \
+        && mv sha512.umd.min.js /z/dist/sha512.hw.js
 
 
 # build ogvjs

scripts/deps-docker/codemirror.patch

@@ -1,6 +1,6 @@
-diff -NarU2 CodeMirror-orig/mode/gfm/gfm.js CodeMirror-edit/mode/gfm/gfm.js
---- CodeMirror-orig/mode/gfm/gfm.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/mode/gfm/gfm.js	2020-05-02 02:13:32.142131800 +0200
+diff -NarU2 codemirror-5.59.3-orig/mode/gfm/gfm.js codemirror-5.59.3/mode/gfm/gfm.js
+--- codemirror-5.59.3-orig/mode/gfm/gfm.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/mode/gfm/gfm.js	2021-02-21 20:42:02.166174775 +0000
 @@ -97,5 +97,5 @@
          }
        }
@@ -15,9 +15,9 @@ diff -NarU2 CodeMirror-orig/mode/gfm/gfm.js CodeMirror-edit/mode/gfm/gfm.js
 +      }*/
        stream.next();
        return null;
-diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
---- CodeMirror-orig/mode/meta.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/mode/meta.js	2020-05-02 03:56:58.852408400 +0200
+diff -NarU2 codemirror-5.59.3-orig/mode/meta.js codemirror-5.59.3/mode/meta.js
+--- codemirror-5.59.3-orig/mode/meta.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/mode/meta.js	2021-02-21 20:42:54.798742821 +0000
 @@ -13,4 +13,5 @@
  
    CodeMirror.modeInfo = [
@@ -28,7 +28,7 @@ diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
      {name: "Gas", mime: "text/x-gas", mode: "gas", ext: ["s"]},
      {name: "Gherkin", mime: "text/x-feature", mode: "gherkin", ext: ["feature"]},
 +    */
-     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history).md$/i},
+     {name: "GitHub Flavored Markdown", mime: "text/x-gfm", mode: "gfm", file: /^(readme|contributing|history)\.md$/i},
 +    /*
      {name: "Go", mime: "text/x-go", mode: "go", ext: ["go"]},
      {name: "Groovy", mime: "text/x-groovy", mode: "groovy", ext: ["groovy", "gradle"], file: /^Jenkinsfile$/},
@@ -56,16 +56,16 @@ diff -NarU2 CodeMirror-orig/mode/meta.js CodeMirror-edit/mode/meta.js
 +    /*
      {name: "XQuery", mime: "application/xquery", mode: "xquery", ext: ["xy", "xquery"]},
      {name: "Yacas", mime: "text/x-yacas", mode: "yacas", ext: ["ys"]},
-@@ -171,4 +180,5 @@
-     {name: "xu", mime: "text/x-xu", mode: "mscgen", ext: ["xu"]},
-     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]}
+@@ -172,4 +181,5 @@
+     {name: "msgenny", mime: "text/x-msgenny", mode: "mscgen", ext: ["msgenny"]},
+     {name: "WebAssembly", mime: "text/webassembly", mode: "wast", ext: ["wat", "wast"]},
 +    */
    ];
    // Ensure all modes have a mime property for backwards compatibility
-diff -NarU2 CodeMirror-orig/src/display/selection.js CodeMirror-edit/src/display/selection.js
---- CodeMirror-orig/src/display/selection.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/display/selection.js	2020-05-02 03:27:30.144662800 +0200
-@@ -83,29 +83,21 @@
+diff -NarU2 codemirror-5.59.3-orig/src/display/selection.js codemirror-5.59.3/src/display/selection.js
+--- codemirror-5.59.3-orig/src/display/selection.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/display/selection.js	2021-02-21 20:44:14.860894328 +0000
+@@ -84,29 +84,21 @@
      let order = getOrder(lineObj, doc.direction)
      iterateBidiSections(order, fromArg || 0, toArg == null ? lineLen : toArg, (from, to, dir, i) => {
 -      let ltr = dir == "ltr"
@@ -105,24 +105,24 @@ diff -NarU2 CodeMirror-orig/src/display/selection.js CodeMirror-edit/src/display
 +          botRight = openEnd && last ? rightSide : toPos.right
          add(topLeft, fromPos.top, topRight - topLeft, fromPos.bottom)
          if (fromPos.bottom < toPos.top) add(leftSide, fromPos.bottom, null, toPos.top)
-diff -NarU2 CodeMirror-orig/src/input/ContentEditableInput.js CodeMirror-edit/src/input/ContentEditableInput.js
---- CodeMirror-orig/src/input/ContentEditableInput.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/input/ContentEditableInput.js	2020-05-02 03:33:05.707995500 +0200
-@@ -391,4 +391,5 @@
+diff -NarU2 codemirror-5.59.3-orig/src/input/ContentEditableInput.js codemirror-5.59.3/src/input/ContentEditableInput.js
+--- codemirror-5.59.3-orig/src/input/ContentEditableInput.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/input/ContentEditableInput.js	2021-02-21 20:44:33.273953867 +0000
+@@ -399,4 +399,5 @@
    let info = mapFromLineView(view, line, pos.line)
  
 +  /*
    let order = getOrder(line, cm.doc.direction), side = "left"
    if (order) {
-@@ -396,4 +397,5 @@
+@@ -404,4 +405,5 @@
      side = partPos % 2 ? "right" : "left"
    }
 +  */
    let result = nodeAndOffsetInLineMap(info.map, pos.ch, side)
    result.offset = result.collapse == "right" ? result.end : result.start
-diff -NarU2 CodeMirror-orig/src/input/movement.js CodeMirror-edit/src/input/movement.js
---- CodeMirror-orig/src/input/movement.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/input/movement.js	2020-05-02 03:31:19.710773500 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/input/movement.js codemirror-5.59.3/src/input/movement.js
+--- codemirror-5.59.3-orig/src/input/movement.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/input/movement.js	2021-02-21 20:45:12.763093671 +0000
 @@ -15,4 +15,5 @@
  
  export function endOfLine(visually, cm, lineObj, lineNo, dir) {
@@ -146,9 +146,9 @@ diff -NarU2 CodeMirror-orig/src/input/movement.js CodeMirror-edit/src/input/move
    return null
 +  */
  }
-diff -NarU2 CodeMirror-orig/src/line/line_data.js CodeMirror-edit/src/line/line_data.js
---- CodeMirror-orig/src/line/line_data.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/line/line_data.js	2020-05-02 03:17:02.785065000 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/line/line_data.js codemirror-5.59.3/src/line/line_data.js
+--- codemirror-5.59.3-orig/src/line/line_data.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/line/line_data.js	2021-02-21 20:45:36.472549599 +0000
 @@ -79,6 +79,6 @@
      // Optionally wire in some hacks into the token-rendering
      // algorithm, to deal with browser quirks.
@@ -158,9 +158,9 @@ diff -NarU2 CodeMirror-orig/src/line/line_data.js CodeMirror-edit/src/line/line_
 +    //  builder.addToken = buildTokenBadBidi(builder.addToken, order)
      builder.map = []
      let allowFrontierUpdate = lineView != cm.display.externalMeasured && lineNo(line)
-diff -NarU2 CodeMirror-orig/src/measurement/position_measurement.js CodeMirror-edit/src/measurement/position_measurement.js
---- CodeMirror-orig/src/measurement/position_measurement.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/measurement/position_measurement.js	2020-05-02 03:35:20.674159600 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/measurement/position_measurement.js codemirror-5.59.3/src/measurement/position_measurement.js
+--- codemirror-5.59.3-orig/src/measurement/position_measurement.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/measurement/position_measurement.js	2021-02-21 20:50:52.372945293 +0000
 @@ -380,5 +380,6 @@
      sticky = "after"
    }
@@ -199,9 +199,9 @@ diff -NarU2 CodeMirror-orig/src/measurement/position_measurement.js CodeMirror-e
 +*/
  
  let measureText
-diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
---- CodeMirror-orig/src/util/bidi.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/util/bidi.js	2020-05-02 03:12:44.418649800 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/util/bidi.js codemirror-5.59.3/src/util/bidi.js
+--- codemirror-5.59.3-orig/src/util/bidi.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/util/bidi.js	2021-02-21 20:52:18.168092225 +0000
 @@ -4,5 +4,5 @@
  
  export function iterateBidiSections(order, from, to, f) {
@@ -239,20 +239,19 @@ diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
 +  var fun = function(str, direction) {
      let outerType = direction == "ltr" ? "L" : "R"
  
-@@ -204,12 +210,16 @@
+@@ -204,5 +210,11 @@
      return direction == "rtl" ? order.reverse() : order
    }
--})()
- 
 +  return function(str, direction) {
 +    var ret = fun(str, direction);
 +    console.log("bidiOrdering inner ([%s], %s) => [%s]", str, direction, ret);
 +    return ret;
 +  }
-+})()
+ })()
 +*/
+ 
  // Get the bidi ordering for the given line (and cache it). Returns
- // false for lines that are fully left-to-right, and an array of
+@@ -210,6 +222,4 @@
  // BidiSpan objects otherwise.
  export function getOrder(line, direction) {
 -  let order = line.order
@@ -260,9 +259,9 @@ diff -NarU2 CodeMirror-orig/src/util/bidi.js CodeMirror-edit/src/util/bidi.js
 -  return order
 +  return false;
  }
-diff -NarU2 CodeMirror-orig/src/util/feature_detection.js CodeMirror-edit/src/util/feature_detection.js
---- CodeMirror-orig/src/util/feature_detection.js	2020-04-21 12:47:20.000000000 +0200
-+++ CodeMirror-edit/src/util/feature_detection.js	2020-05-02 03:16:21.085621400 +0200
+diff -NarU2 codemirror-5.59.3-orig/src/util/feature_detection.js codemirror-5.59.3/src/util/feature_detection.js
+--- codemirror-5.59.3-orig/src/util/feature_detection.js	2021-02-20 21:24:57.000000000 +0000
++++ codemirror-5.59.3/src/util/feature_detection.js	2021-02-21 20:49:22.191269270 +0000
 @@ -25,4 +25,5 @@
  }
  

scripts/deps-docker/easymde.patch

@@ -1,33 +1,57 @@
-diff -NarU2 easymde-orig/gulpfile.js easymde-mod1/gulpfile.js
---- easymde-orig/gulpfile.js	2020-04-06 14:09:36.000000000 +0200
-+++ easymde-mod1/gulpfile.js	2020-05-01 14:33:52.260175200 +0200
+diff -NarU2 easy-markdown-editor-2.14.0-orig/gulpfile.js easy-markdown-editor-2.14.0/gulpfile.js
+--- easy-markdown-editor-2.14.0-orig/gulpfile.js	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/gulpfile.js	2021-02-21 20:55:37.134701007 +0000
 @@ -25,5 +25,4 @@
      './node_modules/codemirror/lib/codemirror.css',
      './src/css/*.css',
 -    './node_modules/codemirror-spell-checker/src/css/spell-checker.css',
  ];
  
-diff -NarU2 easymde-orig/package.json easymde-mod1/package.json
---- easymde-orig/package.json	2020-04-06 14:09:36.000000000 +0200
-+++ easymde-mod1/package.json	2020-05-01 14:33:57.189975800 +0200
+diff -NarU2 easy-markdown-editor-2.14.0-orig/package.json easy-markdown-editor-2.14.0/package.json
+--- easy-markdown-editor-2.14.0-orig/package.json	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/package.json	2021-02-21 20:55:47.761190082 +0000
 @@ -21,5 +21,4 @@
      "dependencies": {
-         "codemirror": "^5.52.2",
+         "codemirror": "^5.59.2",
 -        "codemirror-spell-checker": "1.1.2",
-         "marked": "^0.8.2"
+         "marked": "^2.0.0"
      },
-diff -NarU2 easymde-orig/src/js/easymde.js easymde-mod1/src/js/easymde.js
---- easymde-orig/src/js/easymde.js	2020-04-06 14:09:36.000000000 +0200
-+++ easymde-mod1/src/js/easymde.js	2020-05-01 14:34:19.878774400 +0200
-@@ -11,5 +11,4 @@
+diff -NarU2 easy-markdown-editor-2.14.0-orig/src/js/easymde.js easy-markdown-editor-2.14.0/src/js/easymde.js
+--- easy-markdown-editor-2.14.0-orig/src/js/easymde.js	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/src/js/easymde.js	2021-02-21 20:57:09.143171536 +0000
+@@ -12,5 +12,4 @@
  require('codemirror/mode/gfm/gfm.js');
  require('codemirror/mode/xml/xml.js');
 -var CodeMirrorSpellChecker = require('codemirror-spell-checker');
  var marked = require('marked/lib/marked');
  
-@@ -1889,18 +1888,7 @@
+@@ -1762,9 +1761,4 @@
+         options.autosave.uniqueId = options.autosave.unique_id;
  
+-    // If overlay mode is specified and combine is not provided, default it to true
+-    if (options.overlayMode && options.overlayMode.combine === undefined) {
+-      options.overlayMode.combine = true;
+-    }
+-
+     // Update this options
+     this.options = options;
+@@ -2003,28 +1997,7 @@
      var mode, backdrop;
+ 
+-    // CodeMirror overlay mode
+-    if (options.overlayMode) {
+-      CodeMirror.defineMode('overlay-mode', function(config) {
+-        return CodeMirror.overlayMode(CodeMirror.getMode(config, options.spellChecker !== false ? 'spell-checker' : 'gfm'), options.overlayMode.mode, options.overlayMode.combine);
+-      });
+-
+-      mode = 'overlay-mode';
+-      backdrop = options.parsingConfig;
+-      backdrop.gitHubSpice = false;
+-    } else {
+         mode = options.parsingConfig;
+         mode.name = 'gfm';
+         mode.gitHubSpice = false;
+-    }
 -    if (options.spellChecker !== false) {
 -        mode = 'spell-checker';
 -        backdrop = options.parsingConfig;
@@ -37,16 +61,28 @@ diff -NarU2 easymde-orig/src/js/easymde.js easymde-mod1/src/js/easymde.js
 -        CodeMirrorSpellChecker({
 -            codeMirrorInstance: CodeMirror,
 -        });
--    } else {
-         mode = options.parsingConfig;
-         mode.name = 'gfm';
-         mode.gitHubSpice = false;
 -    }
  
      // eslint-disable-next-line no-unused-vars
-@@ -1927,5 +1915,4 @@
-         configureMouse: configureMouse,
-         inputStyle: (options.inputStyle != undefined) ? options.inputStyle : isMobile() ? 'contenteditable' : 'textarea',
--        spellcheck: (options.nativeSpellcheck != undefined) ? options.nativeSpellcheck : true,
-     });
+diff -NarU2 easy-markdown-editor-2.14.0-orig/types/easymde.d.ts easy-markdown-editor-2.14.0/types/easymde.d.ts
+--- easy-markdown-editor-2.14.0-orig/types/easymde.d.ts	2021-02-14 12:11:48.000000000 +0000
++++ easy-markdown-editor-2.14.0/types/easymde.d.ts	2021-02-21 20:57:42.492620979 +0000
+@@ -160,9 +160,4 @@
+     }
  
+-    interface OverlayModeOptions {
+-      mode: CodeMirror.Mode<any>
+-      combine?: boolean
+-    }
+-
+     interface Options {
+         autoDownloadFontAwesome?: boolean;
+@@ -214,7 +209,5 @@
+ 
+         promptTexts?: PromptTexts;
+-        syncSideBySidePreviewScroll?: boolean;
+-
+-        overlayMode?: OverlayModeOptions
++        syncSideBySidePreviewScroll?: boolean
+     }
+ }

scripts/fusefuzz.py

@@ -4,10 +4,10 @@ import os
 import time
 
 """
-mkdir -p /dev/shm/fusefuzz/{r,v}
-PYTHONPATH=.. python3 -m copyparty -v /dev/shm/fusefuzz/r::r -i 127.0.0.1
-../bin/copyparty-fuse.py /dev/shm/fusefuzz/v http://127.0.0.1:3923/ 2 0
-(d="$PWD"; cd /dev/shm/fusefuzz && "$d"/fusefuzz.py)
+td=/dev/shm/; [ -e $td ] || td=$HOME; mkdir -p $td/fusefuzz/{r,v}
+PYTHONPATH=.. python3 -m copyparty -v $td/fusefuzz/r::r -i 127.0.0.1
+../bin/copyparty-fuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
+(d="$PWD"; cd $td/fusefuzz && "$d"/fusefuzz.py)
 """
 
 

scripts/install-githooks.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+[ -e setup.py ] || ..
+[ -e setup.py ] || {
+    echo u wot
+    exit 1
+}
+
+cd .git/hooks
+rm -f pre-commit
+ln -s ../../scripts/run-tests.sh pre-commit

scripts/make-pypi-release.sh

@@ -3,12 +3,15 @@ set -e
 echo
 
 # osx support
-command -v gtar  >/dev/null &&
-command -v gfind >/dev/null && {
-	tar()  { gtar  "$@"; }
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
 }
 
 which md5sum 2>/dev/null >/dev/null &&
@@ -83,6 +86,8 @@ function have() {
 	python -c "import $1; $1; $1.__version__"
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 . buildenv/bin/activate
 have setuptools
 have wheel

scripts/make-sfx.sh

@@ -11,6 +11,10 @@ echo
 # `re` does a repack of an sfx which you already executed once
 #   (grabs files from the sfx-created tempdir), overrides `clean`
 #
+# `gz` creates a gzip-compressed python sfx instead of bzip2
+#
+# `no-sh` makes just the python sfx, skips the sh/unix sfx
+#
 # `no-ogv` saves ~500k by removing the opus/vorbis audio codecs
 #   (only affects apple devices; everything else has native support)
 #
@@ -18,13 +22,27 @@ echo
 #   (the fancy markdown editor)
 
 
-command -v gtar  >/dev/null &&
-command -v gfind >/dev/null && {
-	tar()  { gtar  "$@"; }
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
 	unexpand() { gunexpand "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
+
+	[ -e /opt/local/bin/bzip2 ] &&
+		bzip2() { /opt/local/bin/bzip2 "$@"; }
+}
+
+gawk=$(command -v gawk || command -v gnuawk || command -v awk)
+awk() { $gawk "$@"; }
+
+pybin=$(command -v python3 || command -v python) || {
+	echo need python
+	exit 1
 }
 
 [ -e copyparty/__main__.py ] || cd ..
@@ -35,11 +53,17 @@ command -v gfind >/dev/null && {
 	exit 1
 }
 
+use_gz=
+do_sh=1
+do_py=1
 while [ ! -z "$1" ]; do
 	[ "$1" = clean  ] && clean=1  && shift && continue
 	[ "$1" = re     ] && repack=1 && shift && continue
+	[ "$1" = gz     ] && use_gz=1 && shift && continue
 	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
 	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
+	[ "$1" = no-sh  ] && do_sh=   && shift && continue
+	[ "$1" = no-py  ] && do_py=   && shift && continue
 	break
 done
 
@@ -59,28 +83,32 @@ cd sfx
 	)/pe-copyparty"
 
 	echo "repack of files in $old"
-	cp -pR "$old/"*{jinja2,copyparty} .
-	mv {x.,}jinja2 2>/dev/null || true
+	cp -pR "$old/"*{dep-j2,copyparty} .
 }
 
 [ $repack ] || {
 	echo collecting jinja2
-	f="../build/Jinja2-2.6.tar.gz"
+	f="../build/Jinja2-2.11.3.tar.gz"
 	[ -e "$f" ] ||
-		(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz;
+		(url=https://files.pythonhosted.org/packages/4f/e7/65300e6b32e69768ded990494809106f87da1d436418d5f1367ed3966fd7/Jinja2-2.11.3.tar.gz;
 		wget -O$f "$url" || curl -L "$url" >$f)
 
 	tar -zxf $f
-	mv Jinja2-*/jinja2 .
-	rm -rf Jinja2-* jinja2/testsuite jinja2/_markupsafe/tests.py jinja2/_stringdefs.py
+	mv Jinja2-*/src/jinja2 .
+	rm -rf Jinja2-*
 	
-	f=jinja2/lexer.py
-	sed -r '/.*föö.*/    raise SyntaxError/' <$f >t
-	tmv $f
-	
-	f=jinja2/_markupsafe/_constants.py
-	awk '!/: [0-9]+,?$/ || /(amp|gt|lt|quot|apos|nbsp).:/' <$f >t
-	tmv $f
+	echo collecting markupsafe
+	f="../build/MarkupSafe-1.1.1.tar.gz"
+	[ -e "$f" ] ||
+		(url=https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz;
+		wget -O$f "$url" || curl -L "$url" >$f)
+
+	tar -zxf $f
+	mv MarkupSafe-*/src/markupsafe .
+	rm -rf MarkupSafe-* markupsafe/_speedups.c
+
+	mkdir dep-j2/
+	mv {markupsafe,jinja2} dep-j2/
 
 	# msys2 tar is bad, make the best of it
 	echo collecting source
@@ -94,8 +122,39 @@ cd sfx
 	rm -f ../tar
 }
 
-ver="$(awk '/^VERSION *= \(/ {
-	gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < ../copyparty/__version__.py)"
+ver=
+git describe --tags >/dev/null 2>/dev/null && {
+	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//')";
+	t_ver=
+
+	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
+		# short format (exact version number)
+		t_ver="$(printf '%s\n' "$ver" | sed -r 's/\./, /g')";
+	}
+
+	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+-[0-9]+-g[0-9a-f]+$' && {
+		# long format (unreleased commit)
+		t_ver="$(printf '%s\n' "$ver" | sed -r 's/\./, /g; s/(.*) (.*)/\1 "\2"/')"
+	}
+
+	[ -z "$t_ver" ] && {
+		printf 'unexpected git version format: [%s]\n' "$git_ver"
+		exit 1
+	}
+
+	dt="$(git log -1 --format=%cd --date=short | sed -E 's/-0?/, /g')"
+	printf 'git %3s: \033[36m%s\033[0m\n' ver "$ver" dt "$dt"
+	sed -ri '
+		s/^(VERSION =)(.*)/#\1\2\n\1 ('"$t_ver"')/;
+		s/^(S_VERSION =)(.*)/#\1\2\n\1 "'"$ver"'"/;
+		s/^(BUILD_DT =)(.*)/#\1\2\n\1 ('"$dt"')/;
+	' copyparty/__version__.py
+}
+
+[ -z "$ver" ] && 
+	ver="$(awk '/^VERSION *= \(/ {
+		gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < copyparty/__version__.py)"
 
 ts=$(date -u +%s)
 hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
@@ -112,7 +171,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/dbg-* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -131,41 +190,115 @@ done
 	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
 }
 
+[ $repack ] ||
+find | grep -E '\.py$' |
+  grep -vE '__version__' |
+  tr '\n' '\0' |
+  xargs -0 $pybin ../scripts/uncomment.py
+
+f=dep-j2/jinja2/constants.py
+awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
+tmv "$f"
+
 # up2k goes from 28k to 22k laff
 echo entabbening
-find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
+find | grep -E '\.css$' | while IFS= read -r f; do
+	awk '{
+		sub(/^[ \t]+/,"");
+		sub(/[ \t]+$/,"");
+		$0=gensub(/^([a-z-]+) *: *(.*[^ ]) *;$/,"\\1:\\2;","1");
+		sub(/ +\{$/,"{");
+		gsub(/, /,",")
+	}
+	!/\}$/ {printf "%s",$0;next}
+	1
+	' <$f | sed 's/;\}$/}/' >t
+	tmv "$f"
+done
+find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
 
+
+gzres() {
+command -v pigz &&
+	pk='pigz -11 -J 34 -I 100' ||
+	pk='gzip'
+
+echo "$pk"
+find | grep -E '\.(js|css)$' | grep -vF /deps/ | while IFS= read -r f; do
+	echo -n .
+	$pk "$f"
+done
+echo
+}
+gzres
+
+
+echo gen tarlist
+for d in copyparty dep-j2; do find $d -type f; done |
+sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
+sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
+
+(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1) >list || true
+
 echo creating tar
 args=(--owner=1000 --group=1000)
 [ "$OSTYPE" = msys ] &&
 	args=()
 
-tar -cf tar "${args[@]}" --numeric-owner copyparty jinja2
+tar -cf tar "${args[@]}" --numeric-owner -T list
+
+pc=bzip2
+pe=bz2
+[ $use_gz ] && pc=gzip && pe=gz
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2
-for n in {2..9}; do cp tar t.$n;  xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz
-rm t.*
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2; }
+[ $do_sh ] && { for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
+rm t.* || true
+exts=()
 
+
+[ $do_sh ] && {
+exts+=(.sh)
 echo creating unix sfx
 (
 	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
 	grep -E '^sfx_eof$' -B 9001;
 	cat tar.xz
 ) >$sfx_out.sh
+}
 
-echo creating generic sfx
-python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
-mv sfx.out $sfx_out.py
-chmod 755 $sfx_out.*
+
+[ $do_py ] && {
+	echo creating generic sfx
+
+	py=../scripts/sfx.py
+	suf=
+	[ $use_gz ] && {
+		sed -r 's/"r:bz2"/"r:gz"/' <$py >$py.t
+		py=$py.t
+		suf=-gz
+	}
+
+	$pybin $py --sfx-make tar.bz2 $ver $ts
+	mv sfx.out $sfx_out$suf.py
+	
+	exts+=($suf.py)
+	[ $use_gz ] &&
+		rm $py
+}
+
+
+chmod 755 $sfx_out*
 
 printf "done:\n"
-printf "  %s\n" "$(realpath $sfx_out)."{sh,py}
-# rm -rf *
+for ext in ${exts[@]}; do
+	printf "  %s\n" "$(realpath $sfx_out)"$ext
+done
 
-# tar -tvf ../sfx/tar | sed -r 's/(.* ....-..-.. ..:.. )(.*)/\2 `` \1/' | sort | sed -r 's/(.*) `` (.*)/\2 \1/'| less
-# for n in {1..9}; do tar -tf tar | grep -vE '/$' | sed -r 's/(.*)\.(.*)/\2.\1/' | sort | sed -r 's/([^\.]+)\.(.*)/\2.\1/' | tar -cT- | bzip2 -c$n | wc -c; done 
+# apk add bash python3 tar xz bzip2
+# while true; do ./make-sfx.sh; for f in ..//dist/copyparty-sfx.{sh,py}; do mv $f $f.$(wc -c <$f | awk '{print$1}'); done; done

scripts/make-tgz-release.sh

@@ -2,12 +2,16 @@
 set -e
 echo
 
-command -v gtar  >/dev/null &&
-command -v gfind >/dev/null && {
-	tar()  { gtar  "$@"; }
+# osx support
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
 }
 
 which md5sum 2>/dev/null >/dev/null &&
@@ -16,27 +20,29 @@ which md5sum 2>/dev/null >/dev/null &&
 
 ver="$1"
 
-[[ "x$ver" == x ]] &&
+[ "x$ver" = x ] &&
 {
 	echo "need argument 1:  version"
 	echo
 	exit 1
 }
 
-[[ -e copyparty/__main__.py ]] || cd ..
-[[ -e copyparty/__main__.py ]] ||
+[ -e copyparty/__main__.py ] || cd ..
+[ -e copyparty/__main__.py ] ||
 {
 	echo "run me from within the project root folder"
 	echo
 	exit 1
 }
 
+mv copyparty/web/deps/marked.full.js.gz srv/ || true
+
 mkdir -p dist
 zip_path="$(pwd)/dist/copyparty-$ver.zip"
 tgz_path="$(pwd)/dist/copyparty-$ver.tar.gz"
 
-[[ -e "$zip_path" ]] ||
-[[ -e "$tgz_path" ]] &&
+[ -e "$zip_path" ] ||
+[ -e "$tgz_path" ] &&
 {
 	echo "found existing archives for this version"
 	echo "  $zip_path"

scripts/profile.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import sys
+
+sys.path.insert(0, ".")
+cmd = sys.argv[1]
+
+if cmd == "cpp":
+    from copyparty.__main__ import main
+
+    argv = ["__main__", "-v", "srv::r", "-v", "../../yt:yt:r"]
+    main(argv=argv)
+
+elif cmd == "test":
+    from unittest import main
+
+    argv = ["__main__", "discover", "-s", "tests"]
+    main(module=None, argv=argv)
+
+else:
+    raise Exception()
+
+# import dis; print(dis.dis(main))
+
+
+# macos:
+#   option1) python3.9 -m pip install --user -U vmprof==0.4.9
+#   option2) python3.9 -m pip install --user -U https://github.com/vmprof/vmprof-python/archive/refs/heads/master.zip
+#
+# python -m vmprof -o prof --lines ./scripts/profile.py test
+
+# linux: ~/.local/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+#   win: %appdata%\..\Roaming\Python\Python39\Scripts\vmprofshow.exe prof tree

scripts/run-tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+pids=()
+for py in python{2,3}; do
+    nice $py -m unittest discover -s tests >/dev/null &
+    pids+=($!)
+done
+
+python3 scripts/test/smoketest.py &
+pids+=($!)
+
+for pid in ${pids[@]}; do
+    wait $pid
+done

scripts/sfx.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
-# coding: utf-8
+# coding: latin-1
 from __future__ import print_function, unicode_literals
 
-import re, os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
+import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform, tempfile, traceback
 import subprocess as sp
 
 """
@@ -27,21 +27,21 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
+WINDOWS = sys.platform in ["win32", "msys"]
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
-cpp = None
 
 
-def eprint(*args, **kwargs):
-    kwargs["file"] = sys.stderr
-    print(*args, **kwargs)
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    print(*a, **ka)
 
 
-def msg(*args, **kwargs):
-    if args:
-        args = ["[SFX]", args[0]] + list(args[1:])
+def msg(*a, **ka):
+    if a:
+        a = ["[SFX]", a[0]] + list(a[1:])
 
-    eprint(*args, **kwargs)
+    eprint(*a, **ka)
 
 
 # skip 1
@@ -156,6 +156,9 @@ def encode(data, size, cksum, ver, ts):
                 skip = True
                 continue
 
+            if ln.strip().startswith("# fmt: "):
+                continue
+
             unpk += ln + "\n"
 
         for k, v in [
@@ -202,93 +205,6 @@ def u8(gen):
             yield s
 
 
-def get_py_win(ret):
-    tops = []
-    p = str(os.getenv("LocalAppdata"))
-    if p:
-        tops.append(os.path.join(p, "Programs", "Python"))
-
-    progfiles = {}
-    for p in ["ProgramFiles", "ProgramFiles(x86)"]:
-        p = str(os.getenv(p))
-        if p:
-            progfiles[p] = 1
-            # 32bit apps get x86 for both
-            if p.endswith(" (x86)"):
-                progfiles[p[:-6]] = 1
-
-    tops += list(progfiles.keys())
-
-    for sysroot in [me, sys.executable]:
-        sysroot = sysroot[:3].upper()
-        if sysroot[1] == ":" and sysroot not in tops:
-            tops.append(sysroot)
-
-    # $WIRESHARK_SLOGAN
-    for top in tops:
-        try:
-            for name1 in u8(sorted(os.listdir(top), reverse=True)):
-                if name1.lower().startswith("python"):
-                    path1 = os.path.join(top, name1)
-                    try:
-                        for name2 in u8(os.listdir(path1)):
-                            if name2.lower() == "python.exe":
-                                path2 = os.path.join(path1, name2)
-                                ret[path2.lower()] = path2
-                    except:
-                        pass
-        except:
-            pass
-
-
-def get_py_nix(ret):
-    ptn = re.compile(r"^(python|pypy)[0-9\.-]*$")
-    for bindir in os.getenv("PATH").split(":"):
-        if not bindir:
-            next
-
-        try:
-            for fn in u8(os.listdir(bindir)):
-                if ptn.match(fn):
-                    fn = os.path.join(bindir, fn)
-                    ret[fn.lower()] = fn
-        except:
-            pass
-
-
-def read_py(binp):
-    cmd = [
-        binp,
-        "-c",
-        "import sys; sys.stdout.write(' '.join(str(x) for x in sys.version_info)); import jinja2",
-    ]
-    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
-    ver, _ = p.communicate()
-    ver = ver.decode("utf-8").split(" ")[:3]
-    ver = [int(x) if x.isdigit() else 0 for x in ver]
-    return ver, p.returncode == 0
-
-
-def get_pys():
-    ver, chk = read_py(sys.executable)
-    if chk or PY2:
-        return [[chk, ver, sys.executable]]
-
-    hits = {sys.executable.lower(): sys.executable}
-    if platform.system() == "Windows":
-        get_py_win(hits)
-    else:
-        get_py_nix(hits)
-
-    ret = []
-    for binp in hits.values():
-        ver, chk = read_py(binp)
-        ret.append([chk, ver, binp])
-        msg("\t".join(str(x) for x in ret[-1]))
-
-    return ret
-
-
 def yieldfile(fn):
     with open(fn, "rb") as f:
         for block in iter(lambda: f.read(64 * 1024), b""):
@@ -296,11 +212,11 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    hasher = hashlib.md5()
+    h = hashlib.md5()
     for block in yieldfile(fn):
-        hasher.update(block)
+        h.update(block)
 
-    return hasher.hexdigest()
+    return h.hexdigest()
 
 
 def unpack():
@@ -309,9 +225,10 @@ def unpack():
     tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
-    final = os.path.join(top, name)
-    mine = os.path.join(top, withpid)
-    tar = os.path.join(mine, "tar")
+    opj = os.path.join
+    final = opj(top, name)
+    mine = opj(top, withpid)
+    tar = opj(mine, "tar")
 
     try:
         if tag in os.listdir(final):
@@ -320,28 +237,24 @@ def unpack():
     except:
         pass
 
-    nwrite = 0
+    sz = 0
     os.mkdir(mine)
     with open(tar, "wb") as f:
         for buf in get_payload():
-            nwrite += len(buf)
+            sz += len(buf)
             f.write(buf)
 
-    if nwrite != SIZE:
-        t = "\n\n  bad file:\n    expected {} bytes, got {}\n".format(SIZE, nwrite)
-        raise Exception(t)
-
-    cksum = hashfile(tar)
-    if cksum != CKSUM:
-        t = "\n\n  bad file:\n    {} expected,\n    {} obtained\n".format(CKSUM, cksum)
-        raise Exception(t)
+    ck = hashfile(tar)
+    if ck != CKSUM:
+        t = "\n\nexpected {} ({} byte)\nobtained {} ({} byte)\nsfx corrupt"
+        raise Exception(t.format(CKSUM, SIZE, ck, sz))
 
     with tarfile.open(tar, "r:bz2") as tf:
         tf.extractall(mine)
 
     os.remove(tar)
 
-    with open(os.path.join(mine, tag), "wb") as f:
+    with open(opj(mine, tag), "wb") as f:
         f.write(b"h\n")
 
     try:
@@ -359,25 +272,25 @@ def unpack():
     except:
         pass
 
+    for fn in u8(os.listdir(top)):
+        if fn.startswith(name) and fn != withpid:
+            try:
+                old = opj(top, fn)
+                if time.time() - os.path.getmtime(old) > 86400:
+                    shutil.rmtree(old)
+            except:
+                pass
+
     try:
         os.symlink(mine, final)
     except:
         try:
             os.rename(mine, final)
+            return final
         except:
             msg("reloc fail,", mine)
-            return mine
 
-    for fn in u8(os.listdir(top)):
-        if fn.startswith(name) and fn not in [name, withpid]:
-            try:
-                old = os.path.join(top, fn)
-                if time.time() - os.path.getmtime(old) > 10:
-                    shutil.rmtree(old)
-            except:
-                pass
-
-    return final
+    return mine
 
 
 def get_payload():
@@ -394,107 +307,124 @@ def get_payload():
         if ofs < 0:
             raise Exception("could not find archive marker")
 
-        # start reading from the final b"\n"
+        # start at final b"\n"
         fpos = ofs + len(ptn) - 3
-        # msg("tar found at", fpos)
         f.seek(fpos)
         dpos = 0
-        leftovers = b""
+        rem = b""
         while True:
             rbuf = f.read(1024 * 32)
             if rbuf:
-                buf = leftovers + rbuf
+                buf = rem + rbuf
                 ofs = buf.rfind(b"\n")
                 if len(buf) <= 4:
-                    leftovers = buf
+                    rem = buf
                     continue
 
                 if ofs >= len(buf) - 4:
-                    leftovers = buf[ofs:]
+                    rem = buf[ofs:]
                     buf = buf[:ofs]
                 else:
-                    leftovers = b"\n# "
+                    rem = b"\n# "
             else:
-                buf = leftovers
+                buf = rem
 
             fpos += len(buf) + 1
-            buf = (
-                buf.replace(b"\n# ", b"")
-                .replace(b"\n#r", b"\r")
-                .replace(b"\n#n", b"\n")
-            )
-            dpos += len(buf) - 1
+            for a, b in [[b"\n# ", b""], [b"\n#r", b"\r"], [b"\n#n", b"\n"]]:
+                buf = buf.replace(a, b)
 
+            dpos += len(buf) - 1
             yield buf
 
             if not rbuf:
                 break
 
 
-def confirm():
+def utime(top):
+    i = 0
+    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    while WINDOWS:
+        t = int(time.time())
+        if i:
+            msg("utime {}, {}".format(i, t))
+
+        for f in files:
+            os.utime(f, (t, t))
+
+        i += 1
+        time.sleep(78123)
+
+
+def confirm(rv):
     msg()
+    msg("retcode", rv if rv else traceback.format_exc())
     msg("*** hit enter to exit ***")
     try:
         raw_input() if PY2 else input()
     except:
         pass
 
+    sys.exit(rv)
 
-def run(tmp, py):
-    global cpp
 
-    msg("OK")
-    msg("will use:", py)
-    msg("bound to:", tmp)
+def run(tmp, j2):
+    msg("jinja2:", j2 or "bundled")
+    msg("sfxdir:", tmp)
+    msg()
 
-    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    # block systemd-tmpfiles-clean.timer
     try:
         import fcntl
 
         fd = os.open(tmp, os.O_RDONLY)
         fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
-    except:
-        pass
+    except Exception as ex:
+        if not WINDOWS:
+            msg("\033[31mflock:", repr(ex))
 
-    fp_py = os.path.join(tmp, "py")
-    try:
-        with open(fp_py, "wb") as f:
-            f.write(py.encode("utf-8") + b"\n")
-    except:
-        pass
+    t = threading.Thread(target=utime, args=(tmp,))
+    t.daemon = True
+    t.start()
 
-    # avoid loading ./copyparty.py
-    cmd = [
-        py,
-        "-c",
-        'import sys, runpy; sys.path.insert(0, r"'
-        + tmp
-        + '"); runpy.run_module("copyparty", run_name="__main__")',
-    ] + list(sys.argv[1:])
+    ld = [tmp, os.path.join(tmp, "dep-j2")]
+    if j2:
+        del ld[-1]
 
-    msg("\n", cmd, "\n")
-    cpp = sp.Popen(str(x) for x in cmd)
-    try:
-        cpp.wait()
-    except:
-        cpp.wait()
-
-    if cpp.returncode != 0:
-        confirm()
-
-    sys.exit(cpp.returncode)
+    if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
+        run_s(ld)
+    else:
+        run_i(ld)
 
 
-def bye(sig, frame):
-    if cpp is not None:
-        cpp.terminate()
+def run_i(ld):
+    for x in ld:
+        sys.path.insert(0, x)
+
+    from copyparty.__main__ import main as p
+
+    p()
+
+
+def run_s(ld):
+    # fmt: off
+    c = "import sys,runpy;" + "".join(['sys.path.insert(0,r"' + x + '");' for x in ld]) + 'runpy.run_module("copyparty",run_name="__main__")'
+    c = [str(x) for x in [sys.executable, "-c", c] + list(sys.argv[1:])]
+    # fmt: on
+    msg("\n", c, "\n")
+    p = sp.Popen(c)
+
+    def bye(*a):
+        p.send_signal(signal.SIGINT)
+
+    signal.signal(signal.SIGTERM, bye)
+    p.wait()
+
+    raise SystemExit(p.returncode)
 
 
 def main():
     sysver = str(sys.version).replace("\n", "\n" + " " * 18)
     pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
-    os.system("")
     msg()
     msg("   this is: copyparty", VER)
     msg(" packed at:", pktime, "UTC,", STAMP)
@@ -523,36 +453,23 @@ def main():
 
     # skip 0
 
-    signal.signal(signal.SIGTERM, bye)
+    tmp = os.path.realpath(unpack())
 
-    tmp = unpack()
-    fp_py = os.path.join(tmp, "py")
-    if os.path.exists(fp_py):
-        with open(fp_py, "rb") as f:
-            py = f.read().decode("utf-8").rstrip()
+    try:
+        from jinja2 import __version__ as j2
+    except:
+        j2 = None
 
-        return run(tmp, py)
-
-    pys = get_pys()
-    pys.sort(reverse=True)
-    j2, ver, py = pys[0]
-    if j2:
-        try:
-            os.rename(os.path.join(tmp, "jinja2"), os.path.join(tmp, "x.jinja2"))
-        except:
-            pass
-
-        return run(tmp, py)
-
-    msg("\n  could not find jinja2; will use py2 + the bundled version\n")
-    for _, ver, py in pys:
-        if ver > [2, 7] and ver < [3, 0]:
-            return run(tmp, py)
-
-    m = "\033[1;31m\n\n\ncould not find a python with jinja2 installed; please do one of these:\n\n  pip install --user jinja2\n\n  install python2\n\n\033[0m"
-    msg(m)
-    confirm()
-    sys.exit(1)
+    try:
+        run(tmp, j2)
+    except SystemExit as ex:
+        c = ex.code
+        if c not in [0, -15]:
+            confirm(ex.code)
+    except KeyboardInterrupt:
+        pass
+    except:
+        confirm(0)
 
 
 if __name__ == "__main__":

scripts/sfx.sh

@@ -47,7 +47,7 @@ grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
 	printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
 	$_py -c 'import jinja2' 2>/dev/null || continue
 	printf '%s\n' "$_py"
-	mv $dir/{,x.}jinja2
+	mv $dir/{,x.}dep-j2
 	break
 done)"
 

scripts/speedtest-fs.py

@@ -17,14 +17,15 @@ __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
 
-def get_spd(nbyte, nsec):
+def get_spd(nbyte, nfiles, nsec):
     if not nsec:
-        return "0.000 MB   0.000 sec   0.000 MB/s"
+        return "0.000 MB   0 files   0.000 sec   0.000 MB/s   0.000 f/s"
 
     mb = nbyte / (1024 * 1024.0)
     spd = mb / nsec
+    nspd = nfiles / nsec
 
-    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+    return f"{mb:.3f} MB   {nfiles} files   {nsec:.3f} sec   {spd:.3f} MB/s   {nspd:.3f} f/s"
 
 
 class Inf(object):
@@ -36,6 +37,7 @@ class Inf(object):
         self.mtx_reports = threading.Lock()
 
         self.n_byte = 0
+        self.n_file = 0
         self.n_sec = 0
         self.n_done = 0
         self.t0 = t0
@@ -63,7 +65,8 @@ class Inf(object):
                 continue
 
             msgs = msgs[-64:]
-            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            spd = get_spd(self.n_byte, len(self.reports), self.n_sec)
+            msgs = [f"{spd}   {x}" for x in msgs]
             print("\n".join(msgs))
 
     def report(self, fn, n_byte, n_sec):
@@ -131,8 +134,9 @@ def main():
 
     num_threads = 8
     read_sz = 32 * 1024
+    targs = (q, inf, read_sz)
     for _ in range(num_threads):
-        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr = threading.Thread(target=worker, args=targs)
         thr.daemon = True
         thr.start()
 
@@ -151,14 +155,14 @@ def main():
     log = inf.reports
     log.sort()
     for nbyte, nsec, fn in log[-64:]:
-        print(f"{get_spd(nbyte, nsec)}   {fn}")
+        spd = get_spd(nbyte, len(log), nsec)
+        print(f"{spd}   {fn}")
 
     print()
     print("\n".join(inf.errors))
 
-    print(get_spd(inf.n_byte, t2 - t0))
+    print(get_spd(inf.n_byte, len(log), t2 - t0))
 
 
 if __name__ == "__main__":
     main()
-

scripts/test/race.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import json
+import threading
+import http.client
+
+
+class Conn(object):
+    def __init__(self, ip, port):
+        self.s = http.client.HTTPConnection(ip, port, timeout=260)
+        self.st = []
+
+    def get(self, vpath):
+        self.st = [time.time()]
+
+        self.s.request("GET", vpath)
+        self.st.append(time.time())
+
+        ret = self.s.getresponse()
+        self.st.append(time.time())
+
+        if ret.status < 200 or ret.status >= 400:
+            raise Exception(ret.status)
+
+        ret = ret.read()
+        self.st.append(time.time())
+
+        return ret
+
+    def get_json(self, vpath):
+        ret = self.get(vpath)
+        return json.loads(ret)
+
+
+class CState(threading.Thread):
+    def __init__(self, cs):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.cs = cs
+        self.start()
+
+    def run(self):
+        colors = [5, 1, 3, 2, 7]
+        remotes = []
+        remotes_ok = False
+        while True:
+            time.sleep(0.001)
+            if not remotes_ok:
+                remotes = []
+                remotes_ok = True
+                for conn in self.cs:
+                    try:
+                        remotes.append(conn.s.sock.getsockname()[1])
+                    except:
+                        remotes.append("?")
+                        remotes_ok = False
+
+            m = []
+            for conn, remote in zip(self.cs, remotes):
+                stage = len(conn.st)
+                m.append(f"\033[3{colors[stage]}m{remote}")
+
+            m = " ".join(m)
+            print(f"{m}\033[0m\n\033[A", end="")
+
+
+def allget(cs, urls):
+    thrs = []
+    for c, url in zip(cs, urls):
+        t = threading.Thread(target=c.get, args=(url,))
+        t.start()
+        thrs.append(t)
+
+    for t in thrs:
+        t.join()
+
+
+def main():
+    os.system("")
+
+    ip, port = sys.argv[1].split(":")
+    port = int(port)
+
+    cs = []
+    for _ in range(64):
+        cs.append(Conn(ip, 3923))
+
+    CState(cs)
+
+    urlbase = "/doujin/c95"
+    j = cs[0].get_json(f"{urlbase}?ls")
+    urls = []
+    for d in j["dirs"]:
+        urls.append(f"{urlbase}/{d['href']}?th=w")
+
+    for n in range(100):
+        print(n)
+        allget(cs, urls)
+
+
+if __name__ == "__main__":
+    main()

scripts/test/smoketest.py

@@ -0,0 +1,209 @@
+import os
+import sys
+import time
+import shlex
+import shutil
+import signal
+import tempfile
+import requests
+import threading
+import subprocess as sp
+
+
+CPP = []
+
+
+class Cpp(object):
+    def __init__(self, args):
+        args = [sys.executable, "-m", "copyparty"] + args
+        print(" ".join([shlex.quote(x) for x in args]))
+
+        self.ls_pre = set(list(os.listdir()))
+        self.p = sp.Popen(args)
+        # , stdout=sp.PIPE, stderr=sp.PIPE)
+
+        self.t = threading.Thread(target=self._run)
+        self.t.daemon = True
+        self.t.start()
+
+    def _run(self):
+        self.so, self.se = self.p.communicate()
+
+    def stop(self, wait):
+        if wait:
+            os.kill(self.p.pid, signal.SIGINT)
+            self.t.join(timeout=2)
+        else:
+            self.p.kill()  # macos py3.8
+
+    def clean(self):
+        t = os.listdir()
+        for f in t:
+            if f not in self.ls_pre and f.startswith("up."):
+                os.unlink(f)
+
+    def await_idle(self, ub, timeout):
+        req = ["scanning</td><td>False", "hash-q</td><td>0", "tag-q</td><td>0"]
+        lim = int(timeout * 10)
+        u = ub + "?h"
+        for n in range(lim):
+            try:
+                time.sleep(0.1)
+                r = requests.get(u, timeout=0.1)
+                for x in req:
+                    if x not in r.text:
+                        print("ST: {}/{} miss {}".format(n, lim, x))
+                        raise Exception()
+                print("ST: idle")
+                return
+            except:
+                pass
+
+
+def tc1():
+    ub = "http://127.0.0.1:4321/"
+    td = os.path.join("srv", "smoketest")
+    try:
+        shutil.rmtree(td)
+    except:
+        if os.path.exists(td):
+            raise
+
+    for _ in range(10):
+        try:
+            os.mkdir(td)
+        except:
+            time.sleep(0.1)  # win10
+
+    assert os.path.exists(td)
+
+    vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
+    if not os.path.exists(vidp):
+        cmd = "ffmpeg -f lavfi -i testsrc=48x32:3 -t 1 -c:v libx264 -tune animation -preset veryslow -crf 69"
+        sp.check_call(cmd.split(" ") + [vidp])
+
+    with open(vidp, "rb") as f:
+        ovid = f.read()
+
+    args = [
+        "-p4321",
+        "-e2dsa",
+        "-e2tsr",
+        "--no-mutagen",
+        "--th-ff-jpg",
+        "--hist",
+        os.path.join(td, "dbm"),
+    ]
+    pdirs = []
+    hpaths = {}
+
+    for d1 in ["r", "w", "a"]:
+        pdirs.append("{}/{}".format(td, d1))
+        pdirs.append("{}/{}/j".format(td, d1))
+        for d2 in ["r", "w", "a"]:
+            d = os.path.join(td, d1, "j", d2)
+            pdirs.append(d)
+            os.makedirs(d)
+
+    pdirs = [x.replace("\\", "/") for x in pdirs]
+    udirs = [x.split("/", 2)[2] for x in pdirs]
+    perms = [x.rstrip("j/")[-1] for x in pdirs]
+    for pd, ud, p in zip(pdirs, udirs, perms):
+        if ud[-1] == "j":
+            continue
+
+        hp = None
+        if pd.endswith("st/a"):
+            hp = hpaths[ud] = os.path.join(td, "db1")
+        elif pd[:-1].endswith("a/j/"):
+            hpaths[ud] = os.path.join(td, "dbm")
+            hp = None
+        else:
+            hp = "-"
+            hpaths[ud] = os.path.join(pd, ".hist")
+
+        arg = "{}:{}:{}".format(pd, ud, p, hp)
+        if hp:
+            arg += ":chist=" + hp
+
+        args += ["-v", arg]
+
+    # return
+    cpp = Cpp(args)
+    CPP.append(cpp)
+    cpp.await_idle(ub, 3)
+
+    for d in udirs:
+        vid = ovid + "\n{}".format(d).encode("utf-8")
+        try:
+            requests.post(ub + d, data={"act": "bput"}, files={"f": ("a.h264", vid)})
+        except:
+            pass
+
+    cpp.clean()
+
+    # GET permission
+    for d, p in zip(udirs, perms):
+        u = "{}{}/a.h264".format(ub, d)
+        r = requests.get(u)
+        ok = bool(r)
+        if ok != (p in ["a"]):
+            raise Exception("get {} with perm {} at {}".format(ok, p, u))
+
+    # stat filesystem
+    for d, p in zip(pdirs, perms):
+        u = "{}/a.h264".format(d)
+        ok = os.path.exists(u)
+        if ok != (p in ["a", "w"]):
+            raise Exception("stat {} with perm {} at {}".format(ok, p, u))
+
+    # GET thumbnail, vreify contents
+    for d, p in zip(udirs, perms):
+        u = "{}{}/a.h264?th=j".format(ub, d)
+        r = requests.get(u)
+        ok = bool(r and r.content[:3] == b"\xff\xd8\xff")
+        if ok != (p in ["a"]):
+            raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
+
+    # check tags
+    cpp.await_idle(ub, 5)
+    for d, p in zip(udirs, perms):
+        u = "{}{}?ls".format(ub, d)
+        r = requests.get(u)
+        j = r.json() if r else False
+        tag = None
+        if j:
+            for f in j["files"]:
+                tag = tag or f["tags"].get("res")
+
+        r_ok = bool(j)
+        w_ok = bool(r_ok and j.get("files"))
+
+        if not r_ok or w_ok != (p in ["a"]):
+            raise Exception("ls {} with perm {} at {}".format(ok, p, u))
+
+        if (tag and p != "a") or (not tag and p == "a"):
+            raise Exception("tag {} with perm {} at {}".format(tag, p, u))
+
+        if tag is not None and tag != "48x32":
+            raise Exception("tag [{}] at {}".format(tag, u))
+
+    cpp.stop(True)
+
+
+def run(tc):
+    try:
+        tc()
+    finally:
+        try:
+            CPP[0].stop(False)
+        except:
+            pass
+
+
+def main():
+    run(tc1)
+
+
+if __name__ == "__main__":
+    main()

scripts/uncomment.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import io
+import sys
+import tokenize
+
+
+def uncomment(fpath):
+    """ modified https://stackoverflow.com/a/62074206 """
+
+    with open(fpath, "rb") as f:
+        orig = f.read().decode("utf-8")
+
+    out = ""
+    for ln in orig.split("\n"):
+        if not ln.startswith("#"):
+            break
+
+        out += ln + "\n"
+
+    io_obj = io.StringIO(orig)
+    prev_toktype = tokenize.INDENT
+    last_lineno = -1
+    last_col = 0
+    for tok in tokenize.generate_tokens(io_obj.readline):
+        # print(repr(tok))
+        token_type = tok[0]
+        token_string = tok[1]
+        start_line, start_col = tok[2]
+        end_line, end_col = tok[3]
+
+        if start_line > last_lineno:
+            last_col = 0
+
+        if start_col > last_col:
+            out += " " * (start_col - last_col)
+
+        is_legalese = (
+            "copyright" in token_string.lower() or "license" in token_string.lower()
+        )
+
+        if token_type == tokenize.STRING:
+            if (
+                prev_toktype != tokenize.INDENT
+                and prev_toktype != tokenize.NEWLINE
+                and start_col > 0
+                or is_legalese
+            ):
+                out += token_string
+            else:
+                out += '"a"'
+        elif token_type != tokenize.COMMENT or is_legalese:
+            out += token_string
+
+        prev_toktype = token_type
+        last_lineno = end_line
+        last_col = end_col
+
+    # out = "\n".join(x for x in out.splitlines() if x.strip())
+
+    with open(fpath, "wb") as f:
+        f.write(out.encode("utf-8"))
+
+
+def main():
+    print("uncommenting", end="")
+    for f in sys.argv[1:]:
+        print(".", end="")
+        uncomment(f)
+
+    print("k")
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -2,27 +2,10 @@
 # coding: utf-8
 from __future__ import print_function
 
-import io
 import os
 import sys
-from glob import glob
 from shutil import rmtree
-
-setuptools_available = True
-try:
-    # need setuptools to build wheel
-    from setuptools import setup, Command, find_packages
-
-except ImportError:
-    # works in a pinch
-    setuptools_available = False
-    from distutils.core import setup, Command
-
-from distutils.spawn import spawn
-
-if "bdist_wheel" in sys.argv and not setuptools_available:
-    print("cannot build wheel without setuptools")
-    sys.exit(1)
+from setuptools import setup, Command, find_packages
 
 
 NAME = "copyparty"
@@ -49,7 +32,7 @@ with open(here + "/README.md", "rb") as f:
 about = {}
 if not VERSION:
     with open(os.path.join(here, NAME, "__version__.py"), "rb") as f:
-        exec(f.read().decode("utf-8").split("\n\n", 1)[1], about)
+        exec (f.read().decode("utf-8").split("\n\n", 1)[1], about)
 else:
     about["__version__"] = VERSION
 
@@ -102,55 +85,35 @@ args = {
     "author_email": "copyparty@ocv.me",
     "url": "https://github.com/9001/copyparty",
     "license": "MIT",
-    "data_files": data_files,
     "classifiers": [
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 4 - Beta",
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python",
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Environment :: Console",
         "Environment :: No Input/Output (Daemon)",
         "Topic :: Communications :: File Sharing",
+        "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
     ],
+    "include_package_data": True,
+    "data_files": data_files,
+    "packages": find_packages(),
+    "install_requires": ["jinja2"],
+    "extras_require": {"thumbnails": ["Pillow"], "audiotags": ["mutagen"]},
+    "entry_points": {"console_scripts": ["copyparty = copyparty.__main__:main"]},
+    "scripts": ["bin/copyparty-fuse.py"],
     "cmdclass": {"clean2": clean2},
 }
 
-
-if setuptools_available:
-    args.update(
-        {
-            "packages": find_packages(),
-            "install_requires": ["jinja2"],
-            "extras_require": {"thumbnails": ["Pillow"]},
-            "include_package_data": True,
-            "entry_points": {
-                "console_scripts": ["copyparty = copyparty.__main__:main"]
-            },
-            "scripts": ["bin/copyparty-fuse.py"],
-        }
-    )
-else:
-    args.update(
-        {
-            "packages": ["copyparty", "copyparty.stolen"],
-            "scripts": ["bin/copyparty-fuse.py"],
-        }
-    )
-
-
-# import pprint
-# pprint.PrettyPrinter().pprint(args)
-# sys.exit(0)
-
 setup(**args)

srv/extend.md

@@ -0,0 +1,141 @@
+# hi
+this showcases my worst idea yet; *extending markdown with inline javascript*
+
+due to obvious reasons it's disabled by default, and can be enabled with `-emp`
+
+the examples are by no means correct, they're as much of a joke as this feature itself
+
+
+### sub-header
+nothing special about this one
+
+
+## except/
+this one becomes a hyperlink to ./except/ thanks to
+* the `copyparty_pre` plugin at the end of this file
+* which is invoked as a markdown filter every time the document is modified
+* which looks for headers ending with a `/` and erwrites all headers below that
+
+it is a passthrough to the markdown extension api, see https://marked.js.org/using_pro
+
+in addition to the markdown extension functions, `ctor` will be called on document init
+
+
+### these/
+and this one becomes ./except/these/
+
+
+#### ones.md
+finally ./except/these/ones.md
+
+
+### also-this.md
+whic hshoud be ./except/also-this.md
+
+
+
+
+# ok
+now for another extension type, `copyparty_post` which is called to manipulate the generated dom instead
+
+`copyparty_post` can have the following functions, all optional
+* `ctor` is called on document init
+* `render` is called when the dom is done but still in-memory
+* `render2` is called with the live browser dom as-displayed
+
+## post example
+
+the values in the `ex:` columns are linkified to `example.com/$value`
+
+| ex:foo       | bar      | ex:baz |
+| ------------ | -------- | ------ |
+| asdf         | nice     | fgsfds |
+| more one row | hi hello | aaa    |
+
+and the table can be sorted by clicking the headers
+
+the difference is that with `copyparty_pre` you'll probably break various copyparty features but if you use `copyparty_post` then future copyparty versions will probably break you
+
+
+
+
+# heres the plugins
+if there is anything below ths line in the preview then the plugin feature is disabled (good)
+
+
+
+
+```copyparty_pre
+ctor() {
+    md_plug['h'] = {
+        on: false,
+        lv: -1,
+        path: []
+    }
+},
+walkTokens(token) {
+    if (token.type == 'heading') {
+        var h = md_plug['h'],
+            is_dir = token.text.endsWith('/');
+        
+        if (h.lv >= token.depth) {
+            h.on = false;
+        }
+        if (!h.on && is_dir) {
+            h.on = true;
+            h.lv = token.depth;
+            h.path = [token.text];
+        }
+        else if (h.on && h.lv < token.depth) {
+            h.path = h.path.slice(0, token.depth - h.lv);
+            h.path.push(token.text);
+        }
+        if (!h.on)
+            return false;
+
+        var path = h.path.join('');
+        var emoji = is_dir ? '📂' : '📜';
+        token.tokens[0].text = '<a href="' + path + '">' + emoji + ' ' + path + '</a>';
+    }
+    if (token.type == 'paragraph') {
+        //console.log(JSON.parse(JSON.stringify(token.tokens)));
+        for (var a = 0; a < token.tokens.length; a++) {
+            var t = token.tokens[a];
+            if (t.type == 'text' || t.type == 'strong' || t.type == 'em') {
+                var ret = '', text = t.text;
+                for (var b = 0; b < text.length; b++)
+                    ret += (Math.random() > 0.5) ? text[b] : text[b].toUpperCase();
+                
+                t.text = ret;
+            }
+        }
+    }
+    return true;
+}
+```
+
+
+
+```copyparty_post
+render(dom) {
+    var ths = dom.querySelectorAll('th');
+    for (var a = 0; a < ths.length; a++) {
+        var th = ths[a];
+        if (th.textContent.indexOf('ex:') === 0) {
+            th.textContent = th.textContent.slice(3);
+            var nrow = 0;
+            while ((th = th.previousSibling) != null)
+                nrow++;
+            
+            var trs = ths[a].parentNode.parentNode.parentNode.querySelectorAll('tr');
+            for (var b = 1; b < trs.length; b++) {
+                var td = trs[b].childNodes[nrow];
+                td.innerHTML = '<a href="//example.com/' + td.innerHTML + '">' + td.innerHTML + '</a>';
+            }
+        }
+    }
+},
+render2(dom) {
+    window.makeSortable(dom.getElementsByTagName('table')[0]);
+}
+```