v0.10.18

idk forgot to pull

.vscode/launch.json

@@ -14,6 +14,8 @@
                 "-emp",
                 "-e2dsa",
                 "-e2ts",
+                "-mtp",
+                ".bpm=f,bin/mtag/audio-bpm.py",
                 "-a",
                 "ed:wark",
                 "-v",

.vscode/launch.py

@@ -0,0 +1,35 @@
+# takes arguments from launch.json
+# is used by no_dbg in tasks.json
+# launches 10x faster than mspython debugpy
+# and is stoppable with ^C
+
+import os
+import sys
+import shlex
+
+sys.path.insert(0, os.getcwd())
+
+import jstyleson
+from copyparty.__main__ import main as copyparty
+
+with open(".vscode/launch.json", "r") as f:
+    tj = f.read()
+
+oj = jstyleson.loads(tj)
+argv = oj["configurations"][0]["args"]
+
+try:
+    sargv = " ".join([shlex.quote(x) for x in argv])
+    print(sys.executable + " -m copyparty " + sargv + "\n")
+except:
+    pass
+
+argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
+try:
+    copyparty(["a"] + argv)
+except SystemExit as ex:
+    if ex.code:
+        raise
+
+print("\n\033[32mokke\033[0m")
+sys.exit(1)

.vscode/tasks.json

@@ -8,8 +8,8 @@
         },
         {
             "label": "no_dbg",
-            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -e2ts -a ed:wark -v srv::r:aed:cnodupe -v dist:dist:r ;exit 1",
-            "type": "shell"
+            "type": "shell",
+            "command": "${config:python.pythonPath} .vscode/launch.py"
         }
     ]
 }

README.md

@@ -12,6 +12,43 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
 * code standard: `black`
 
+📷 screenshots: [browser](#the-browser) // [upload](#uploading) // [md-viewer](#markdown-viewer) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [ie4](#browser-support)
+
+
+## readme toc
+
+* top
+    * [quickstart](#quickstart)
+    * [notes](#notes)
+    * [status](#status)
+* [bugs](#bugs)
+    * [not my bugs](#not-my-bugs)
+* [the browser](#the-browser)
+    * [tabs](#tabs)
+    * [hotkeys](#hotkeys)
+    * [tree-mode](#tree-mode)
+    * [zip downloads](#zip-downloads)
+    * [uploading](#uploading)
+        * [file-search](#file-search)
+    * [markdown viewer](#markdown-viewer)
+    * [other tricks](#other-tricks)
+* [searching](#searching)
+    * [search configuration](#search-configuration)
+    * [metadata from audio files](#metadata-from-audio-files)
+    * [file parser plugins](#file-parser-plugins)
+    * [complete examples](#complete-examples)
+* [browser support](#browser-support)
+* [client examples](#client-examples)
+* [up2k](#up2k)
+* [dependencies](#dependencies)
+    * [optional gpl stuff](#optional-gpl-stuff)
+* [sfx](#sfx)
+    * [sfx repack](#sfx-repack)
+* [install on android](#install-on-android)
+* [dev env setup](#dev-env-setup)
+* [how to release](#how-to-release)
+* [todo](#todo)
+
 
 ## quickstart
 
@@ -27,9 +64,9 @@ you may also want these, especially on servers:
 ## notes
 
 * iPhone/iPad: use Firefox to download files
-* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
-* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
-* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
+* Android-Firefox: takes a while to select files (their fix for ☝️)
+* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
 * paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
   * because no browsers currently implement the media-query to do this properly orz
 
@@ -48,7 +85,7 @@ you may also want these, especially on servers:
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ folders as zip / tar files
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view
@@ -69,14 +106,142 @@ summary: it works! you can use it! (but technically not even close to beta)
 
 # bugs
 
-* probably, pls let me know
+* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
+* Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
+* probably more, pls let me know
+
+## not my bugs
+
+* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
+  * this is an msys2 bug, the regular windows edition of python is fine
+
+
+# the browser
+
+![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/115978054-65106380-a57d-11eb-98f8-59e3dee73557.png)
+
+
+## tabs
+
+* `[🔎]` search by size, date, path/name, mp3-tags ... see [searching](#searching)
+* `[🚀]` and `[🎈]` are the uploaders, see [uploading](#uploading)
+* `[📂]` mkdir, create directories
+* `[📝]` new-md, create a new markdown document
+* `[📟]` send-msg, either to server-log or into textfiles if `--urlform save`
+* `[⚙️]` client configuration options
+
+
+## hotkeys
+
+the browser has the following hotkeys
+* `I/K` prev/next folder
+* `P` parent folder
+* when playing audio:
+  * `0..9` jump to 10%..90%
+  * `U/O` skip 10sec back/forward
+  * `J/L` prev/next song
+    * `J` also starts playing the folder
+
+
+## tree-mode
+
+by default there's a breadcrumbs path; you can replace this with a tree-browser sidebar thing by clicking the 🌲
+
+click `[-]` and `[+]` to adjust the size, and the `[a]` toggles if the tree should widen dynamically as you go deeper or stay fixed-size
+
+
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+you can also zip a selection of files or folders by clicking them in the browser, that brings up a selection editor and zip button in the bottom right
+
+![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/116008321-372a2e00-a614-11eb-9a4a-4a1fd9074224.png)
+
+## uploading
+
+two upload methods are available in the html client:
+* 🎈 bup, the basic uploader, supports almost every browser since netscape 4.0
+* 🚀 up2k, the fancy one
+
+up2k has several advantages:
+* you can drop folders into the browser (files are added recursively)
+* files are processed in chunks, and each chunk is checksummed
+  * uploads resume if they are interrupted (for example by a reboot)
+  * server detects any corruption; the client reuploads affected chunks
+  * the client doesn't upload anything that already exists on the server
+* the last-modified timestamp of the file is preserved
+
+see [up2k](#up2k) for details on how it works
+
+![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/115978061-680b5400-a57d-11eb-9ef6-cbb5f60aeccc.png)
+
+the up2k UI is the epitome of polished inutitive experiences:
+* "parallel uploads" specifies how many chunks to upload at the same time
+* `[🏃]` analysis of other files should continue while one is uploading
+* `[💭]` ask for confirmation before files are added to the list
+* `[💤]` sync uploading between other copyparty tabs so only one is active
+* `[🔎]` switch between upload and file-search mode
+
+and then theres the tabs below it,
+* `[ok]` is uploads which completed successfully
+* `[ng]` is the uploads which failed / got rejected (already exists, ...)
+* `[done]` shows a combined list of `[ok]` and `[ng]`, chronological order
+* `[busy]` files which are currently hashing, pending-upload, or uploading
+  * plus up to 3 entries each from `[done]` and `[que]` for context
+* `[que]` is all the files that are still queued
+
+protip: you can avoid scaring away users by hiding some of the UI with hacks like [docs/minimal-up2k.html](docs/minimal-up2k.html)
+
+### file-search
+
+![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/116008320-36919780-a614-11eb-803f-04162326a700.png)
+
+in the 🚀 up2k tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere already
+
+files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
+* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else
+
+adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files
+
+note that since up2k has to read the file twice, 🎈 bup can be up to 2x faster if your internet connection is faster than the read-speed of your HDD
+
+up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well thanks to tls also functioning as an integrity check
+
+
+## markdown viewer
+
+![copyparty-md-read-fs8](https://user-images.githubusercontent.com/241032/115978057-66419080-a57d-11eb-8539-d2be843991aa.png)
+
+* the document preview has a max-width which is the same as an A4 paper when printed
+
+
+## other tricks
+
+* you can link a particular timestamp in an audio file by adding it to the URL, such as `&20` / `&20s` / `&1m20` / `&t=1:20` after the `.../#af-c8960dab`
 
 
 # searching
 
+![copyparty-search-fs8](https://user-images.githubusercontent.com/241032/115978060-6772bd80-a57d-11eb-81d3-174e869b72c3.png)
+
 when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
 * make search queries by `size`/`date`/`directory-path`/`filename`, or...
-* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+* drag/drop a local file to see if the same contents exist somewhere on the server, see [file-search](#file-search)
 
 path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
 * path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
@@ -98,19 +263,24 @@ through arguments:
 * `-e2tsr` deletes all existing tags, so a full reindex
 
 the same arguments can be set as volume flags, in addition to `d2d` and `d2t` for disabling:
-* `-v ~/music::ce2dsa:ce2tsr` does a full reindex of everything on startup
-* `-v ~/music::cd2d` disables **all** indexing, even if any `-e2*` are on
-* `-v ~/music::cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
+* `-v ~/music::r:ce2dsa:ce2tsr` does a full reindex of everything on startup
+* `-v ~/music::r:cd2d` disables **all** indexing, even if any `-e2*` are on
+* `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 
 `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
 
+
+## metadata from audio files
+
 `-mte` decides which tags to index and display in the browser (and also the display order), this can be changed per-volume:
-* `-v ~/music::cmte=title,artist` indexes and displays *title* followed by *artist*
+* `-v ~/music::r:cmte=title,artist` indexes and displays *title* followed by *artist*
 
 if you add/remove a tag from `mte` you will need to run with `-e2tsr` once to rebuild the database, otherwise only new files will be affected
 
 `-mtm` can be used to add or redefine a metadata mapping, say you have media files with `foo` and `bar` tags and you want them to display as `qux` in the browser (preferring `foo` if both are present), then do `-mtm qux=foo,bar` and now you can `-mte artist,title,qux`
 
+tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric value
+
 see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
 
 `--no-mutagen` disables mutagen and uses ffprobe instead, which...
@@ -120,6 +290,58 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
 * more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
 
 
+## file parser plugins
+
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (as argument or volume flag), there is a default timeout of 30sec
+
+* `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
+* `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
+* `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
+
+
+## complete examples
+
+* read-only music server with bpm and key scanning  
+  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+
+
+# browser support
+
+![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/116009043-a1909d80-a617-11eb-9140-037ad6604899.png)
+
+`ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
+
+| feature         | ie6 | ie9 | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+| --------------- | --- | --- | ---- | ---- | ----- | ---- | --- | ---- |
+| browse files    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| basic uploader  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| make directory  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| send message    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| set sort order  |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| zip selection   |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| directory tree  |  -  |  -  | `*1` | yep  | yep   | yep  | yep | yep  |
+| up2k            |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| icons work      |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown editor |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown viewer |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| play mp3/m4a    |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| play ogg/opus   |  -  |  -  |  -   |  -   | yep   | yep  | `*2` | yep |
+
+* internet explorer 6 to 8 behave the same
+* firefox 52 and chrome 49 are the last winxp versions
+* `*1` only public folders (login session is dropped) and no history / back-button
+* `*2` using a wasm decoder which can sometimes get stuck and consumes a bit more power
+
+quick summary of more eccentric web-browsers trying to view a directory index:
+* safari (14.0.3/macos) is chrome with janky wasm, so playing opus can deadlock the javascript engine
+* safari (14.0.1/iOS) same as macos, except it recovers from the deadlocks if you poke it a bit
+* links (2.21/macports) can browse, login, upload/mkdir/msg
+* lynx (2.8.9/macports) can browse, login, upload/mkdir/msg
+* w3m (0.5.3/macports) can browse, login, upload at 100kB/s, mkdir/msg
+* netsurf (3.10/arch) is basically ie6 with much better css (javascript has almost no effect)
+* ie4 and netscape 4.0 can browse (text is yellow on white), upload with `?b=u`
+* SerenityOS (22d13d8) hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)
@@ -144,6 +366,22 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
     b512 <movie.mkv
 
 
+# up2k
+
+quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+* the up2k client splits a file into an "optimal" number of chunks
+  * 1 MiB each, unless that becomes more than 256 chunks
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256# or chunksize >= 32M
+* client posts the list of hashes, filename, size, last-modified
+* server creates the `wark`, an identifier for this upload
+  * `sha512( salt + filesize + chunk_hashes )`
+  * and a sparse file is created for the chunks to drop into
+* client uploads each chunk
+  * header entries for the chunk-hash and wark
+  * server writes chunks into place based on the hash
+* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+
+
 # dependencies
 
 * `jinja2` (is built into the SFX)
@@ -156,9 +394,16 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
 * `Pillow` (requires py2.7 or py3.5+)
 
 
+## optional gpl stuff
+
+some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
+
+these are standalone programs and will never be imported / evaluated by copyparty
+
+
 # sfx
 
-currently there are two self-contained binaries:
+currently there are two self-contained "binaries":
 * [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
 * [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
 
@@ -210,6 +455,7 @@ pip install black bandit pylint flake8  # vscode tooling
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies
+* `git tag v1.2.3 && git push origin --tags`
 * create github release with `make-tgz-release.sh`
 * upload to pypi with `make-pypi-release.(sh|bat)`
 * create sfx with `make-sfx.sh`
@@ -219,15 +465,20 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
+* separate sqlite table per tag
+* audio fingerprinting
+* readme.md as epilogue
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* drop onto folders
 * `os.copy_file_range` for up2k cloning
-* up2k partials ui
 * support pillow-simd
-* cache sha512 chunks on client
-* comment field
-* ~~look into android thumbnail cache file format~~ bad idea
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||
+
+discarded ideas
+
+* up2k partials ui
+* cache sha512 chunks on client
+* comment field
+* look into android thumbnail cache file format

bin/README.md

@@ -1,4 +1,4 @@
-# copyparty-fuse.py
+# [`copyparty-fuse.py`](copyparty-fuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
 * **supports Linux** -- expect `117 MiB/s` sequential read
@@ -29,7 +29,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse🅱️.py
+# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
 * mount a copyparty server as a local filesystem (read-only)
 * does the same thing except more correct, `samba` approves
 * **supports Linux** -- expect `18 MiB/s` (wait what)
@@ -37,5 +37,11 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# copyparty-fuse-streaming.py
+# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
 * pretend this doesn't exist
+
+
+
+# [`mtag/`](mtag/)
+* standalone programs which perform misc. file analysis
+* copyparty can Popen programs like these during file indexing to collect additional metadata

bin/copyparty-fuse.py

@@ -1008,6 +1008,12 @@ def main():
         log = null_log
         dbg = null_log
 
+    if ar.a and ar.a.startswith("$"):
+        fn = ar.a[1:]
+        log("reading password from file [{}]".format(fn))
+        with open(fn, "rb") as f:
+            ar.a = f.read().decode("utf-8").strip()
+
     if WINDOWS:
         os.system("rem")
 

bin/mtag/README.md

@@ -0,0 +1,34 @@
+standalone programs which take an audio file as argument
+
+some of these rely on libraries which are not MIT-compatible
+
+* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
+* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
+
+
+# dependencies
+
+run [`install-deps.sh`](install-deps.sh) to build/install most dependencies required by these programs (supports windows/linux/macos)
+
+*alternatively* (or preferably) use packages from your distro instead, then you'll need at least these:
+
+* from distro: `numpy vamp-plugin-sdk beatroot-vamp mixxx-keyfinder ffmpeg`
+* from pypy: `keyfinder vamp`
+
+
+# usage from copyparty
+
+`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
+
+* `f,` makes the detected value replace any existing values
+* the `.` in `.bpm` indicates numeric value
+* assumes the python files are in the folder you're launching copyparty from, replace the filename with a relative/absolute path if that's not the case
+* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
+
+
+## usage with volume-flags
+
+instead of affecting all volumes, you can set the options for just one volume like so:
+```
+copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
+```

bin/mtag/audio-bpm.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import vamp
+import tempfile
+import numpy as np
+import subprocess as sp
+
+from copyparty.util import fsenc
+
+"""
+dep: vamp
+dep: beatroot-vamp
+dep: ffmpeg
+"""
+
+
+def det(tf):
+    # fmt: off
+    sp.check_call([
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-v", "fatal",
+        "-ss", "13",
+        "-y", "-i", fsenc(sys.argv[1]),
+        "-ac", "1",
+        "-ar", "22050",
+        "-t", "300",
+        "-f", "f32le",
+        tf
+    ])
+    # fmt: on
+
+    with open(tf, "rb") as f:
+        d = np.fromfile(f, dtype=np.float32)
+        try:
+            # 98% accuracy on jcore
+            c = vamp.collect(d, 22050, "beatroot-vamp:beatroot")
+            cl = c["list"]
+        except:
+            # fallback; 73% accuracy
+            plug = "vamp-example-plugins:fixedtempo"
+            c = vamp.collect(d, 22050, plug, parameters={"maxdflen": 40})
+            print(c["list"][0]["label"].split(" ")[0])
+            return
+
+        # throws if detection failed:
+        bpm = float(cl[-1]["timestamp"] - cl[1]["timestamp"])
+        bpm = round(60 * ((len(cl) - 1) / bpm), 2)
+        print(f"{bpm:.2f}")
+
+
+def main():
+    with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f:
+        f.write(b"h")
+        tf = f.name
+
+    try:
+        det(tf)
+    except:
+        pass
+    finally:
+        os.unlink(tf)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/audio-key.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import sys
+import keyfinder
+
+"""
+dep: github/mixxxdj/libkeyfinder
+dep: pypi/keyfinder
+dep: ffmpeg
+
+note: cannot fsenc
+"""
+
+
+try:
+    print(keyfinder.key(sys.argv[1]).camelot())
+except:
+    pass

bin/mtag/install-deps.sh

@@ -0,0 +1,265 @@
+#!/bin/bash
+set -e
+
+
+# install dependencies for audio-*.py
+#
+# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
+# win64: requires msys2-mingw64 environment
+# macos: requires macports
+#
+# has the following manual dependencies, especially on mac:
+#   https://www.vamp-plugins.org/pack.html
+#
+# installs stuff to the following locations:
+#   ~/pe/
+#   whatever your python uses for --user packages
+#
+# does the following terrible things:
+#   modifies the keyfinder python lib to load the .so in ~/pe
+
+
+linux=1
+
+win=
+[ ! -z "$MSYSTEM" ] || [ -e /msys2.exe ] && {
+	[ "$MSYSTEM" = MINGW64 ] || {
+		echo windows detected, msys2-mingw64 required
+		exit 1
+	}
+	pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	win=1
+	linux=
+}
+
+mac=
+[ $(uname -s) = Darwin ] && {
+	#pybin="$(printf '%s\n' /opt/local/bin/python* | (sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) | (sort -nr || cat) | (sed -E 's/([^ ]*) (.*)/\2\1/' || cat) | grep -E '/(python|pypy)[0-9\.-]*$' | head -n 1)"
+	pybin=/opt/local/bin/python3.9
+	[ -e "$pybin" ] || {
+		echo mac detected, python3 from macports required
+		exit 1
+	}
+	pkgs='ffmpeg python39 py39-wheel'
+	ninst=$(port installed | awk '/^  /{print$1}' | sort | uniq | grep -E '^('"$(echo "$pkgs" | tr ' ' '|')"')$' | wc -l)
+	[ $ninst -eq 3 ] || {
+		sudo port install $pkgs
+	}
+	mac=1
+	linux=
+}
+
+hash -r
+
+[ $mac ] || {
+	command -v python3 && pybin=python3 || pybin=python
+}
+
+$pybin -m pip install --user numpy
+
+
+command -v gnutar && tar() { gnutar "$@"; }
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+
+
+need() {
+	command -v $1 >/dev/null || {
+		echo need $1
+		exit 1
+	}
+}
+need cmake
+need ffmpeg
+need $pybin
+#need patchelf
+
+
+td="$(mktemp -d)"
+cln() {
+	rm -rf "$td"
+}
+trap cln EXIT
+cd "$td"
+pwd
+
+
+dl_text() {
+	command -v curl >/dev/null && exec curl "$@"
+	exec wget -O- "$@"
+}
+dl_files() {
+	local yolo= ex=
+	[ $1 = "yolo" ] && yolo=1 && ex=k && shift
+	command -v curl >/dev/null && exec curl -${ex}JOL "$@"
+	
+	[ $yolo ] && ex=--no-check-certificate
+	exec wget --trust-server-names $ex "$@"
+}
+export -f dl_files
+
+
+github_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.tarball_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"tarball_url": "/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+gitlab_tarball() {
+	dl_text "$1" |
+	tee json |
+	(
+		# prefer jq if available
+		jq -r '.[0].assets.sources[]|select(.format|test("tar.gz")).url' ||
+
+		# fallback to abomination
+		tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' |
+	tee links |
+	xargs -0 bash -c 'dl_files "$@"' _
+}
+
+
+install_keyfinder() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
+	
+	github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
+
+	tar -xf mixxxdj-libkeyfinder-*
+	rm -- *.tar.gz
+	cd mixxxdj-libkeyfinder*
+	
+	h="$HOME"
+	so="lib/libkeyfinder.so"
+	memes=()
+
+	[ $win ] &&
+		so="bin/libkeyfinder.dll" &&
+		h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
+		memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
+	
+	[ $mac ] &&
+		so="lib/libkeyfinder.dylib"
+
+	cmake -DCMAKE_INSTALL_PREFIX="$h/pe/keyfinder" "${memes[@]}" -S . -B build
+	cmake --build build --parallel $(nproc || echo 4)
+	cmake --install build
+
+	libpath="$h/pe/keyfinder/$so"
+	[ $linux ] && [ ! -e "$libpath" ] &&
+		so=lib64/libkeyfinder.so
+	
+	libpath="$h/pe/keyfinder/$so"
+	[ -e "$libpath" ] || {
+		echo "so not found at $sop"
+		exit 1
+	}
+	
+	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
+	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
+	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
+	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
+	$pybin -m pip install --user keyfinder
+
+	pypath="$($pybin -c 'import keyfinder; print(keyfinder.__file__)')"
+	for pyso in "${pypath%/*}"/*.so; do
+		[ -e "$pyso" ] || break
+		patchelf --set-rpath "${libpath%/*}" "$pyso" ||
+			echo "WARNING: patchelf failed (only fatal on musl-based distros)"
+	done
+	
+	mv "$pypath"{,.bak}
+	(
+		printf 'import ctypes\nctypes.cdll.LoadLibrary("%s")\n' "$libpath"
+		cat "$pypath.bak"
+	) >"$pypath"
+
+	echo
+	echo libkeyfinder successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+have_beatroot() {
+	$pybin -c 'import vampyhost, sys; plugs = vampyhost.list_plugins(); sys.exit(0 if "beatroot-vamp:beatroot" in plugs else 1)'
+}
+
+
+install_vamp() {
+	# windows support:
+	#   use msys2 in mingw-w64 mode
+	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
+	
+	$pybin -m pip install --user vamp
+
+	have_beatroot || {
+		printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
+		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
+		sha512sum -c <(
+			echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874  -"
+		) <beatroot-vamp-v1.0.tar.gz
+		tar -xf beatroot-vamp-v1.0.tar.gz 
+		cd beatroot-vamp-v1.0
+		make -f Makefile.linux -j4
+		# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
+		mkdir ~/vamp
+		cp -pv beatroot-vamp.* ~/vamp/
+	}
+	
+	have_beatroot &&
+		printf '\033[32mfound the vamp beatroot plugin, nice\033[0m\n' ||
+		printf '\033[31mWARNING: could not find the vamp beatroot plugin, please install it for optimal results\033[0m\n'
+}
+
+
+# not in use because it kinda segfaults, also no windows support
+install_soundtouch() {
+	gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
+	
+	tar -xvf soundtouch-*
+	rm -- *.tar.gz
+	cd soundtouch-*
+	
+	# https://github.com/jrising/pysoundtouch
+	./bootstrap
+	./configure --enable-integer-samples CXXFLAGS="-fPIC" --prefix="$HOME/pe/soundtouch"
+	make -j$(nproc || echo 4)
+	make install
+	
+	CFLAGS=-I$HOME/pe/soundtouch/include/ \
+	LDFLAGS=-L$HOME/pe/soundtouch/lib \
+	$pybin -m pip install --user git+https://github.com/snowxmas/pysoundtouch.git
+	
+	pypath="$($pybin -c 'import importlib; print(importlib.util.find_spec("soundtouch").origin)')"
+	libpath="$(echo "$HOME/pe/soundtouch/lib/")"
+	patchelf --set-rpath "$libpath" "$pypath"
+
+	echo
+	echo soundtouch successfully installed to the following locations:
+	echo "  $libpath"
+	echo "  $pypath"
+}
+
+
+[ "$1" = keyfinder ] && { install_keyfinder; exit $?; }
+[ "$1" = soundtouch ] && { install_soundtouch; exit $?; }
+[ "$1" = vamp ] && { install_vamp; exit $?; }
+
+echo no args provided, installing keyfinder and vamp
+install_keyfinder
+install_vamp

bin/mtag/sleep.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+
+import time
+import random
+
+v = random.random() * 6
+time.sleep(v)
+print(f"{v:.2f}")

contrib/systemd/copyparty.service

@@ -12,7 +12,7 @@
 Description=copyparty file server
 
 [Service]
-ExecStart=/usr/bin/python /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 [Install]

copyparty/__init__.py

@@ -16,12 +16,17 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 
 class EnvParams(object):
     def __init__(self):
         self.mod = os.path.dirname(os.path.realpath(__file__))
+        if self.mod.endswith("__init__"):
+            self.mod = os.path.dirname(self.mod)
+
         if sys.platform == "win32":
             self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
         elif sys.platform == "darwin":

copyparty/__main__.py

@@ -16,12 +16,14 @@ import shutil
 import filecmp
 import locale
 import argparse
+import threading
+import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, VT100
+from .__init__ import E, WINDOWS, VT100, PY2
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab
+from .util import py_desc, align_tab, IMPLICATIONS
 
 HAVE_SSL = True
 try:
@@ -53,6 +55,16 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+class Dodge11874(RiceFormatter):
+    def __init__(self, *args, **kwargs):
+        kwargs["width"] = 9003
+        super(Dodge11874, self).__init__(*args, **kwargs)
+
+
+def warn(msg):
+    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+
+
 def ensure_locale():
     for x in [
         "en_US.UTF-8",
@@ -160,34 +172,19 @@ def configure_ssl_ciphers(al):
         sys.exit(0)
 
 
-def main():
-    time.strptime("19970815", "%Y%m%d")  # python#7980
-    if WINDOWS:
-        os.system("rem")  # enables colors
+def sighandler(sig=None, frame=None):
+    msg = [""] * 5
+    for th in threading.enumerate():
+        msg.append(str(th))
+        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
 
-    desc = py_desc().replace("[", "\033[1;30m[")
+    msg.append("\n")
+    print("\n".join(msg))
 
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
-
-    ensure_locale()
-    if HAVE_SSL:
-        ensure_cert()
-
-    deprecated = [["-e2s", "-e2ds"]]
-    for dk, nk in deprecated:
-        try:
-            idx = sys.argv.index(dk)
-        except:
-            continue
-
-        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
-        sys.argv[idx] = nk
-        time.sleep(2)
 
+def run_argparse(argv, formatter):
     ap = argparse.ArgumentParser(
-        formatter_class=RiceFormatter,
+        formatter_class=formatter,
         prog="copyparty",
         description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
         epilog=dedent(
@@ -199,6 +196,9 @@ def main():
             
             list of cflags:
               "cnodupe" rejects existing files (instead of symlinking them)
+              "ce2d" sets -e2d (all -e2* args can be set using ce2* cflags)
+              "cd2t" disables metadata collection, overrides -e2t*
+              "cd2d" disables all database stuff, overrides -e2*
 
             example:\033[35m
               -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
@@ -237,15 +237,18 @@ def main():
     ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
     ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
     ap.add_argument("-q", action="store_true", help="quiet")
+    ap.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
     ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
-    ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
+    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 
     ap2 = ap.add_argument_group('database options')
@@ -260,26 +263,57 @@ def main():
     ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
 
     ap2 = ap.add_argument_group('SSL/TLS options')
     ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
     ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", type=str, help="ssl/tls versions to allow")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="ssl/tls versions to allow")
     ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     
-    al = ap.parse_args()
+    return ap.parse_args(args=argv[1:])
     # fmt: on
 
+
+def main(argv=None):
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if WINDOWS:
+        os.system("rem")  # enables colors
+
+    if argv is None:
+        argv = sys.argv
+
+    desc = py_desc().replace("[", "\033[1;30m[")
+
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
+    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+
+    ensure_locale()
+    if HAVE_SSL:
+        ensure_cert()
+
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        argv[idx] = nk
+        time.sleep(2)
+
+    try:
+        al = run_argparse(argv, RiceFormatter)
+    except AssertionError:
+        al = run_argparse(argv, Dodge11874)
+
     # propagate implications
-    for k1, k2 in [
-        ["e2dsa", "e2ds"],
-        ["e2ds", "e2d"],
-        ["e2tsr", "e2ts"],
-        ["e2ts", "e2t"],
-        ["e2t", "e2d"],
-    ]:
+    for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):
             setattr(al, k2, True)
 
@@ -300,7 +334,15 @@ def main():
         if al.ciphers:
             configure_ssl_ciphers(al)
     else:
-        print("\033[33m  ssl module does not exist; cannot enable https\033[0m\n")
+        warn("ssl module does not exist; cannot enable https")
+
+    if PY2 and WINDOWS and al.e2d:
+        warn(
+            "windows py2 cannot do unicode filenames with -e2d\n"
+            + "  (if you crash with codec errors then that is why)"
+        )
+
+    # signal.signal(signal.SIGINT, sighandler)
 
     SvcHub(al).run()
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 9, 4)
-CODENAME = "the strongest music server"
-BUILD_DT = (2021, 3, 5)
+VERSION = (0, 10, 18)
+CODENAME = "zip it"
+BUILD_DT = (2021, 5, 13)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
 import re
+import os
+import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
-from .util import undot, Pebkac, fsdec, fsenc, statdir
+from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint
 
 
 class VFS(object):
@@ -21,6 +23,14 @@ class VFS(object):
         self.nodes = {}  # child nodes
         self.all_vols = {vpath: self}  # flattened recursive
 
+    def __repr__(self):
+        return "VFS({})".format(
+            ", ".join(
+                "{}={!r}".format(k, self.__dict__[k])
+                for k in "realpath vpath uread uwrite flags".split()
+            )
+        )
+
     def _trk(self, vol):
         self.all_vols[vol.vpath] = vol
         return vol
@@ -44,6 +54,7 @@ class VFS(object):
                 self.uwrite,
                 self.flags,
             )
+            self._trk(vn)
             self.nodes[name] = vn
             return self._trk(vn.add(src, dst))
 
@@ -100,17 +111,42 @@ class VFS(object):
         if rem:
             rp += "/" + rem
 
-        return fsdec(os.path.realpath(fsenc(rp)))
+        try:
+            return fsdec(os.path.realpath(fsenc(rp)))
+        except:
+            if not WINDOWS:
+                raise
+
+            # cpython bug introduced in 3.8, still exists in 3.9.1;
+            # some win7sp1 and win10:20H2 boxes cannot realpath a
+            # networked drive letter such as b"n:" or b"n:\\"
+            #
+            # requirements to trigger:
+            #  * bytestring (not unicode str)
+            #  * just the drive letter (subfolders are ok)
+            #  * networked drive (regular disks and vmhgfs are ok)
+            #  * on an enterprise network (idk, cannot repro with samba)
+            #
+            # hits the following exceptions in succession:
+            #  * access denied at L601: "path = _getfinalpathname(path)"
+            #  * "cant concat str to bytes" at L621: "return path + tail"
+            #
+            return os.path.realpath(rp)
 
     def ls(self, rem, uname, scandir, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
-        real = list(statdir(print, scandir, lstat, abspath))
+        real = list(statdir(nuprint, scandir, lstat, abspath))
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
-                if uname in vn2.uread or "*" in vn2.uread:
+                if (
+                    uname in vn2.uread
+                    or "*" in vn2.uread
+                    or uname in vn2.uwrite
+                    or "*" in vn2.uwrite
+                ):
                     virt_vis[name] = vn2
 
             # no vfs nodes in the list of real inodes
@@ -118,6 +154,78 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+
+                rm = [x for x in rd if x[0] not in flt]
+                [rd.remove(x) for x in rm]
+
+                rm = [x for x in vd.keys() if x not in flt]
+                [vd.pop(x) for x in rm]
+
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)
@@ -138,6 +246,7 @@ class AuthSrv(object):
         self.args = args
         self.log_func = log_func
         self.warn_anonwrite = warn_anonwrite
+        self.line_ctr = 0
 
         if WINDOWS:
             self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -147,14 +256,8 @@ class AuthSrv(object):
         self.mutex = threading.Lock()
         self.reload()
 
-    def log(self, msg):
-        self.log_func("auth", msg)
-
-    def invert(self, orig):
-        if PY2:
-            return {v: k for k, v in orig.iteritems()}
-        else:
-            return {v: k for k, v in orig.items()}
+    def log(self, msg, c=0):
+        self.log_func("auth", msg, c)
 
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
@@ -169,7 +272,9 @@ class AuthSrv(object):
     def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount):
         vol_src = None
         vol_dst = None
+        self.line_ctr = 0
         for ln in [x.decode("utf-8").strip() for x in fd]:
+            self.line_ctr += 1
             if not ln and vol_src is not None:
                 vol_src = None
                 vol_dst = None
@@ -199,17 +304,45 @@ class AuthSrv(object):
                 mflags[vol_dst] = {}
                 continue
 
-            lvl, uname = ln.split(" ")
-            if lvl in "ra":
-                mread[vol_dst].append(uname)
-            if lvl in "wa":
-                mwrite[vol_dst].append(uname)
-            if lvl == "c":
-                cval = True
-                if "=" in uname:
-                    uname, cval = uname.split("=", 1)
+            if len(ln) > 1:
+                lvl, uname = ln.split(" ")
+            else:
+                lvl = ln
+                uname = "*"
 
-                mflags[vol_dst][uname] = cval
+            self._read_vol_str(
+                lvl, uname, mread[vol_dst], mwrite[vol_dst], mflags[vol_dst]
+            )
+
+    def _read_vol_str(self, lvl, uname, mr, mw, mf):
+        if lvl == "c":
+            cval = True
+            if "=" in uname:
+                uname, cval = uname.split("=", 1)
+
+            self._read_volflag(mf, uname, cval, False)
+            return
+
+        if uname == "":
+            uname = "*"
+
+        if lvl in "ra":
+            mr.append(uname)
+
+        if lvl in "wa":
+            mw.append(uname)
+
+    def _read_volflag(self, flags, name, value, is_list):
+        if name not in ["mtp"]:
+            flags[name] = value
+            return
+
+        if not is_list:
+            value = [value]
+        elif not value:
+            return
+
+        flags[name] = flags.get(name, []) + value
 
     def reload(self):
         """
@@ -232,7 +365,7 @@ class AuthSrv(object):
 
         if self.args.v:
             # list of src:dst:permset:permset:...
-            # permset is [rwa]username
+            # permset is [rwa]username or [c]flag
             for v_str in self.args.v:
                 m = self.re_vol.match(v_str)
                 if not m:
@@ -249,27 +382,17 @@ class AuthSrv(object):
 
                 perms = perms.split(":")
                 for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
-                    if lvl == "c":
-                        cval = True
-                        if "=" in uname:
-                            uname, cval = uname.split("=", 1)
-
-                        mflags[dst][uname] = cval
-                        continue
-
-                    if uname == "":
-                        uname = "*"
-
-                    if lvl in "ra":
-                        mread[dst].append(uname)
-
-                    if lvl in "wa":
-                        mwrite[dst].append(uname)
+                    self._read_vol_str(lvl, uname, mread[dst], mwrite[dst], mflags[dst])
 
         if self.args.c:
             for cfg_fn in self.args.c:
                 with open(cfg_fn, "rb") as f:
-                    self._parse_config_file(f, user, mread, mwrite, mflags, mount)
+                    try:
+                        self._parse_config_file(f, user, mread, mwrite, mflags, mount)
+                    except:
+                        m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
+                        print(m.format(cfg_fn, self.line_ctr))
+                        raise
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
@@ -304,12 +427,14 @@ class AuthSrv(object):
 
         if missing_users:
             self.log(
-                "\033[31myou must -a the following users: "
-                + ", ".join(k for k in sorted(missing_users))
-                + "\033[0m"
+                "you must -a the following users: "
+                + ", ".join(k for k in sorted(missing_users)),
+                c=1,
             )
             raise Exception("invalid config")
 
+        all_mte = {}
+        errors = False
         for vol in vfs.all_vols.values():
             if (self.args.e2ds and vol.uwrite) or self.args.e2dsa:
                 vol.flags["e2ds"] = True
@@ -321,23 +446,88 @@ class AuthSrv(object):
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
+            for k1, k2 in IMPLICATIONS:
+                if k1 in vol.flags:
+                    vol.flags[k2] = True
+
             # default tag-list if unset
             if "mte" not in vol.flags:
                 vol.flags["mte"] = self.args.mte
 
+            # append parsers from argv to volume-flags
+            self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
+
+            # d2d drops all database features for a volume
+            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"]]:
+                if not vol.flags.get(grp, False):
+                    continue
+
+                vol.flags["d2t"] = True
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # mt* needs e2t so drop those too
+            for grp, rm in [["e2t", "mt"]]:
+                if vol.flags.get(grp, False):
+                    continue
+
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
+            # verify tags mentioned by -mt[mp] are used by -mte
+            local_mtp = {}
+            local_only_mtp = {}
+            for a in vol.flags.get("mtp", []) + vol.flags.get("mtm", []):
+                a = a.split("=")[0]
+                local_mtp[a] = True
+                local = True
+                for b in self.args.mtp or []:
+                    b = b.split("=")[0]
+                    if a == b:
+                        local = False
+
+                if local:
+                    local_only_mtp[a] = True
+
+            local_mte = {}
+            for a in vol.flags.get("mte", "").split(","):
+                local = True
+                all_mte[a] = True
+                local_mte[a] = True
+                for b in self.args.mte.split(","):
+                    if not a or not b:
+                        continue
+
+                    if a == b:
+                        local = False
+
+            for mtp in local_only_mtp.keys():
+                if mtp not in local_mte:
+                    m = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)'
+                    self.log(m.format(vol.vpath, mtp), 1)
+                    errors = True
+
+        for mtp in self.args.mtp or []:
+            mtp = mtp.split("=")[0]
+            if mtp not in all_mte:
+                m = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volume-flag)'
+                self.log(m.format(mtp), 1)
+                errors = True
+
+        if errors:
+            sys.exit(1)
+
         try:
             v, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == v.realpath:
                 self.warn_anonwrite = False
-                msg = "\033[31manyone can read/write the current directory: {}\033[0m"
-                self.log(msg.format(v.realpath))
+                msg = "anyone can read/write the current directory: {}"
+                self.log(msg.format(v.realpath), c=1)
         except Pebkac:
             self.warn_anonwrite = True
 
         with self.mutex:
             self.vfs = vfs
             self.user = user
-            self.iuser = self.invert(user)
+            self.iuser = {v: k for k, v in user.items()}
 
         # import pprint
         # pprint.pprint({"usr": user, "rd": mread, "wr": mwrite, "mnt": mount})

copyparty/broker_mp.py

@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()

copyparty/broker_mpw.py

@@ -49,11 +49,11 @@ class MpWorker(object):
         # print('k')
         pass
 
-    def log(self, src, msg):
-        self.q_yield.put([0, "log", [src, msg]])
+    def log(self, src, msg, c=0):
+        self.q_yield.put([0, "log", [src, msg, c]])
 
-    def logw(self, msg):
-        self.log("mp{}".format(self.n), msg)
+    def logw(self, msg, c=0):
+        self.log("mp{}".format(self.n), msg, c)
 
     def httpdrop(self, addr):
         self.q_yield.put([0, "httpdrop", [addr]])
@@ -73,7 +73,9 @@ class MpWorker(object):
                 if PY2:
                     sck = pickle.loads(sck)  # nosec
 
-                self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
+                if self.args.log_conn:
+                    self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+                
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:

copyparty/broker_thr.py

@@ -28,7 +28,9 @@ class BrokerThr(object):
     def put(self, want_retval, dest, *args):
         if dest == "httpconn":
             sck, addr = args
-            self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
+
             self.httpsrv.accept(sck, addr)
 
         else:

copyparty/httpcli.py

@@ -7,13 +7,16 @@ import gzip
 import time
 import copy
 import json
+import string
 import socket
 import ctypes
 from datetime import datetime
 import calendar
 
-from .__init__ import E, PY2, WINDOWS
+from .__init__ import E, PY2, WINDOWS, ANYWIN
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .szip import StreamZip
+from .star import StreamTar
 
 if not PY2:
     unicode = str
@@ -41,17 +44,21 @@ class HttpCli(object):
         self.absolute_urls = False
         self.out_headers = {"Access-Control-Allow-Origin": "*"}
 
-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)
 
     def _check_nonfatal(self, ex):
-        return ex.code < 400 or ex.code == 404
+        return ex.code < 400 or ex.code in [404, 429]
 
     def _assert_safe_rem(self, rem):
         # sanity check to prevent any disasters
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def j2(self, name, **kwargs):
+        tpl = self.conn.hsrv.j2[name]
+        return tpl.render(**kwargs) if kwargs else tpl
+
     def run(self):
         """returns true if connection can be reused"""
         self.keepalive = False
@@ -63,18 +70,18 @@ class HttpCli(object):
 
             if not headerlines[0]:
                 # seen after login with IE6.0.2900.5512.xpsp.080413-2111 (xp-sp3)
-                self.log("\033[1;31mBUG: trailing newline from previous request\033[0m")
+                self.log("BUG: trailing newline from previous request", c="1;31")
                 headerlines.pop(0)
 
             try:
-                self.mode, self.req, _ = headerlines[0].split(" ")
+                self.mode, self.req, self.http_ver = headerlines[0].split(" ")
             except:
                 raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
 
         except Pebkac as ex:
             # self.log("pebkac at httpcli.run #1: " + repr(ex))
             self.keepalive = self._check_nonfatal(ex)
-            self.loud_reply(str(ex), status=ex.code)
+            self.loud_reply(unicode(ex), status=ex.code)
             return self.keepalive
 
         # time.sleep(0.4)
@@ -86,30 +93,13 @@ class HttpCli(object):
             self.headers[k.lower()] = v.strip()
 
         v = self.headers.get("connection", "").lower()
-        self.keepalive = not v.startswith("close")
+        self.keepalive = not v.startswith("close") and self.http_ver != "HTTP/1.0"
 
         v = self.headers.get("x-forwarded-for", None)
         if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
             self.ip = v.split(",")[0]
             self.log_src = self.conn.set_rproxy(self.ip)
 
-        self.uname = "*"
-        if "cookie" in self.headers:
-            cookies = self.headers["cookie"].split(";")
-            for k, v in [x.split("=", 1) for x in cookies]:
-                if k.strip() != "cppwd":
-                    continue
-
-                v = unescape_cookie(v)
-                if v in self.auth.iuser:
-                    self.uname = self.auth.iuser[v]
-
-                break
-
-        if self.uname:
-            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
-            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
-
         # split req into vpath + uparam
         uparam = {}
         if "?" not in self.req:
@@ -130,13 +120,33 @@ class HttpCli(object):
                 else:
                     uparam[k.lower()] = False
 
+        self.ouparam = {k: v for k, v in uparam.items()}
+
+        cookies = self.headers.get("cookie") or {}
+        if cookies:
+            cookies = [x.split("=", 1) for x in cookies.split(";") if "=" in x]
+            cookies = {k.strip(): unescape_cookie(v) for k, v in cookies}
+            for kc, ku in [["cppwd", "pw"], ["b", "b"]]:
+                if kc in cookies and ku not in uparam:
+                    uparam[ku] = cookies[kc]
+
         self.uparam = uparam
+        self.cookies = cookies
         self.vpath = unquotep(vpath)
 
+        pwd = uparam.get("pw")
+        self.uname = self.auth.iuser.get(pwd, "*")
+        if self.uname:
+            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
+            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
+
         ua = self.headers.get("user-agent", "")
-        if ua.startswith("rclone/"):
+        self.is_rclone = ua.startswith("rclone/")
+        if self.is_rclone:
             uparam["raw"] = False
             uparam["dots"] = False
+            uparam["b"] = False
+            cookies["b"] = False
 
         try:
             if self.mode in ["GET", "HEAD"]:
@@ -153,17 +163,21 @@ class HttpCli(object):
         except Pebkac as ex:
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
-                self.keepalive = self._check_nonfatal(ex)
-                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
+                if not self._check_nonfatal(ex):
+                    self.keepalive = False
+
+                self.log("{}\033[0m, {}".format(str(ex), self.vpath), 3)
+                msg = "<pre>{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                self.reply(msg.encode("utf-8", "replace"), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
 
     def send_headers(self, length, status=200, mime=None, headers={}):
-        response = ["HTTP/1.1 {} {}".format(status, HTTPCODE[status])]
+        response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
 
         if length is not None:
-            response.append("Content-Length: " + str(length))
+            response.append("Content-Length: " + unicode(length))
 
         # close if unknown length, otherwise take client's preference
         response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
@@ -172,10 +186,8 @@ class HttpCli(object):
         self.out_headers.update(headers)
 
         # default to utf8 html if no content-type is set
-        try:
-            mime = mime or self.out_headers["Content-Type"]
-        except KeyError:
-            mime = "text/html; charset=UTF-8"
+        if not mime:
+            mime = self.out_headers.get("Content-Type", "text/html; charset=UTF-8")
 
         self.out_headers["Content-Type"] = mime
 
@@ -204,6 +216,43 @@ class HttpCli(object):
         self.log(body.rstrip())
         self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
+    def urlq(self, add={}, rm=[]):
+        """
+        generates url query based on uparam (b, pw, all others)
+        removing anything in rm, adding pairs in add
+        """
+
+        if self.is_rclone:
+            return ""
+
+        kv = {
+            k: v
+            for k, v in self.uparam.items()
+            if k not in rm and self.cookies.get(k) != v
+        }
+        kv.update(add)
+        if not kv:
+            return ""
+
+        r = ["{}={}".format(k, quotep(v)) if v else k for k, v in kv.items()]
+        return "?" + "&amp;".join(r)
+
+    def redirect(self, vpath, suf="", msg="aight", flavor="go to", use302=False):
+        html = self.j2(
+            "msg",
+            h2='<a href="/{}">{} /{}</a>'.format(
+                quotep(vpath) + suf, flavor, html_escape(vpath, crlf=True) + suf
+            ),
+            pre=msg,
+            click=True,
+        ).encode("utf-8", "replace")
+
+        if use302:
+            h = {"Location": "/" + vpath, "Cache-Control": "no-cache"}
+            self.reply(html, status=302, headers=h)
+        else:
+            self.reply(html)
+
     def handle_get(self):
         logmsg = "{:4} {}".format(self.mode, self.req)
 
@@ -226,23 +275,27 @@ class HttpCli(object):
             return self.tx_tree()
 
         # conditional redirect to single volumes
-        if self.vpath == "" and not self.uparam:
+        if self.vpath == "" and not self.ouparam:
             nread = len(self.rvol)
             nwrite = len(self.wvol)
             if nread + nwrite == 1 or (self.rvol == self.wvol and nread == 1):
                 if nread == 1:
-                    self.vpath = self.rvol[0]
+                    vpath = self.rvol[0]
                 else:
-                    self.vpath = self.wvol[0]
+                    vpath = self.wvol[0]
 
-                self.absolute_urls = True
+                if self.vpath != vpath:
+                    self.redirect(vpath, flavor="redirecting to", use302=True)
+                    return True
 
-        # go home if verboten
         self.readable, self.writable = self.conn.auth.vfs.can_access(
             self.vpath, self.uname
         )
         if not self.readable and not self.writable:
-            self.log("inaccessible: [{}]".format(self.vpath))
+            if self.vpath:
+                self.log("inaccessible: [{}]".format(self.vpath))
+                raise Pebkac(404)
+
             self.uparam = {"h": False}
 
         if "h" in self.uparam:
@@ -312,8 +365,19 @@ class HttpCli(object):
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
                 for buf in reader:
-                    buf = buf.decode("utf-8", "replace")
-                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
+                    orig = buf.decode("utf-8", "replace")
+                    m = "urlform_raw {} @ {}\n  {}\n"
+                    self.log(m.format(len(orig), self.vpath, orig))
+                    try:
+                        plain = unquote(buf.replace(b"+", b" "))
+                        plain = plain.decode("utf-8", "replace")
+                        if buf.startswith(b"msg="):
+                            plain = plain[4:]
+
+                        m = "urlform_dec {} @ {}\n  {}\n"
+                        self.log(m.format(len(plain), self.vpath, plain))
+                    except Exception as ex:
+                        self.log(repr(ex))
 
             if "get" in opt:
                 return self.handle_get()
@@ -388,8 +452,30 @@ class HttpCli(object):
         if act == "tput":
             return self.handle_text_upload()
 
+        if act == "zip":
+            return self.handle_zip_post()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
+    def handle_zip_post(self):
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                break
+
+        if v is None:
+            raise Pebkac(422, "need zip or tar keyword")
+
+        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        items = self.parser.require("files", 1024 * 1024)
+        if not items:
+            raise Pebkac(422, "need files list")
+
+        items = items.replace("\r", "").split("\n")
+        items = [unquotep(x) for x in items if items]
+
+        return self.tx_zip(k, v, vn, rem, items, self.args.ed)
+
     def handle_post_json(self):
         try:
             remains = int(self.headers["content-length"])
@@ -417,15 +503,18 @@ class HttpCli(object):
         if "srch" in self.uparam or "srch" in body:
             return self.handle_search(body)
 
-        # prefer this over undot; no reason to allow traversion
-        if "/" in body["name"]:
-            raise Pebkac(400, "folders verboten")
-
         # up2k-php compat
         for k in "chunkpit.php", "handshake.php":
             if self.vpath.endswith(k):
                 self.vpath = self.vpath[: -len(k)]
 
+        sub = None
+        name = undot(body["name"])
+        if "/" in name:
+            sub, name = name.rsplit("/", 1)
+            self.vpath = "/".join([self.vpath, sub]).strip("/")
+            body["name"] = name
+
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
 
         body["vtop"] = vfs.vpath
@@ -434,12 +523,22 @@ class HttpCli(object):
         body["addr"] = self.ip
         body["vcfg"] = vfs.flags
 
-        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
-        response = x.get()
-        response = json.dumps(response)
+        if sub:
+            try:
+                dst = os.path.join(vfs.realpath, rem)
+                os.makedirs(dst)
+            except:
+                if not os.path.isdir(dst):
+                    raise Pebkac(400, "some file got your folder name")
 
-        self.log(response)
-        self.reply(response.encode("utf-8"), mime="application/json")
+        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
+        ret = x.get()
+        if sub:
+            ret["name"] = "/".join([sub, ret["name"]])
+
+        ret = json.dumps(ret)
+        self.log(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
         return True
 
     def handle_search(self, body):
@@ -450,19 +549,30 @@ class HttpCli(object):
 
         idx = self.conn.get_u2idx()
         t0 = time.time()
+        if idx.p_end:
+            penalty = 0.7
+            t_idle = t0 - idx.p_end
+            if idx.p_dur > 0.7 and t_idle < penalty:
+                m = "rate-limit ({:.1f} sec), cost {:.2f}, idle {:.2f}"
+                raise Pebkac(429, m.format(penalty, idx.p_dur, t_idle))
+
         if "srch" in body:
             # search by up2k hashlist
             vbody = copy.deepcopy(body)
             vbody["hash"] = len(vbody["hash"])
             self.log("qj: " + repr(vbody))
             hits = idx.fsearch(vols, body)
-            self.log("q#: {} ({:.2f}s)".format(repr(hits), time.time() - t0))
-            taglist = []
+            msg = repr(hits)
+            taglist = {}
         else:
             # search by query params
             self.log("qj: " + repr(body))
             hits, taglist = idx.search(vols, body)
-            self.log("q#: {} ({:.2f}s)".format(len(hits), time.time() - t0))
+            msg = len(hits)
+
+        idx.p_end = time.time()
+        idx.p_dur = idx.p_end - t0
+        self.log("q#: {} ({:.2f}s)".format(msg, idx.p_dur))
 
         order = []
         cfg = self.args.mte.split(",")
@@ -521,7 +631,7 @@ class HttpCli(object):
             if len(cstart) > 1 and path != os.devnull:
                 self.log(
                     "clone {} to {}".format(
-                        cstart[0], " & ".join(str(x) for x in cstart[1:])
+                        cstart[0], " & ".join(unicode(x) for x in cstart[1:])
                     )
                 )
                 ofs = 0
@@ -545,7 +655,7 @@ class HttpCli(object):
             self.loud_reply(x, status=500)
             return False
 
-        if not WINDOWS and num_left == 0:
+        if not ANYWIN and num_left == 0:
             times = (int(time.time()), int(lastmod))
             self.log("no more chunks, setting times {}".format(times))
             try:
@@ -564,13 +674,16 @@ class HttpCli(object):
 
         if pwd in self.auth.iuser:
             msg = "login ok"
+            dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
+            exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
         else:
             msg = "naw dude"
             pwd = "x"  # nosec
+            exp = "Fri, 15 Aug 1997 01:00:00 GMT"
 
-        h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
-        html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
-        self.reply(html.encode("utf-8"), headers=h)
+        ck = "cppwd={}; Path=/; Expires={}; SameSite=Lax".format(pwd, exp)
+        html = self.j2("msg", h1=msg, h2='<a href="/">ack</a>', redir="/")
+        self.reply(html.encode("utf-8"), headers={"Set-Cookie": ck})
         return True
 
     def handle_mkdir(self):
@@ -599,13 +712,7 @@ class HttpCli(object):
                 raise Pebkac(500, "mkdir failed, check the logs")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        esc_paths = [quotep(vpath), html_escape(vpath)]
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
-            pre="aight",
-            click=True,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(vpath)
         return True
 
     def handle_new_md(self):
@@ -632,14 +739,7 @@ class HttpCli(object):
                 f.write(b"`GRUNNUR`\n")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
-                quotep(vpath), html_escape(vpath)
-            ),
-            pre="aight",
-            click=True,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(vpath, "?edit")
         return True
 
     def handle_plain_upload(self):
@@ -658,7 +758,9 @@ class HttpCli(object):
 
                 if p_file and not nullwrite:
                     fdir = os.path.join(vfs.realpath, rem)
-                    fname = sanitize_fn(p_file)
+                    fname = sanitize_fn(
+                        p_file, bad=[".prologue.html", ".epilogue.html"]
+                    )
 
                     if not os.path.isdir(fsenc(fdir)):
                         raise Pebkac(404, "that folder does not exist")
@@ -697,7 +799,7 @@ class HttpCli(object):
                     raise
 
         except Pebkac as ex:
-            errmsg = str(ex)
+            errmsg = unicode(ex)
 
         td = max(0.1, time.time() - t0)
         sz_total = sum(x[0] for x in files)
@@ -738,13 +840,7 @@ class HttpCli(object):
                     ).encode("utf-8")
                 )
 
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">return to /{}</a>'.format(
-                quotep(self.vpath), html_escape(self.vpath)
-            ),
-            pre=msg,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(self.vpath, msg=msg, flavor="return to")
         self.parser.drop()
         return True
 
@@ -844,13 +940,14 @@ class HttpCli(object):
         return True
 
     def _chk_lastmod(self, file_ts):
+        date_fmt = "%a, %d %b %Y %H:%M:%S GMT"
         file_dt = datetime.utcfromtimestamp(file_ts)
-        file_lastmod = file_dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
+        file_lastmod = file_dt.strftime(date_fmt)
 
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
-                cli_dt = time.strptime(cli_lastmod, "%a, %d %b %Y %H:%M:%S GMT")
+                cli_dt = time.strptime(cli_lastmod, date_fmt)
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
             except Exception as ex:
@@ -1006,7 +1103,7 @@ class HttpCli(object):
             mime=guess_mime(req_path)[0] or "application/octet-stream",
         )
 
-        logmsg += str(status) + logtail
+        logmsg += unicode(status) + logtail
 
         if self.mode == "HEAD" or not do_send:
             self.log(logmsg)
@@ -1020,35 +1117,93 @@ class HttpCli(object):
                 remains = sendfile_py(lower, upper, f, self.s)
 
         if remains > 0:
-            logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
+            logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
 
         spd = self._spd((upper - lower) - remains)
         self.log("{},  {}".format(logmsg, spd))
         return ret
 
+    def tx_zip(self, fmt, uarg, vn, rem, items, dots):
+        if self.args.no_zip:
+            raise Pebkac(400, "not enabled")
+
+        logmsg = "{:4} {} ".format("", self.req)
+        self.keepalive = False
+
+        if not uarg:
+            uarg = ""
+
+        if fmt == "tar":
+            mime = "application/x-tar"
+            packer = StreamTar
+        else:
+            mime = "application/zip"
+            packer = StreamZip
+
+        fn = items[0] if items and items[0] else self.vpath
+        if fn:
+            fn = fn.rstrip("/").split("/")[-1]
+        else:
+            fn = self.headers.get("host", "hey")
+
+        afn = "".join(
+            [x if x in (string.ascii_letters + string.digits) else "_" for x in fn]
+        )
+
+        bascii = unicode(string.ascii_letters + string.digits).encode("utf-8")
+        ufn = fn.encode("utf-8", "xmlcharrefreplace")
+        if PY2:
+            ufn = [unicode(x) if x in bascii else "%{:02x}".format(ord(x)) for x in ufn]
+        else:
+            ufn = [
+                chr(x).encode("utf-8")
+                if x in bascii
+                else "%{:02x}".format(x).encode("ascii")
+                for x in ufn
+            ]
+        ufn = b"".join(ufn).decode("ascii")
+
+        cdis = "attachment; filename=\"{}.{}\"; filename*=UTF-8''{}.{}"
+        cdis = cdis.format(afn, fmt, ufn, fmt)
+        self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis})
+
+        fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
+        # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
+        bsent = 0
+        for buf in bgen.gen():
+            if not buf:
+                break
+
+            try:
+                self.s.sendall(buf)
+                bsent += len(buf)
+            except:
+                logmsg += " \033[31m" + unicode(bsent) + "\033[0m"
+                break
+
+        spd = self._spd(bsent)
+        self.log("{},  {}".format(logmsg, spd))
+        return True
+
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
-        if "edit2" in self.uparam:
-            html_path = "web/mde.html"
-            template = self.conn.tpl_mde
-        else:
-            html_path = "web/md.html"
-            template = self.conn.tpl_md
 
-        html_path = os.path.join(E.mod, html_path)
+        tpl = "mde" if "edit2" in self.uparam else "md"
+        html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
+        template = self.j2(tpl)
 
         st = os.stat(fsenc(fs_path))
-        # sz_md = st.st_size
         ts_md = st.st_mtime
 
         st = os.stat(fsenc(html_path))
         ts_html = st.st_mtime
 
-        # TODO dont load into memory ;_;
-        #   (trivial fix, count the &'s)
-        with open(fsenc(fs_path), "rb") as f:
-            md = f.read().replace(b"&", b"&amp;")
-            sz_md = len(md)
+        sz_md = 0
+        for buf in yieldfile(fs_path):
+            sz_md += len(buf)
+            for c, v in [[b"&", 4], [b"<", 3], [b">", 3]]:
+                sz_md += (len(buf) - len(buf.replace(c, b""))) * v
 
         file_ts = max(ts_md, ts_html)
         file_lastmod, do_send = self._chk_lastmod(file_ts)
@@ -1056,38 +1211,46 @@ class HttpCli(object):
         self.out_headers["Cache-Control"] = "no-cache"
         status = 200 if do_send else 304
 
+        boundary = "\roll\tide"
         targs = {
             "edit": "edit" in self.uparam,
-            "title": html_escape(self.vpath),
+            "title": html_escape(self.vpath, crlf=True),
             "lastmod": int(ts_md * 1000),
             "md_plug": "true" if self.args.emp else "false",
             "md_chk_rate": self.args.mcr,
-            "md": "",
+            "md": boundary,
         }
-        sz_html = len(template.render(**targs).encode("utf-8"))
-        self.send_headers(sz_html + sz_md, status)
+        html = template.render(**targs).encode("utf-8")
+        html = html.split(boundary.encode("utf-8"))
+        if len(html) != 2:
+            raise Exception("boundary appears in " + html_path)
 
-        logmsg += str(status)
+        self.send_headers(sz_md + len(html[0]) + len(html[1]), status)
+
+        logmsg += unicode(status)
         if self.mode == "HEAD" or not do_send:
             self.log(logmsg)
             return True
 
-        # TODO jinja2 can stream this right?
-        targs["md"] = md.decode("utf-8", "replace")
-        html = template.render(**targs).encode("utf-8")
         try:
-            self.s.sendall(html)
+            self.s.sendall(html[0])
+            for buf in yieldfile(fs_path):
+                self.s.sendall(html_bescape(buf))
+
+            self.s.sendall(html[1])
+
         except:
             self.log(logmsg + " \033[31md/c\033[0m")
             return False
 
-        self.log(logmsg + " " + str(len(html)))
+        self.log(logmsg + " " + unicode(len(html)))
         return True
 
     def tx_mounts(self):
+        suf = self.urlq(rm=["h"])
         rvol = [x + "/" if x else x for x in self.rvol]
         wvol = [x + "/" if x else x for x in self.wvol]
-        html = self.conn.tpl_mounts.render(this=self, rvol=rvol, wvol=wvol)
+        html = self.j2("splash", this=self, rvol=rvol, wvol=wvol, url_suf=suf)
         self.reply(html.encode("utf-8"))
         return True
 
@@ -1115,7 +1278,8 @@ class HttpCli(object):
         excl = None
         if target:
             excl, target = (target.split("/", 1) + [""])[:2]
-            ret["k" + excl] = self.gen_tree("/".join([top, excl]).strip("/"), target)
+            sub = self.gen_tree("/".join([top, excl]).strip("/"), target)
+            ret["k" + quotep(excl)] = sub
 
         try:
             vn, rem = self.auth.vfs.get(top, self.uname, True, False)
@@ -1136,7 +1300,7 @@ class HttpCli(object):
             vfs_ls = exclude_dotfiles(vfs_ls)
 
         for fn in [x for x in vfs_ls if x != excl]:
-            dirs.append(fn)
+            dirs.append(quotep(fn))
 
         for x in vfs_virt.keys():
             if x != excl:
@@ -1155,7 +1319,7 @@ class HttpCli(object):
                 else:
                     vpath += "/" + node
 
-                vpnodes.append([quotep(vpath) + "/", html_escape(node)])
+                vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
         vn, rem = self.auth.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
@@ -1166,6 +1330,94 @@ class HttpCli(object):
             # print(abspath)
             raise Pebkac(404)
 
+        srv_info = []
+
+        try:
+            if not self.args.nih:
+                srv_info.append(unicode(socket.gethostname()).split(".")[0])
+        except:
+            self.log("#wow #whoa")
+
+        try:
+            # some fuses misbehave
+            if not self.args.nid:
+                if WINDOWS:
+                    bfree = ctypes.c_ulonglong(0)
+                    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
+                        ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+                    )
+                    srv_info.append(humansize(bfree.value) + " free")
+                else:
+                    sv = os.statvfs(abspath)
+                    free = humansize(sv.f_frsize * sv.f_bfree, True)
+                    total = humansize(sv.f_frsize * sv.f_blocks, True)
+
+                    srv_info.append(free + " free")
+                    srv_info.append(total)
+        except:
+            pass
+
+        srv_info = "</span> /// <span>".join(srv_info)
+
+        perms = []
+        if self.readable:
+            perms.append("read")
+        if self.writable:
+            perms.append("write")
+
+        url_suf = self.urlq()
+        is_ls = "ls" in self.uparam
+        ts = ""  # "?{}".format(time.time())
+
+        tpl = "browser"
+        if "b" in self.uparam:
+            tpl = "browser2"
+
+        logues = ["", ""]
+        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
+            fn = os.path.join(abspath, fn)
+            if os.path.exists(fsenc(fn)):
+                with open(fsenc(fn), "rb") as f:
+                    logues[n] = f.read().decode("utf-8")
+
+        ls_ret = {
+            "dirs": [],
+            "files": [],
+            "taglist": [],
+            "srvinf": srv_info,
+            "perms": perms,
+            "logues": logues,
+        }
+        j2a = {
+            "vdir": quotep(self.vpath),
+            "vpnodes": vpnodes,
+            "files": [],
+            "ts": ts,
+            "perms": json.dumps(perms),
+            "taglist": [],
+            "tag_order": [],
+            "have_up2k_idx": ("e2d" in vn.flags),
+            "have_tags_idx": ("e2t" in vn.flags),
+            "have_zip": (not self.args.no_zip),
+            "have_b_u": (self.writable and self.uparam.get("b") == "u"),
+            "url_suf": url_suf,
+            "logues": logues,
+            "title": html_escape(self.vpath, crlf=True),
+            "srv_info": srv_info,
+        }
+        if not self.readable:
+            if is_ls:
+                ret = json.dumps(ls_ret)
+                self.reply(ret.encode("utf-8", "replace"), mime="application/json")
+                return True
+
+            if not os.path.isdir(fsenc(abspath)):
+                raise Pebkac(404)
+
+            html = self.j2(tpl, **j2a)
+            self.reply(html.encode("utf-8", "replace"))
+            return True
+
         if not os.path.isdir(fsenc(abspath)):
             if abspath.endswith(".md") and "raw" not in self.uparam:
                 return self.tx_md(abspath)
@@ -1175,6 +1427,11 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                return self.tx_zip(k, v, vn, rem, [], self.args.ed)
+
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
@@ -1204,8 +1461,6 @@ class HttpCli(object):
         if rem == ".hist":
             hidden = ["up2k."]
 
-        is_ls = "ls" in self.uparam
-
         icur = None
         if "e2t" in vn.flags:
             idx = self.conn.get_u2idx()
@@ -1235,11 +1490,14 @@ class HttpCli(object):
 
             is_dir = stat.S_ISDIR(inf.st_mode)
             if is_dir:
-                margin = "DIR"
                 href += "/"
+                if self.args.no_zip:
+                    margin = "DIR"
+                else:
+                    margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
-                    base, html_escape(hist[fn][2], quote=True), hist[fn][0]
+                    base, html_escape(hist[fn][2], quote=True, crlf=True), hist[fn][0]
                 )
             else:
                 margin = "-"
@@ -1275,99 +1533,44 @@ class HttpCli(object):
             del f["rd"]
             if icur:
                 q = "select w from up where rd = ? and fn = ?"
-                r = icur.execute(q, (rd, fn)).fetchone()
+                try:
+                    r = icur.execute(q, (rd, fn)).fetchone()
+                except:
+                    args = s3enc(idx.mem_cur, rd, fn)
+                    r = icur.execute(q, args).fetchone()
+
+                tags = {}
+                f["tags"] = tags
+
                 if not r:
                     continue
 
                 w = r[0][:16]
-                tags = {}
                 q = "select k, v from mt where w = ? and k != 'x'"
                 for k, v in icur.execute(q, (w,)):
                     taglist[k] = True
                     tags[k] = v
 
-                f["tags"] = tags
-
         if icur:
-            taglist = [k for k in self.args.mte.split(",") if k in taglist]
+            taglist = [k for k in vn.flags.get("mte", "").split(",") if k in taglist]
             for f in dirs:
                 f["tags"] = {}
 
-        srv_info = []
-
-        try:
-            if not self.args.nih:
-                srv_info.append(str(socket.gethostname()).split(".")[0])
-        except:
-            self.log("#wow #whoa")
-            pass
-
-        try:
-            # some fuses misbehave
-            if not self.args.nid:
-                if WINDOWS:
-                    bfree = ctypes.c_ulonglong(0)
-                    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
-                        ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
-                    )
-                    srv_info.append(humansize(bfree.value) + " free")
-                else:
-                    sv = os.statvfs(abspath)
-                    free = humansize(sv.f_frsize * sv.f_bfree, True)
-                    total = humansize(sv.f_frsize * sv.f_blocks, True)
-
-                    srv_info.append(free + " free")
-                    srv_info.append(total)
-        except:
-            pass
-
-        srv_info = "</span> /// <span>".join(srv_info)
-
-        perms = []
-        if self.readable:
-            perms.append("read")
-        if self.writable:
-            perms.append("write")
-
-        logues = ["", ""]
-        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
-            fn = os.path.join(abspath, fn)
-            if os.path.exists(fsenc(fn)):
-                with open(fsenc(fn), "rb") as f:
-                    logues[n] = f.read().decode("utf-8")
-
         if is_ls:
             [x.pop(k) for k in ["name", "dt"] for y in [dirs, files] for x in y]
-            ret = {
-                "dirs": dirs,
-                "files": files,
-                "srvinf": srv_info,
-                "perms": perms,
-                "logues": logues,
-                "taglist": taglist,
-            }
-            ret = json.dumps(ret)
+            ls_ret["dirs"] = dirs
+            ls_ret["files"] = files
+            ls_ret["taglist"] = taglist
+            ret = json.dumps(ls_ret)
             self.reply(ret.encode("utf-8", "replace"), mime="application/json")
             return True
 
-        ts = ""
-        # ts = "?{}".format(time.time())
+        j2a["files"] = dirs + files
+        j2a["logues"] = logues
+        j2a["taglist"] = taglist
+        if "mte" in vn.flags:
+            j2a["tag_order"] = json.dumps(vn.flags["mte"].split(","))
 
-        dirs.extend(files)
-
-        html = self.conn.tpl_browser.render(
-            vdir=quotep(self.vpath),
-            vpnodes=vpnodes,
-            files=dirs,
-            ts=ts,
-            perms=json.dumps(perms),
-            taglist=taglist,
-            tag_order=json.dumps(self.args.mte.split(",")),
-            have_up2k_idx=("e2d" in vn.flags),
-            have_tags_idx=("e2t" in vn.flags),
-            logues=logues,
-            title=html_escape(self.vpath),
-            srv_info=srv_info,
-        )
+        html = self.j2(tpl, **j2a)
         self.reply(html.encode("utf-8", "replace"))
         return True

copyparty/httpconn.py

@@ -12,23 +12,6 @@ try:
 except:
     HAVE_SSL = False
 
-try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * {} -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-""".format(
-            os.path.basename(sys.executable)
-        )
-    )
-    sys.exit(1)
-
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
@@ -57,14 +40,6 @@ class HttpConn(object):
         self.log_func = hsrv.log
         self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
-
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -81,8 +56,8 @@ class HttpConn(object):
     def respath(self, res_name):
         return os.path.join(E.mod, "web", res_name)
 
-    def log(self, msg):
-        self.log_func(self.log_src, msg)
+    def log(self, msg, c=0):
+        self.log_func(self.log_src, msg, c)
 
     def get_u2idx(self):
         if not self.u2idx:
@@ -112,7 +87,9 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
-                self.log(err)
+                if method:
+                    self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 
@@ -129,7 +106,7 @@ class HttpConn(object):
 
         if is_https:
             if self.sr:
-                self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
+                self.log("TODO: cannot do https in jython", c="1;31")
                 return
 
             self.log_src = self.log_src.replace("[36m", "[35m")
@@ -180,7 +157,7 @@ class HttpConn(object):
                     pass
 
                 else:
-                    self.log("\033[35mhandshake\033[0m " + em)
+                    self.log("handshake\033[0m " + em, c=5)
 
                 return
 

copyparty/httpsrv.py

@@ -2,10 +2,28 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
 from .__init__ import E, MACOS
 from .httpconn import HttpConn
 from .authsrv import AuthSrv
@@ -30,6 +48,13 @@ class HttpSrv(object):
         self.workload_thr_alive = False
         self.auth = AuthSrv(self.args, self.log)
 
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+        }
+
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
             self.cert_path = cert_path
@@ -38,7 +63,9 @@ class HttpSrv(object):
 
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log("%s %s" % addr, "\033[1;30m|%sC-cthr\033[0m" % ("-" * 5,))
+        if self.args.log_conn:
+            self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+
         thr = threading.Thread(target=self.thr_client, args=(sck, addr))
         thr.daemon = True
         thr.start()
@@ -66,11 +93,15 @@ class HttpSrv(object):
                 thr.start()
 
         try:
-            self.log("%s %s" % addr, "\033[1;30m|%sC-crun\033[0m" % ("-" * 6,))
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+
             cli.run()
 
         finally:
-            self.log("%s %s" % addr, "\033[1;30m|%sC-cdone\033[0m" % ("-" * 7,))
+            if self.args.log_conn:
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+
             try:
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
@@ -78,7 +109,8 @@ class HttpSrv(object):
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "\033[1;30mshut({}): {}\033[0m".format(sck.fileno(), ex),
+                        "shut({}): {}".format(sck.fileno(), ex),
+                        c="1;30",
                     )
                 if ex.errno not in [10038, 10054, 107, 57, 9]:
                     # 10038 No longer considered a socket

copyparty/mtag.py

@@ -8,7 +8,10 @@ import shutil
 import subprocess as sp
 
 from .__init__ import PY2, WINDOWS
-from .util import fsenc, fsdec
+from .util import fsenc, fsdec, REKOBO_LKEY
+
+if not PY2:
+    unicode = str
 
 
 class MTag(object):
@@ -18,13 +21,14 @@ class MTag(object):
         self.prefer_mt = False
         mappings = args.mtm
         self.backend = "ffprobe" if args.no_mutagen else "mutagen"
+        or_ffprobe = " or ffprobe"
 
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
                 import mutagen
             except:
-                self.log("\033[33mcould not load mutagen, trying ffprobe instead")
+                self.log("could not load mutagen, trying ffprobe instead", c=3)
                 self.backend = "ffprobe"
 
         if self.backend == "ffprobe":
@@ -32,7 +36,7 @@ class MTag(object):
             self.prefer_mt = True
             # about 20x slower
             if PY2:
-                cmd = ["ffprobe", "-version"]
+                cmd = [b"ffprobe", b"-version"]
                 try:
                     sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
                 except:
@@ -41,9 +45,15 @@ class MTag(object):
                 if not shutil.which("ffprobe"):
                     self.usable = False
 
+            if self.usable and WINDOWS and sys.version_info < (3, 8):
+                self.usable = False
+                or_ffprobe = " or python >= 3.8"
+                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                self.log(msg, c=1)
+
         if not self.usable:
-            msg = "\033[31mneed mutagen or ffprobe to read media tags so please run this:\n  {} -m pip install --user mutagen \033[0m"
-            self.log(msg.format(os.path.basename(sys.executable)))
+            msg = "need mutagen{} to read media tags so please run this:\n  {} -m pip install --user mutagen"
+            self.log(msg.format(or_ffprobe, os.path.basename(sys.executable)), c=1)
             return
 
         # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -115,8 +125,8 @@ class MTag(object):
         }
         # self.get = self.compare
 
-    def log(self, msg):
-        self.log_func("mtag", msg)
+    def log(self, msg, c=0):
+        self.log_func("mtag", msg, c)
 
     def normalize_tags(self, ret, md):
         for k, v in dict(md).items():
@@ -133,7 +143,7 @@ class MTag(object):
                 ret[mk] = [pref, v[0]]
 
         # take first value
-        ret = {k: str(v[1]).strip() for k, v in ret.items()}
+        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
 
         # track 3/7 => track 3
         for k, v in ret.items():
@@ -141,6 +151,12 @@ class MTag(object):
                 v = v.split("/")[0].strip().lstrip("0")
                 ret[k] = v or 0
 
+        # normalize key notation to rkeobo
+        okey = ret.get("key")
+        if okey:
+            key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
+            ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
+
         return ret
 
     def compare(self, abspath):
@@ -206,7 +222,7 @@ class MTag(object):
         return self.normalize_tags(ret, md)
 
     def get_ffprobe(self, abspath):
-        cmd = ["ffprobe", "-hide_banner", "--", fsenc(abspath)]
+        cmd = [b"ffprobe", b"-hide_banner", b"--", fsenc(abspath)]
         p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
         r = p.communicate()
         txt = r[1].decode("utf-8", "replace")
@@ -215,7 +231,7 @@ class MTag(object):
         """
         note:
           tags which contain newline will be truncated on first \n,
-          ffmpeg emits \n and spacepads the : to align visually
+          ffprobe emits \n and spacepads the : to align visually
         note:
           the Stream ln always mentions Audio: if audio
           the Stream ln usually has kb/s, is more accurate
@@ -285,9 +301,7 @@ class MTag(object):
                             sec *= 60
                             sec += int(f)
                     except:
-                        self.log(
-                            "\033[33minvalid timestr from ffmpeg: [{}]".format(tstr)
-                        )
+                        self.log("invalid timestr from ffprobe: [{}]".format(tstr), c=3)
 
                 ret[".dur"] = sec
                 m = ptn_br1.search(ln)
@@ -304,3 +318,30 @@ class MTag(object):
         ret = {k: [0, v] for k, v in ret.items()}
 
         return self.normalize_tags(ret, md)
+
+    def get_bin(self, parsers, abspath):
+        pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
+        pypath = str(os.pathsep.join(pypath))
+        env = os.environ.copy()
+        env["PYTHONPATH"] = pypath
+
+        ret = {}
+        for tagname, (binpath, timeout) in parsers.items():
+            try:
+                cmd = [sys.executable, binpath, abspath]
+                args = {"env": env, "timeout": timeout}
+
+                if WINDOWS:
+                    args["creationflags"] = 0x4000
+                else:
+                    cmd = ["nice"] + cmd
+
+                cmd = [fsenc(x) for x in cmd]
+                v = sp.check_output(cmd, **args).strip()
+                if v:
+                    ret[tagname] = v.decode("utf-8")
+            except:
+                pass
+
+        return ret

copyparty/star.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen)
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/sutil.py

@@ -0,0 +1,25 @@
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/svchub.py

@@ -9,7 +9,6 @@ from datetime import datetime, timedelta
 import calendar
 
 from .__init__ import PY2, WINDOWS, MACOS, VT100
-from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
 from .util import mp
@@ -66,10 +65,10 @@ class SvcHub(object):
             self.broker.shutdown()
             print("nailed it")
 
-    def _log_disabled(self, src, msg):
+    def _log_disabled(self, src, msg, c=0):
         pass
 
-    def _log_enabled(self, src, msg):
+    def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
         with self.log_mutex:
             now = time.time()
@@ -92,6 +91,13 @@ class SvcHub(object):
                     msg = self.ansi_re.sub("", msg)
                 if "\033" in src:
                     src = self.ansi_re.sub("", src)
+            elif c:
+                if isinstance(c, int):
+                    msg = "\033[3{}m{}".format(c, msg)
+                elif "\033" not in c:
+                    msg = "\033[{}m{}\033[0m".format(c, msg)
+                else:
+                    msg = "{}{}\033[0m".format(c, msg)
 
             ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             msg = fmt.format(ts, src, msg)

copyparty/szip.py

@@ -0,0 +1,271 @@
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, ok="/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -68,22 +68,29 @@ class TcpSrv(object):
             self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
 
         while True:
-            self.log("tcpsrv", "\033[1;30m|%sC-ncli\033[0m" % ("-" * 1,))
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
+
             if self.num_clients.v >= self.args.nc:
                 time.sleep(0.1)
                 continue
 
-            self.log("tcpsrv", "\033[1;30m|%sC-acc1\033[0m" % ("-" * 2,))
+            if self.args.log_conn:
+                self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
+
             ready, _, _ = select.select(self.srv, [], [])
             for srv in ready:
                 sck, addr = srv.accept()
                 sip, sport = srv.getsockname()
-                self.log(
-                    "%s %s" % addr,
-                    "\033[1;30m|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
-                        "-" * 3, sip, sport % 8, sport
-                    ),
-                )
+                if self.args.log_conn:
+                    self.log(
+                        "%s %s" % addr,
+                        "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                            "-" * 3, sip, sport % 8, sport
+                        ),
+                        c="1;30",
+                    )
+
                 self.num_clients.add()
                 self.hub.broker.put(False, "httpconn", sck, addr)
 

copyparty/u2idx.py

@@ -1,10 +1,13 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
+import time
+import threading
 from datetime import datetime
 
-from .util import u8safe
+from .util import u8safe, s3dec, html_escape, Pebkac
 from .up2k import up2k_wark_from_hashlist
 
 
@@ -19,15 +22,21 @@ class U2idx(object):
     def __init__(self, args, log_func):
         self.args = args
         self.log_func = log_func
+        self.timeout = args.srch_time
 
         if not HAVE_SQLITE3:
             self.log("could not load sqlite3; searchign wqill be disabled")
             return
 
         self.cur = {}
+        self.mem_cur = sqlite3.connect(":memory:")
+        self.mem_cur.execute(r"create table a (b text)")
 
-    def log(self, msg):
-        self.log_func("u2idx", msg)
+        self.p_end = None
+        self.p_dur = 0
+
+    def log(self, msg, c=0):
+        self.log_func("u2idx", msg, c)
 
     def fsearch(self, vols, body):
         """search by up2k hashlist"""
@@ -37,7 +46,14 @@ class U2idx(object):
         fsize = body["size"]
         fhash = body["hash"]
         wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
-        return self.run_query(vols, "w = ?", [wark], "", [])[0]
+
+        uq = "substr(w,1,16) = ? and w = ?"
+        uv = [wark[:16], wark]
+
+        try:
+            return self.run_query(vols, uq, uv, {})[0]
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
 
     def get_cur(self, ptop):
         cur = self.cur.get(ptop)
@@ -67,17 +83,64 @@ class U2idx(object):
 
         uq, uv = _sqlize(qobj)
 
-        tq = ""
-        tv = []
         qobj = {}
         if "tags" in body:
             _conv_txt(qobj, body, "tags", "mt.v")
-            tq, tv = _sqlize(qobj)
 
-        return self.run_query(vols, uq, uv, tq, tv)
+        if "adv" in body:
+            _conv_adv(qobj, body, "adv")
 
-    def run_query(self, vols, uq, uv, tq, tv):
-        self.log("qs: {} {} ,  {} {}".format(uq, repr(uv), tq, repr(tv)))
+        try:
+            return self.run_query(vols, uq, uv, qobj)
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
+
+    def run_query(self, vols, uq, uv, targs):
+        self.log("qs: {} {} ,  {}".format(uq, repr(uv), repr(targs)))
+
+        done_flag = []
+        self.active_id = "{:.6f}_{}".format(
+            time.time(), threading.current_thread().ident
+        )
+        thr = threading.Thread(
+            target=self.terminator,
+            args=(
+                self.active_id,
+                done_flag,
+            ),
+        )
+        thr.daemon = True
+        thr.start()
+
+        if not targs:
+            if not uq:
+                q = "select * from up"
+                v = ()
+            else:
+                q = "select * from up where " + uq
+                v = tuple(uv)
+        else:
+            q = "select up.* from up"
+            keycmp = "substr(up.w,1,16)"
+            where = []
+            v = []
+            ctr = 0
+            for tq, tv in sorted(targs.items()):
+                ctr += 1
+                tq = tq.split("\n")[0]
+                keycmp2 = "mt{}.w".format(ctr)
+                q += " inner join mt mt{} on {} = {}".format(ctr, keycmp, keycmp2)
+                keycmp = keycmp2
+                where.append(tq.replace("mt.", keycmp[:-1]))
+                v.append(tv)
+
+            if uq:
+                where.append(uq)
+                v.extend(uv)
+
+            q += " where " + (" and ".join(where))
+
+        # self.log("q2: {} {}".format(q, repr(v)))
 
         ret = []
         lim = 1000
@@ -87,18 +150,7 @@ class U2idx(object):
             if not cur:
                 continue
 
-            if not tq:
-                if not uq:
-                    q = "select * from up"
-                    v = ()
-                else:
-                    q = "select * from up where " + uq
-                    v = tuple(uv)
-            else:
-                # naive assumption: tags first
-                q = "select up.* from up inner join mt on substr(up.w,1,16) = mt.w where {}"
-                q = q.format(" and ".join([tq, uq]) if uq else tq)
-                v = tuple(tv + uv)
+            self.active_cur = cur
 
             sret = []
             c = cur.execute(q, v)
@@ -108,6 +160,9 @@ class U2idx(object):
                 if lim <= 0:
                     break
 
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
                 rp = os.path.join(vtop, rd, fn).replace("\\", "/")
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
 
@@ -115,17 +170,35 @@ class U2idx(object):
                 w = hit["w"]
                 del hit["w"]
                 tags = {}
-                q = "select k, v from mt where w = ? and k != 'x'"
-                for k, v in cur.execute(q, (w,)):
+                q2 = "select k, v from mt where w = ? and k != 'x'"
+                for k, v2 in cur.execute(q2, (w,)):
                     taglist[k] = True
-                    tags[k] = v
+                    tags[k] = v2
 
                 hit["tags"] = tags
 
             ret.extend(sret)
 
+        done_flag.append(True)
+        self.active_id = None
+
+        # undupe hits from multiple metadata keys
+        if len(ret) > 1:
+            ret = [ret[0]] + [
+                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+            ]
+
         return ret, list(taglist.keys())
 
+    def terminator(self, identifier, done_flag):
+        for _ in range(self.timeout):
+            time.sleep(1)
+            if done_flag:
+                return
+
+        if identifier == self.active_id:
+            self.active_cur.connection.interrupt()
+
 
 def _open(ptop):
     db_path = os.path.join(ptop, ".hist", "up2k.db")
@@ -181,6 +254,23 @@ def _conv_txt(q, body, k, sql):
         q[qk + "\n" + v] = u8safe(v)
 
 
+def _conv_adv(q, body, k):
+    ptn = re.compile(r"^(\.?[a-z]+) *(==?|!=|<=?|>=?) *(.*)$")
+
+    parts = body[k].split(" ")
+    parts = [x.strip() for x in parts if x.strip()]
+
+    for part in parts:
+        m = ptn.match(part)
+        if not m:
+            p = html_escape(part)
+            raise Pebkac(400, "invalid argument [" + p + "]")
+
+        k, op, v = m.groups()
+        qk = "mt.k = '{}' and mt.v {} ?".format(k, op)
+        q[qk + "\n" + v] = u8safe(v)
+
+
 def _sqlize(qobj):
     keys = []
     values = []

copyparty/up2k.py

@@ -13,9 +13,10 @@ import base64
 import hashlib
 import threading
 import traceback
+import subprocess as sp
 from copy import deepcopy
 
-from .__init__ import WINDOWS
+from .__init__ import WINDOWS, ANYWIN
 from .util import (
     Pebkac,
     Queue,
@@ -25,9 +26,10 @@ from .util import (
     sanitize_fn,
     ren_open,
     atomic_move,
-    w8b64enc,
-    w8b64dec,
+    s3enc,
+    s3dec,
     statdir,
+    s2hms,
 )
 from .mtag import MTag
 from .authsrv import AuthSrv
@@ -64,16 +66,20 @@ class Up2k(object):
         self.flags = {}
         self.cur = {}
         self.mtag = None
-        self.n_mtag_thr_alive = 0
-        self.n_mtag_tags_added = 0
+        self.pending_tags = None
 
         self.mem_cur = None
+        self.sqlite_ver = None
+        self.no_expr_idx = False
         if HAVE_SQLITE3:
             # mojibake detector
             self.mem_cur = self._orz(":memory:")
             self.mem_cur.execute(r"create table a (b text)")
+            self.sqlite_ver = tuple([int(x) for x in sqlite3.sqlite_version.split(".")])
+            if self.sqlite_ver < (3, 9):
+                self.no_expr_idx = True
 
-        if WINDOWS:
+        if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
             thr = threading.Thread(target=self._lastmodder)
@@ -87,7 +93,7 @@ class Up2k(object):
             self.log("could not initialize sqlite3, will use in-memory registry only")
 
         # this is kinda jank
-        auth = AuthSrv(self.args, self.log, False)
+        auth = AuthSrv(self.args, self.log_func, False)
         have_e2d = self.init_indexes(auth)
 
         if have_e2d:
@@ -95,39 +101,21 @@ class Up2k(object):
             thr.daemon = True
             thr.start()
 
-            thr = threading.Thread(target=self._tagger)
-            thr.daemon = True
-            thr.start()
-
             thr = threading.Thread(target=self._hasher)
             thr.daemon = True
             thr.start()
 
-    def log(self, msg):
-        self.log_func("up2k", msg + "\033[K")
+            if self.mtag:
+                thr = threading.Thread(target=self._tagger)
+                thr.daemon = True
+                thr.start()
 
-    def w8enc(self, rd, fn):
-        ret = []
-        for v in [rd, fn]:
-            try:
-                self.mem_cur.execute("select * from a where b = ?", (v,))
-                ret.append(v)
-            except:
-                ret.append("//" + w8b64enc(v))
-                # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+                thr = threading.Thread(target=self._run_all_mtp)
+                thr.daemon = True
+                thr.start()
 
-        return tuple(ret)
-
-    def w8dec(self, rd, fn):
-        ret = []
-        for k, v in [["d", rd], ["f", fn]]:
-            if v.startswith("//"):
-                ret.append(w8b64dec(v[2:]))
-                # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
-            else:
-                ret.append(v)
-
-        return tuple(ret)
+    def log(self, msg, c=0):
+        self.log_func("up2k", msg + "\033[K", c)
 
     def _vis_job_progress(self, job):
         perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
@@ -141,24 +129,46 @@ class Up2k(object):
 
         return ret
 
+    def _expr_idx_filter(self, flags):
+        if not self.no_expr_idx:
+            return False, flags
+
+        ret = {k: v for k, v in flags.items() if not k.startswith("e2t")}
+        if ret.keys() == flags.keys():
+            return False, flags
+
+        return True, ret
+
     def init_indexes(self, auth):
         self.pp = ProgressPrinter()
         vols = auth.vfs.all_vols.values()
         t0 = time.time()
         have_e2d = False
 
+        if self.no_expr_idx:
+            modified = False
+            for vol in vols:
+                m, f = self._expr_idx_filter(vol.flags)
+                if m:
+                    vol.flags = f
+                    modified = True
+
+            if modified:
+                msg = "disabling -e2t because your sqlite belongs in a museum"
+                self.log(msg, c=3)
+
         live_vols = []
         for vol in vols:
             try:
                 os.listdir(vol.realpath)
                 live_vols.append(vol)
             except:
-                self.log("\033[31mcannot access " + vol.realpath)
+                self.log("cannot access " + vol.realpath, c=1)
 
         vols = live_vols
 
         need_mtag = False
-        for vol in auth.vfs.all_vols.values():
+        for vol in vols:
             if "e2t" in vol.flags:
                 need_mtag = True
 
@@ -204,8 +214,8 @@ class Up2k(object):
         self.log(msg.format(len(vols), time.time() - t0))
 
         if needed_mutagen:
-            msg = "\033[31mcould not read tags because no backends are available (mutagen or ffprobe)\033[0m"
-            self.log(msg)
+            msg = "could not read tags because no backends are available (mutagen or ffprobe)"
+            self.log(msg, c=1)
 
         return have_e2d
 
@@ -214,6 +224,18 @@ class Up2k(object):
             if ptop in self.registry:
                 return None
 
+            _, flags = self._expr_idx_filter(flags)
+
+            ft = "\033[0;32m{}{:.0}"
+            ff = "\033[0;35m{}{:.0}"
+            fv = "\033[0;36m{}:\033[1;30m{}"
+            a = [
+                (ft if v is True else ff if v is False else fv).format(k, str(v))
+                for k, v in flags.items()
+            ]
+            if a:
+                self.log(" ".join(sorted(a)) + "\033[0m")
+
             reg = {}
             path = os.path.join(ptop, ".hist", "up2k.snap")
             if "e2d" in flags and os.path.exists(path):
@@ -267,9 +289,12 @@ class Up2k(object):
         dbw = [reg[0], 0, time.time()]
         self.pp.n = next(dbw[0].execute("select count(w) from up"))[0]
 
-        # can be symlink so don't `and d.startswith(top)``
-        excl = set([d.realpath for d in all_vols if d != vol])
-        n_add = self._build_dir(dbw, top, excl, top)
+        excl = [
+            vol.realpath + "/" + d.vpath[len(vol.vpath) :].lstrip("/")
+            for d in all_vols
+            if d != vol and (d.vpath.startswith(vol.vpath + "/") or not vol.vpath)
+        ]
+        n_add = self._build_dir(dbw, top, set(excl), top)
         n_rm = self._drop_lost(dbw[0], top)
         if dbw[1]:
             self.log("commit {} new files".format(dbw[1]))
@@ -312,7 +337,7 @@ class Up2k(object):
                 try:
                     c = dbw[0].execute(sql, (rd, fn))
                 except:
-                    c = dbw[0].execute(sql, self.w8enc(rd, fn))
+                    c = dbw[0].execute(sql, s3enc(self.mem_cur, rd, fn))
 
                 in_db = list(c.fetchall())
                 if in_db:
@@ -366,7 +391,7 @@ class Up2k(object):
         for dwark, dts, dsz, drd, dfn in c:
             nchecked += 1
             if drd.startswith("//") or dfn.startswith("//"):
-                drd, dfn = self.w8dec(drd, dfn)
+                drd, dfn = s3dec(drd, dfn)
 
             abspath = os.path.join(top, drd, dfn)
             # almost zero overhead dw
@@ -430,19 +455,7 @@ class Up2k(object):
 
             mpool = False
             if self.mtag.prefer_mt and not self.args.no_mtag_mt:
-                # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
-                # both do crazy runahead so lets reinvent another wheel
-                nw = os.cpu_count()
-                if not self.n_mtag_thr_alive:
-                    msg = 'using {} cores for tag reader "{}"'
-                    self.log(msg.format(nw, self.mtag.backend))
-
-                self.n_mtag_thr_alive = nw
-                mpool = Queue(nw)
-                for _ in range(nw):
-                    thr = threading.Thread(target=self._tag_thr, args=(mpool,))
-                    thr.daemon = True
-                    thr.start()
+                mpool = self._start_mpool()
 
             c2 = cur.connection.cursor()
             c3 = cur.connection.cursor()
@@ -453,16 +466,21 @@ class Up2k(object):
                 if c2.execute(q, (w[:16],)).fetchone():
                     continue
 
+                if "mtp" in flags:
+                    q = "insert into mt values (?,'t:mtp','a')"
+                    c2.execute(q, (w[:16],))
+
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
                 abspath = os.path.join(ptop, rd, fn)
                 self.pp.msg = "c{} {}".format(n_left, abspath)
-                args = c3, entags, w, abspath
+                args = [entags, w, abspath]
                 if not mpool:
-                    n_tags = self._tag_file(*args)
+                    n_tags = self._tag_file(c3, *args)
                 else:
-                    mpool.put(args)
-                    with self.mutex:
-                        n_tags = self.n_mtag_tags_added
-                        self.n_mtag_tags_added = 0
+                    mpool.put(["mtag"] + args)
+                    n_tags = len(self._flush_mpool(c3))
 
                 n_add += n_tags
                 n_buf += n_tags
@@ -474,41 +492,257 @@ class Up2k(object):
                     last_write = time.time()
                     n_buf = 0
 
-            if self.n_mtag_thr_alive:
-                mpool.join()
-                for _ in range(self.n_mtag_thr_alive):
-                    mpool.put(None)
+            self._stop_mpool(mpool, c3)
 
             c3.close()
             c2.close()
 
         return n_add, n_rm, True
 
+    def _flush_mpool(self, wcur):
+        with self.mutex:
+            ret = []
+            for x in self.pending_tags:
+                self._tag_file(wcur, *x)
+                ret.append(x[1])
+
+            self.pending_tags = []
+            return ret
+
+    def _run_all_mtp(self):
+        t0 = time.time()
+        self.mtp_force = {}
+        self.mtp_parsers = {}
+        for ptop, flags in self.flags.items():
+            if "mtp" in flags:
+                self._run_one_mtp(ptop)
+
+        td = time.time() - t0
+        msg = "mtp finished in {:.2f} sec ({})"
+        self.log(msg.format(td, s2hms(td, True)))
+
+    def _run_one_mtp(self, ptop):
+        db_path = os.path.join(ptop, ".hist", "up2k.db")
+        sz0 = os.path.getsize(db_path) // 1024
+
+        entags = self.entags[ptop]
+
+        force = {}
+        timeout = {}
+        parsers = {}
+        for parser in self.flags[ptop]["mtp"]:
+            orig = parser
+            tag, parser = parser.split("=", 1)
+            if tag not in entags:
+                continue
+
+            while True:
+                try:
+                    bp = os.path.expanduser(parser)
+                    if os.path.exists(bp):
+                        parsers[tag] = [bp, timeout.get(tag, 30)]
+                        break
+                except:
+                    pass
+
+                try:
+                    arg, parser = parser.split(",", 1)
+                    arg = arg.lower()
+
+                    if arg == "f":
+                        force[tag] = True
+                        continue
+
+                    if arg.startswith("t"):
+                        timeout[tag] = int(arg[1:])
+                        continue
+
+                    raise Exception()
+
+                except:
+                    self.log("invalid argument: " + orig, 1)
+                    return
+
+        self.mtp_force[ptop] = force
+        self.mtp_parsers[ptop] = parsers
+
+        q = "select count(w) from mt where k = 't:mtp'"
+        with self.mutex:
+            cur = self.cur[ptop]
+            cur = cur.connection.cursor()
+            wcur = cur.connection.cursor()
+            n_left = cur.execute(q).fetchone()[0]
+
+        mpool = self._start_mpool()
+        batch_sz = mpool.maxsize * 3
+        t_prev = time.time()
+        n_prev = n_left
+        n_done = 0
+        to_delete = {}
+        in_progress = {}
+        while True:
+            with self.mutex:
+                q = "select w from mt where k = 't:mtp' limit ?"
+                warks = cur.execute(q, (batch_sz,)).fetchall()
+                warks = [x[0] for x in warks]
+                jobs = []
+                for w in warks:
+                    q = "select rd, fn from up where substr(w,1,16)=? limit 1"
+                    rd, fn = cur.execute(q, (w,)).fetchone()
+                    rd, fn = s3dec(rd, fn)
+                    abspath = os.path.join(ptop, rd, fn)
+
+                    q = "select k from mt where w = ?"
+                    have = cur.execute(q, (w,)).fetchall()
+                    have = [x[0] for x in have]
+
+                    if ".dur" not in have and ".dur" in entags:
+                        # skip non-audio
+                        to_delete[w] = True
+                        n_left -= 1
+                        continue
+
+                    if w in in_progress:
+                        continue
+
+                    task_parsers = {
+                        k: v for k, v in parsers.items() if k in force or k not in have
+                    }
+                    jobs.append([task_parsers, None, w, abspath])
+                    in_progress[w] = True
+
+            done = self._flush_mpool(wcur)
+
+            with self.mutex:
+                for w in done:
+                    to_delete[w] = True
+                    in_progress.pop(w)
+                    n_done += 1
+
+                for w in to_delete.keys():
+                    q = "delete from mt where w = ? and k = 't:mtp'"
+                    cur.execute(q, (w,))
+
+                to_delete = {}
+
+            if not warks:
+                break
+
+            if not jobs:
+                continue
+
+            try:
+                now = time.time()
+                s = ((now - t_prev) / (n_prev - n_left)) * n_left
+                h, s = divmod(s, 3600)
+                m, s = divmod(s, 60)
+                n_prev = n_left
+                t_prev = now
+            except:
+                h = 1
+                m = 1
+
+            msg = "mtp: {} done, {} left, eta {}h {:02d}m"
+            with self.mutex:
+                msg = msg.format(n_done, n_left, int(h), int(m))
+                self.log(msg, c=6)
+
+            for j in jobs:
+                n_left -= 1
+                mpool.put(j)
+
+            with self.mutex:
+                cur.connection.commit()
+
+        done = self._stop_mpool(mpool, wcur)
+        with self.mutex:
+            for w in done:
+                q = "delete from mt where w = ? and k = 't:mtp'"
+                cur.execute(q, (w,))
+
+            cur.connection.commit()
+            if n_done:
+                self.vac(cur, db_path, n_done, 0, sz0)
+
+            wcur.close()
+            cur.close()
+
+    def _start_mpool(self):
+        # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
+        # both do crazy runahead so lets reinvent another wheel
+        nw = os.cpu_count() if hasattr(os, "cpu_count") else 4
+        if self.pending_tags is None:
+            self.log("using {}x {}".format(nw, self.mtag.backend))
+            self.pending_tags = []
+
+        mpool = Queue(nw)
+        for _ in range(nw):
+            thr = threading.Thread(target=self._tag_thr, args=(mpool,))
+            thr.daemon = True
+            thr.start()
+
+        return mpool
+
+    def _stop_mpool(self, mpool, wcur):
+        if not mpool:
+            return
+
+        for _ in range(mpool.maxsize):
+            mpool.put(None)
+
+        mpool.join()
+        done = self._flush_mpool(wcur)
+        return done
+
     def _tag_thr(self, q):
         while True:
             task = q.get()
             if not task:
-                break
+                q.task_done()
+                return
 
             try:
-                write_cur, entags, wark, abspath = task
-                tags = self.mtag.get(abspath)
+                parser, entags, wark, abspath = task
+                if parser == "mtag":
+                    tags = self.mtag.get(abspath)
+                else:
+                    tags = self.mtag.get_bin(parser, abspath)
+                    vtags = [
+                        "\033[36m{} \033[33m{}".format(k, v) for k, v in tags.items()
+                    ]
+                    self.log("{}\033[0m [{}]".format(" ".join(vtags), abspath))
+
                 with self.mutex:
-                    n = self._tag_file(write_cur, entags, wark, abspath, tags)
-                    self.n_mtag_tags_added += n
+                    self.pending_tags.append([entags, wark, abspath, tags])
             except:
-                with self.mutex:
-                    self.n_mtag_thr_alive -= 1
-                raise
-            finally:
-                q.task_done()
+                ex = traceback.format_exc()
+                if parser == "mtag":
+                    parser = self.mtag.backend
+
+                msg = "{} failed to read tags from {}:\n{}"
+                self.log(msg.format(parser, abspath, ex), c=3)
+
+            q.task_done()
 
     def _tag_file(self, write_cur, entags, wark, abspath, tags=None):
-        tags = tags or self.mtag.get(abspath)
-        tags = {k: v for k, v in tags.items() if k in entags}
+        if tags is None:
+            tags = self.mtag.get(abspath)
+
+        if entags:
+            tags = {k: v for k, v in tags.items() if k in entags}
+            if not tags:
+                # indicate scanned without tags
+                tags = {"x": 0}
+
         if not tags:
-            # indicate scanned without tags
-            tags = {"x": 0}
+            return 0
+
+        for k in tags.keys():
+            q = "delete from mt where w = ? and ({})".format(
+                " or ".join(["k = ?"] * len(tags))
+            )
+            args = [wark[:16]] + list(tags.keys())
+            write_cur.execute(q, tuple(args))
 
         ret = 0
         for k, v in tags.items():
@@ -520,6 +754,7 @@ class Up2k(object):
 
     def _orz(self, db_path):
         return sqlite3.connect(db_path, check_same_thread=False).cursor()
+        # x.set_trace_callback(trace)
 
     def _open_db(self, db_path):
         existed = os.path.exists(db_path)
@@ -615,8 +850,12 @@ class Up2k(object):
                 except:
                     pass
 
+        idx = r"create index up_w on up(substr(w,1,16))"
+        if self.no_expr_idx:
+            idx = r"create index up_w on up(w)"
+
         for cmd in [
-            r"create index up_w on up(substr(w,1,16))",
+            idx,
             r"create table mt (w text, k text, v int)",
             r"create index mt_w on mt(w)",
             r"create index mt_k on mt(k)",
@@ -652,7 +891,7 @@ class Up2k(object):
             if cj["ptop"] not in self.registry:
                 raise Pebkac(410, "location unavailable")
 
-        cj["name"] = sanitize_fn(cj["name"])
+        cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
         wark = self._get_wark(cj)
         now = time.time()
@@ -661,12 +900,17 @@ class Up2k(object):
             cur = self.cur.get(cj["ptop"], None)
             reg = self.registry[cj["ptop"]]
             if cur:
-                q = r"select * from up where substr(w,1,16) = ? and w = ?"
-                argv = (wark[:16], wark)
+                if self.no_expr_idx:
+                    q = r"select * from up where w = ?"
+                    argv = (wark,)
+                else:
+                    q = r"select * from up where substr(w,1,16) = ? and w = ?"
+                    argv = (wark[:16], wark)
+
                 cur = cur.execute(q, argv)
                 for _, dtime, dsize, dp_dir, dp_fn in cur:
                     if dp_dir.startswith("//") or dp_fn.startswith("//"):
-                        dp_dir, dp_fn = self.w8dec(dp_dir, dp_fn)
+                        dp_dir, dp_fn = s3dec(dp_dir, dp_fn)
 
                     dp_abs = os.path.join(cj["ptop"], dp_dir, dp_fn).replace("\\", "/")
                     # relying on path.exists to return false on broken symlinks
@@ -788,8 +1032,13 @@ class Up2k(object):
                 raise OSError()
             elif fs1 == fs2:
                 # same fs; make symlink as relative as possible
-                nsrc = src.replace("\\", "/").split("/")
-                ndst = dst.replace("\\", "/").split("/")
+                v = []
+                for p in [src, dst]:
+                    if WINDOWS:
+                        p = p.replace("\\", "/")
+                    v.append(p.split("/"))
+
+                nsrc, ndst = v
                 nc = 0
                 for a, b in zip(nsrc, ndst):
                     if a != b:
@@ -797,7 +1046,8 @@ class Up2k(object):
                     nc += 1
                 if nc > 1:
                     lsrc = nsrc[nc:]
-                    lsrc = "../" * (len(lsrc) - 1) + "/".join(lsrc)
+                    hops = len(ndst[nc:]) - 1
+                    lsrc = "../" * hops + "/".join(lsrc)
             os.symlink(fsenc(lsrc), fsenc(ldst))
         except (AttributeError, OSError) as ex:
             self.log("cannot symlink; creating copy: " + repr(ex))
@@ -807,6 +1057,8 @@ class Up2k(object):
         with self.mutex:
             job = self.registry[ptop].get(wark, None)
             if not job:
+                known = " ".join([x for x in self.registry[ptop].keys()])
+                self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
@@ -846,8 +1098,9 @@ class Up2k(object):
 
             atomic_move(src, dst)
 
-            if WINDOWS:
-                self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))])
+            if ANYWIN:
+                a = [dst, job["size"], (int(time.time()), int(job["lmod"]))]
+                self.lastmod_q.put(a)
 
             # legit api sware 2 me mum
             if self.idx_wark(
@@ -882,7 +1135,7 @@ class Up2k(object):
         try:
             db.execute(sql, (rd, fn))
         except:
-            db.execute(sql, self.w8enc(rd, fn))
+            db.execute(sql, s3enc(self.mem_cur, rd, fn))
 
     def db_add(self, db, wark, rd, fn, ts, sz):
         sql = "insert into up values (?,?,?,?,?)"
@@ -890,7 +1143,7 @@ class Up2k(object):
         try:
             db.execute(sql, v)
         except:
-            rd, fn = self.w8enc(rd, fn)
+            rd, fn = s3enc(self.mem_cur, rd, fn)
             v = (wark, ts, sz, rd, fn)
             db.execute(sql, v)
 
@@ -919,7 +1172,7 @@ class Up2k(object):
         ret = []
         with open(path, "rb", 512 * 1024) as f:
             while fsz > 0:
-                self.pp.msg = "{} MB".format(int(fsz / 1024 / 1024))
+                self.pp.msg = "{} MB, {}".format(int(fsz / 1024 / 1024), path)
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
                 fsz -= rem
@@ -948,6 +1201,17 @@ class Up2k(object):
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f:
             f, job["tnam"] = f["orz"]
+            if (
+                ANYWIN
+                and self.args.sparse
+                and self.args.sparse * 1024 * 1024 <= job["size"]
+            ):
+                fp = os.path.join(pdir, job["tnam"])
+                try:
+                    sp.check_call(["fsutil", "sparse", "setflag", fp])
+                except:
+                    self.log("could not sparse [{}]".format(fp), 3)
+
             f.seek(job["size"] - 1)
             f.write(b"e")
 
@@ -959,13 +1223,19 @@ class Up2k(object):
 
             # self.log("lmod: got {}".format(len(ready)))
             time.sleep(5)
-            for path, times in ready:
+            for path, sz, times in ready:
                 self.log("lmod: setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:
                     self.log("lmod: failed to utime ({}, {})".format(path, times))
 
+                if self.args.sparse and self.args.sparse * 1024 * 1024 <= sz:
+                    try:
+                        sp.check_call(["fsutil", "sparse", "setflag", path, "0"])
+                    except:
+                        self.log("could not unsparse [{}]".format(path), 3)
+
     def _snapshot(self):
         persist_interval = 30  # persist unfinished uploads index every 30 sec
         discard_interval = 21600  # drop unfinished uploads after 6 hours inactivity
@@ -1029,24 +1299,37 @@ class Up2k(object):
     def _tagger(self):
         while True:
             ptop, wark, rd, fn = self.tagq.get()
+            if "e2t" not in self.flags[ptop]:
+                continue
+
             abspath = os.path.join(ptop, rd, fn)
-            self.log("tagging " + abspath)
+            tags = self.mtag.get(abspath)
+            ntags1 = len(tags)
+            if self.mtp_parsers.get(ptop, {}):
+                parser = {
+                    k: v
+                    for k, v in self.mtp_parsers[ptop].items()
+                    if k in self.mtp_force[ptop] or k not in tags
+                }
+                tags.update(self.mtag.get_bin(parser, abspath))
+
             with self.mutex:
                 cur = self.cur[ptop]
                 if not cur:
-                    self.log("\033[31mno cursor to write tags with??")
+                    self.log("no cursor to write tags with??", c=1)
                     continue
 
+                # TODO is undef if vol 404 on startup
                 entags = self.entags[ptop]
                 if not entags:
-                    self.log("\033[33mno entags okay.jpg")
+                    self.log("no entags okay.jpg", c=3)
                     continue
 
-                if "e2t" in self.flags[ptop]:
-                    self._tag_file(cur, entags, wark, abspath)
-
+                self._tag_file(cur, entags, wark, abspath, tags)
                 cur.connection.commit()
 
+            self.log("tagged {} ({}+{})".format(abspath, ntags1, len(tags) - ntags1))
+
     def _hasher(self):
         while True:
             ptop, rd, fn = self.hashq.get()

copyparty/util.py

@@ -10,12 +10,13 @@ import select
 import struct
 import hashlib
 import platform
+import traceback
 import threading
 import mimetypes
 import contextlib
 import subprocess as sp  # nosec
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -48,6 +49,7 @@ HTTPCODE = {
     200: "OK",
     204: "No Content",
     206: "Partial Content",
+    302: "Found",
     304: "Not Modified",
     400: "Bad Request",
     403: "Forbidden",
@@ -56,11 +58,58 @@ HTTPCODE = {
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
 }
 
 
+IMPLICATIONS = [
+    ["e2dsa", "e2ds"],
+    ["e2ds", "e2d"],
+    ["e2tsr", "e2ts"],
+    ["e2ts", "e2t"],
+    ["e2t", "e2d"],
+]
+
+
+REKOBO_KEY = {
+    v: ln.split(" ", 1)[0]
+    for ln in """
+1B 6d B
+2B 7d Gb F#
+3B 8d Db C#
+4B 9d Ab G#
+5B 10d Eb D#
+6B 11d Bb A#
+7B 12d F
+8B 1d C
+9B 2d G
+10B 3d D
+11B 4d A
+12B 5d E
+1A 6m Abm G#m
+2A 7m Ebm D#m
+3A 8m Bbm A#m
+4A 9m Fm
+5A 10m Cm
+6A 11m Gm
+7A 12m Dm
+8A 1m Am
+9A 2m Em
+10A 3m Bm
+11A 4m Gbm F#m
+12A 5m Dbm C#m
+""".strip().split(
+        "\n"
+    )
+    for v in ln.strip().split(" ")[1:]
+    if v
+}
+
+REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
+
+
 class Counter(object):
     def __init__(self, v=0):
         self.v = v
@@ -119,19 +168,51 @@ class ProgressPrinter(threading.Thread):
                 continue
 
             msg = self.msg
-            m = " {}\033[K\r".format(msg)
-            try:
-                print(m, end="")
-            except UnicodeEncodeError:
-                try:
-                    print(m.encode("utf-8", "replace").decode(), end="")
-                except:
-                    print(m.encode("ascii", "replace").decode(), end="")
+            uprint(" {}\033[K\r".format(msg))
 
         print("\033[K", end="")
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
 
 
+def uprint(msg):
+    try:
+        print(msg, end="")
+    except UnicodeEncodeError:
+        try:
+            print(msg.encode("utf-8", "replace").decode(), end="")
+        except:
+            print(msg.encode("ascii", "replace").decode(), end="")
+
+
+def nuprint(msg):
+    uprint("{}\n".format(msg))
+
+
+def rice_tid():
+    tid = threading.current_thread().ident
+    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
+
+
+def trace(*args, **kwargs):
+    t = time.time()
+    stack = "".join(
+        "\033[36m{}\033[33m{}".format(x[0].split(os.sep)[-1][:-3], x[1])
+        for x in traceback.extract_stack()[3:-1]
+    )
+    parts = ["{:.6f}".format(t), rice_tid(), stack]
+
+    if args:
+        parts.append(repr(args))
+
+    if kwargs:
+        parts.append(repr(kwargs))
+
+    msg = "\033[0m ".join(parts)
+    # _tracebuf.append(msg)
+    nuprint(msg)
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -470,6 +551,16 @@ def get_spd(nbyte, t0, t=None):
     return "{} \033[0m{}/s\033[0m".format(s1, s2)
 
 
+def s2hms(s, optional_h=False):
+    s = int(s)
+    h, s = divmod(s, 3600)
+    m, s = divmod(s, 60)
+    if not h and optional_h:
+        return "{}:{:02}".format(m, s)
+
+    return "{}:{:02}:{:02}".format(h, m, s)
+
+
 def undot(path):
     ret = []
     for node in path.split("/"):
@@ -486,11 +577,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
-    fn = fn.replace("\\", "/").split("/")[-1]
+def sanitize_fn(fn, ok="", bad=[]):
+    if "/" not in ok:
+        fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -500,15 +592,16 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
+        ]
+        for bad, good in [x for x in remap if x[0] not in ok]:
             fn = fn.replace(bad, good)
 
-        bad = ["con", "prn", "aux", "nul"]
+        bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):
             bad += "com{0} lpt{0}".format(n).split(" ")
 
-        if fn.lower() in bad:
-            fn = "_" + fn
+    if fn.lower() in bad:
+        fn = "_" + fn
 
     return fn.strip()
 
@@ -524,17 +617,24 @@ def exclude_dotfiles(filepaths):
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 
@@ -597,6 +697,31 @@ else:
     fsdec = w8dec
 
 
+def s3enc(mem_cur, rd, fn):
+    ret = []
+    for v in [rd, fn]:
+        try:
+            mem_cur.execute("select * from a where b = ?", (v,))
+            ret.append(v)
+        except:
+            ret.append("//" + w8b64enc(v))
+            # self.log("mojien/{} [{}] {}".format(k, v, ret[-1][2:]))
+
+    return tuple(ret)
+
+
+def s3dec(rd, fn):
+    ret = []
+    for k, v in [["d", rd], ["f", fn]]:
+        if v.startswith("//"):
+            ret.append(w8b64dec(v[2:]))
+            # self.log("mojide/{} [{}] {}".format(k, ret[-1], v[2:]))
+        else:
+            ret.append(v)
+
+    return tuple(ret)
+
+
 def atomic_move(src, dst):
     if not PY2:
         os.replace(src, dst)
@@ -665,6 +790,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()
@@ -734,7 +869,8 @@ def statdir(logger, scandir, lstat, top):
                     try:
                         yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
                     except Exception as ex:
-                        logger("scan-stat: {} @ {}".format(repr(ex), fsdec(fh.path)))
+                        msg = "scan-stat: \033[36m{} @ {}"
+                        logger(msg.format(repr(ex), fsdec(fh.path)))
         else:
             src = "listdir"
             fun = os.lstat if lstat else os.stat
@@ -743,9 +879,11 @@ def statdir(logger, scandir, lstat, top):
                 try:
                     yield [fsdec(name), fun(abspath)]
                 except Exception as ex:
-                    logger("list-stat: {} @ {}".format(repr(ex), fsdec(abspath)))
+                    msg = "list-stat: \033[36m{} @ {}"
+                    logger(msg.format(repr(ex), fsdec(abspath)))
+
     except Exception as ex:
-        logger("{}: {} @ {}".format(src, repr(ex), top))
+        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
 
 
 def unescape_cookie(orig):
@@ -802,7 +940,11 @@ def chkcmd(*argv):
 def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
-        return struct.unpack(b"I", f.read(4))[0]
+        rv = f.read(4)
+        try:
+            return struct.unpack(b"I", rv)[0]
+        except:
+            return struct.unpack("I", rv)[0]
 
 
 def py_desc():
@@ -812,7 +954,11 @@ def py_desc():
     if ofs > 0:
         py_ver = py_ver[:ofs]
 
-    bitness = struct.calcsize(b"P") * 8
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
     host_os = platform.system()
     compiler = platform.python_compiler()
 

copyparty/web/browser.css

@@ -6,7 +6,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -42,12 +42,8 @@ body {
 #path #entree {
 	margin-left: -.7em;
 }
-#treetab {
-	display: none;
-}
 #files {
 	border-spacing: 0;
-	margin-top: 2em;
 	z-index: 1;
 	position: relative;
 }
@@ -55,11 +51,10 @@ body {
 	display: block;
 	padding: .3em 0;
 }
-#files[ts] tbody div a {
+#files tbody div a {
 	color: #f5a;
 }
-a,
-#files[ts] tbody div a:last-child {
+a, #files tbody div a:last-child {
 	color: #fc5;
 	padding: .2em;
 	text-decoration: none;
@@ -73,7 +68,7 @@ a,
 	color: #999;
 	font-weight: normal;
 }
-#files tr+tr:hover {
+#files tr:hover {
 	background: #1c1c1c;
 }
 #files thead th {
@@ -95,8 +90,6 @@ a,
 #files td {
 	margin: 0;
 	padding: 0 .5em;
-}
-#files td {
 	border-bottom: 1px solid #111;
 }
 #files td+td+td {
@@ -158,6 +151,15 @@ a,
 .logue {
 	padding: .2em 1.5em;
 }
+.logue:empty {
+	display: none;
+}
+#pro.logue {
+	margin-bottom: .8em;
+}
+#epi.logue {
+	margin: .8em 0;
+}
 #srv_info {
 	opacity: .5;
 	font-size: .8em;
@@ -178,6 +180,21 @@ a,
 	color: #840;
 	text-shadow: 0 0 .3em #b80;
 }
+#files tbody tr.sel td {
+	color: #fff;
+	background: #925;
+	border-color: #c37;
+}
+#files tr.sel a {
+	color: #fff;
+}
+#files tr.sel a.play {
+	color: #fc5;
+}
+#files tr.sel a.play.act {
+	color: #fff;
+	text-shadow: 0 0 1px #fff;
+}
 #blocked {
 	position: fixed;
 	top: 0;
@@ -234,7 +251,7 @@ a,
 	height: 100%;
 	background: #3c3c3c;
 }
-#wtoggle {
+#wtico {
 	cursor: url(/.cpr/dd/1.png), pointer;
 	animation: cursor 500ms infinite;
 }
@@ -264,6 +281,33 @@ a,
 	padding: .2em 0 0 .07em;
 	color: #fff;
 }
+#wzip {
+	display: none;
+	margin-right: .3em;
+	padding-right: .3em;
+	border-right: .1em solid #555;
+}
+#wtoggle,
+#wtoggle * {
+	line-height: 1em;
+}
+#wtoggle.sel {
+	width: 6.4em;
+}
+#wtoggle.sel #wzip {
+	display: inline-block;
+}
+#wtoggle.sel #wzip a {
+	font-size: .4em;
+	padding: 0 .3em;
+	margin: -.3em .2em;
+	position: relative;
+	display: inline-block;
+}
+#wtoggle.sel #wzip #selzip {
+	top: -.6em;
+	padding: .4em .3em;
+}
 #barpos,
 #barbuf {
 	position: absolute;
@@ -307,10 +351,10 @@ a,
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
-@media (min-width: 90em) {
+@media (min-width: 80em) {
 	#barpos,
 	#barbuf {
-		width: calc(100% - 24em);
+		width: calc(100% - 21em);
 		left: 9.8em;
 		top: .7em;
 		height: 1.6em;
@@ -320,6 +364,9 @@ a,
 		bottom: -3.2em;
 		height: 3.2em;
 	}
+	#pvol {
+		max-width: 9em;
+	}
 }
 
 
@@ -401,7 +448,7 @@ input[type="checkbox"]:checked+label {
 
 
 
-#op_search table {
+#srch_form {
 	border: 1px solid #3a3a3a;
 	box-shadow: 0 0 1em #222 inset;
 	background: #2d2d2d;
@@ -410,14 +457,25 @@ input[type="checkbox"]:checked+label {
 	margin-bottom: 0;
 	padding: 0 .5em .5em 0;
 }
+#srch_form table {
+	display: inline-block;
+}
 #srch_form td {
 	padding: .6em .6em;
 }
+#srch_form td:first-child {
+	width: 3em;
+	padding-right: .2em;
+	text-align: right;
+}
 #op_search input {
 	margin: 0;
 }
 #srch_q {
 	white-space: pre;
+	color: #f80;
+	height: 1em;
+	margin: .2em 0 -1em 1.6em;
 }
 #files td div span {
 	color: #fff;
@@ -440,16 +498,40 @@ input[type="checkbox"]:checked+label {
 	border-collapse: collapse;
 	width: 100%;
 }
-#files td div a:last-child {
-	width: 100%;
-}
-#tree,
-#treefiles {
-	vertical-align: top;
+#wrap {
+	margin-top: 2em;
 }
 #tree {
-	padding-top: 2em;
+	display: none;
+	position: absolute;
+	left: 0;
+	bottom: 0;
+	top: 7em;
+	padding-top: .2em;
+	overflow-y: auto;
+	-ms-scroll-chaining: none;
+	overscroll-behavior-y: none;
+	scrollbar-color: #eb0 #333;
 }
+#thx_ff {
+	padding: 5em 0;
+}
+#tree::-webkit-scrollbar-track,
+#tree::-webkit-scrollbar {
+	background: #333;
+}
+#tree::-webkit-scrollbar-thumb {
+	background: #eb0;
+}
+#tree:hover {
+	z-index: 2;
+}
+#treeul {
+	position: relative;
+	left: -1.7em;
+	width: calc(100% + 1.3em);
+}
+.tglbtn,
 #tree>a+a {
 	padding: .2em .4em;
 	font-size: 1.2em;
@@ -460,9 +542,11 @@ input[type="checkbox"]:checked+label {
 	position: relative;
 	top: -.2em;
 }
+.tglbtn:hover,
 #tree>a+a:hover {
 	background: #805;
 }
+.tglbtn.on,
 #tree>a+a.on {
 	background: #fc4;
 	color: #400;
@@ -472,24 +556,22 @@ input[type="checkbox"]:checked+label {
 	padding: .3em .5em;
 	font-size: 1.5em;
 }
-#treefiles #files tbody {
-	border-radius: 0 .7em 0 .7em;
-}
-#treefiles #files thead th:nth-child(1) {
-	border-radius: .7em 0 0 0;
-}
 #tree ul,
 #tree li {
 	padding: 0;
 	margin: 0;
 }
 #tree ul {
-	border-left: .2em solid #444;
+	border-left: .2em solid #555;
 }
 #tree li {
 	margin-left: 1em;
 	list-style: none;
-	white-space: nowrap;
+	border-top: 1px solid #4c4c4c;
+	border-bottom: 1px solid #222;
+}
+#tree li:last-child {
+	border-bottom: none;
 }
 #treeul a.hl {
 	color: #400;
@@ -503,24 +585,12 @@ input[type="checkbox"]:checked+label {
 #treeul a+a {
 	width: calc(100% - 2em);
 	background: #333;
+	line-height: 1em;
 }
 #treeul a+a:hover {
 	background: #222;
 	color: #fff;
 }
-#treeul {
-	position: relative;
-	overflow: hidden;
-	left: -1.7em;
-}
-#treeul:hover {
-	z-index: 2;
-	overflow: visible;
-}
-#treeul:hover a+a {
-	width: auto;
-	min-width: calc(100% - 2em);
-}
 #treeul a:first-child {
 	font-family: monospace, monospace;
 }
@@ -566,7 +636,8 @@ input[type="checkbox"]:checked+label {
 #files td.min a {
 	display: none;
 }
-#files tr.play td {
+#files tr.play td,
+#files tr.play div a {
 	background: #fc4;
 	border-color: transparent;
 	color: #400;
@@ -579,3 +650,240 @@ input[type="checkbox"]:checked+label {
 	color: #300;
 	background: #fea;
 }
+#op_cfg {
+	max-width: none;
+	margin-right: 1.5em;
+}
+#op_cfg>div>a {
+	line-height: 2em;
+}
+#op_cfg>div>span {
+	display: inline-block;
+	padding: .2em .4em;
+}
+#op_cfg h3 {
+	margin: .8em 0 0 .6em;
+	padding: 0;
+	border-bottom: 1px solid #555;
+}
+#opdesc {
+	display: none;
+}
+#ops:hover #opdesc {
+	display: block;
+	background: linear-gradient(0deg,#555, #4c4c4c 80%, #444);
+	box-shadow: 0 .3em 1em #222;
+	padding: 1em;
+	border-radius: .3em;
+	position: absolute;
+	z-index: 3;
+	top: 6em;
+	right: 1.5em;
+}
+#ops:hover #opdesc.off {
+	display: none;
+}
+#opdesc code {
+	background: #3c3c3c;
+	padding: .2em .3em;
+	border-top: 1px solid #777;
+	border-radius: .3em;
+	font-family: monospace, monospace;
+	line-height: 2em;
+}
+#pvol,
+#barbuf,
+#barpos,
+#u2conf label {
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+}
+
+
+
+
+
+html.light {
+	color: #333;
+	background: #eee;
+	text-shadow: none;
+}
+html.light #ops,
+html.light .opbox,
+html.light #srch_form {
+	background: #f7f7f7;
+	box-shadow: 0 0 .3em #ddd;
+	border-color: #f7f7f7;
+}
+html.light #ops a.act {
+	box-shadow: 0 .2em .2em #ccc;
+	background: #fff;
+	border-color: #07a;
+	padding-top: .4em;
+}
+html.light #op_cfg h3 {
+	border-color: #ccc;
+}
+html.light .tglbtn,
+html.light #tree > a + a {
+	color: #666;
+	background: #ddd;
+	box-shadow: none;
+}
+html.light .tglbtn:hover,
+html.light #tree > a + a:hover {
+	background: #caf;
+}
+html.light .tglbtn.on,
+html.light #tree > a + a.on {
+	background: #4a0;
+	color: #fff;
+}
+html.light #srv_info {
+	color: #c83;
+	text-shadow: 1px 1px 0 #fff;
+}
+html.light #srv_info span {
+	color: #000;
+}
+html.light #treeul a+a {
+	background: inherit;
+	color: #06a;
+}
+html.light #treeul a.hl {
+	background: #07a;
+	color: #fff;
+}
+html.light #tree li {
+	border-color: #ddd #fff #f7f7f7 #fff;
+}
+html.light #tree ul {
+	border-color: #ccc;
+}
+html.light a,
+html.light #ops a,
+html.light #files tbody div a:last-child {
+	color: #06a;
+}
+html.light #files tbody {
+	background: #f7f7f7;
+}
+html.light #files {
+	box-shadow: 0 0 .3em #ccc;
+}
+html.light #files thead th {
+	background: #eee;
+}
+html.light #files tr td {
+	border-top: 1px solid #ddd;
+}
+html.light #files td {
+	border-bottom: 1px solid #f7f7f7;
+}
+html.light #files tbody tr:last-child td {
+	border-bottom: .2em solid #ccc;
+}
+html.light #files td:nth-child(2n) {
+	color: #d38;
+}
+html.light #files tr:hover td {
+	background: #fff;
+}
+html.light #files tbody a.play {
+	color: #c0f;
+}
+html.light tr.play td {
+	background: #fc5;
+}
+html.light tr.play a {
+	color: #406;
+}
+html.light #files th:hover .cfg,
+html.light #files th.min .cfg {
+	background: #ccc;
+}
+html.light #files > thead > tr > th.min span {
+	background: linear-gradient(90deg, rgba(204,204,204,0), rgba(204,204,204,0.5) 70%, #ccc);
+}
+html.light #blocked {
+	background: #eee;
+}
+html.light #blk_play a,
+html.light #blk_abrt a {
+	background: #fff;
+	box-shadow: 0 .2em .4em #ddd;
+}
+html.light #widget a {
+	color: #fc5;
+}
+html.light #files tr.sel:hover td {
+	background: #c37;
+}
+html.light #files tr.sel td {
+	color: #fff;
+}
+html.light #files tr.sel a {
+	color: #fff;
+}
+html.light #files tr.sel a.play.act {
+	color: #fb0;
+}
+html.light input[type="checkbox"] + label {
+	color: #333;
+}
+html.light .opview input[type="text"] {
+	background: #fff;
+	color: #333;
+	box-shadow: 0 0 2px #888;
+	border-color: #38d;
+}
+html.light #ops:hover #opdesc {
+	background: #fff;
+	box-shadow: 0 .3em 1em #ccc;
+}
+html.light #opdesc code {
+	background: #060;
+	color: #fff;
+}
+html.light #u2tab a>span,
+html.light #files td div span {
+	color: #000;
+}
+html.light #path {
+	background: #f7f7f7;
+	text-shadow: none;
+	box-shadow: 0 0 .3em #bbb;
+}
+html.light #path a {
+	color: #333;
+}
+html.light #path a:not(:last-child)::after {
+	border-color: #ccc;
+	background: none;
+	border-width: .1em .1em 0 0;
+	margin: -.2em .3em -.2em -.3em;
+}
+html.light #path a:hover {
+	background: none;
+	color: #60a;
+}
+html.light #files tbody div a {
+	color: #d38;
+}
+html.light #files a:hover,
+html.light #files tr.sel a:hover {
+	color: #000;
+	background: #fff;
+}
+html.light #tree {
+	scrollbar-color: #a70 #ddd;
+}
+html.light #tree::-webkit-scrollbar-track,
+html.light #tree::-webkit-scrollbar {
+	background: #ddd;
+}
+#tree::-webkit-scrollbar-thumb {
+	background: #da0;
+}

copyparty/web/browser.html

@@ -12,28 +12,45 @@
 
 <body>
     <div id="ops">
-        <a href="#" data-dest="">---</a>
-        <a href="#" data-perm="read" data-dest="search">🔎</a>
+        <a href="#" data-dest="" data-desc="close submenu">---</a>
         {%- if have_up2k_idx %}
-        <a href="#" data-dest="up2k">🚀</a>
+        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
+        <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
         {%- else %}
-        <a href="#" data-perm="write" data-dest="up2k">🚀</a>
+        <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
         {%- endif %}
-        <a href="#" data-perm="write" data-dest="bup">🎈</a>
-        <a href="#" data-perm="write" data-dest="mkdir">📂</a>
-        <a href="#" data-perm="write" data-dest="new_md">📝</a>
-        <a href="#" data-perm="write" data-dest="msg">📟</a>
+        <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
+        <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
+        <a href="#" data-perm="read write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
+        <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
+        <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
+        <div id="opdesc"></div>
     </div>
 
     <div id="op_search" class="opview">
         {%- if have_tags_idx %}
-        <table id="srch_form" class="tags"></table>
+        <div id="srch_form" class="tags"></div>
         {%- else %}
-        <table id="srch_form"></table>
+        <div id="srch_form"></div>
         {%- endif %}
         <div id="srch_q"></div>
     </div>
+
     {%- include 'upload.html' %}
+
+    <div id="op_cfg" class="opview opbox">
+        <h3>switches</h3>
+        <div>
+            <a id="tooltips" class="tglbtn" href="#">tooltips</a>
+            <a id="lightmode" class="tglbtn" href="#">lightmode</a>
+        </div>
+        {%- if have_zip %}
+        <h3>folder download</h3>
+        <div id="arc_fmt"></div>
+        {%- endif %}
+        <h3>key notation</h3>
+        <div id="key_notation"></div>
+    </div>
     
     <h1 id="path">
         <a href="#" id="entree">🌲</a>
@@ -42,36 +59,34 @@
         {%- endfor %}
     </h1>
     
-    <div id="pro" class="logue">{{ logues[0] }}</div>
+    <div id="tree">
+        <a href="#" id="detree">🍞...</a>
+        <a href="#" step="2" id="twobytwo">+</a>
+        <a href="#" step="-2" id="twig">&ndash;</a>
+        <a href="#" class="tglbtn" id="dyntree">a</a>
+        <ul id="treeul"></ul>
+        <div id="thx_ff">&nbsp;</div>
+    </div>
 
-    <table id="treetab">
-        <tr>
-            <td id="tree">
-                <a href="#" id="detree">🍞...</a>
-                <a href="#" step="2" id="twobytwo">+</a>
-                <a href="#" step="-2" id="twig">&ndash;</a>
-                <a href="#" id="dyntree">a</a>
-                <ul id="treeul"></ul>
-            </td>
-            <td id="treefiles"></td>
-        </tr>
-    </table>
+<div id="wrap">
+
+    <div id="pro" class="logue">{{ logues[0] }}</div>
 
     <table id="files">
         <thead>
             <tr>
-                <th></th>
-                <th><span>File Name</span></th>
-                <th sort="int"><span>Size</span></th>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
                 {%- for k in taglist %}
                     {%- if k.startswith('.') %}
-                        <th sort="int"><span>{{ k[1:] }}</span></th>
+                        <th name="tags/{{ k }}" sort="int"><span>{{ k[1:] }}</span></th>
                     {%- else %}
-                        <th><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
+                        <th name="tags/{{ k }}"><span>{{ k[0]|upper }}{{ k[1:] }}</span></th>
                     {%- endif %}
                 {%- endfor %}
-                <th><span>T</span></th>
-                <th><span>Date</span></th>
+                <th name="ext"><span>T</span></th>
+                <th name="ts"><span>Date</span></th>
             </tr>
         </thead>
         <tbody>
@@ -93,12 +108,21 @@
 
     <h2><a href="?h">control-panel</a></h2>
 
+</div>
+
     {%- if srv_info %}
     <div id="srv_info"><span>{{ srv_info }}</span></div>
     {%- endif %}
 
     <div id="widget">
-        <div id="wtoggle">♫</div>
+        <div id="wtoggle">
+            <span id="wzip">
+                <a href="#" id="selall">sel.<br />all</a>
+                <a href="#" id="selinv">sel.<br />inv.</a>
+                <a href="#" id="selzip">zip</a>
+            </span><a
+                href="#" id="wtico">♫</a>
+        </div>
         <div id="widgeti">
             <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
             <canvas id="pvol" width="288" height="38"></canvas>

copyparty/web/browser2.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>{{ title }}</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+</head>
+
+<body>
+    {%- if srv_info %}
+    <p><span>{{ srv_info }}</span></p>
+    {%- endif %}
+
+    {%- if have_b_u %}
+    <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+        <input type="hidden" name="act" value="bput" />
+        <input type="file" name="f" multiple /><br />
+        <input type="submit" value="start upload" />
+    </form>
+    <br />
+    {%- endif %}
+
+    {%- if logues[0] %}
+    <div>{{ logues[0] }}</div><br />
+    {%- endif %}
+
+    <table id="files">
+        <thead>
+            <tr>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
+                <th name="ts"><span>Date</span></th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+
+{%- for f in files %}
+    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+{%- endfor %}
+
+        </tbody>
+    </table>
+    
+    {%- if logues[1] %}
+    <div>{{ logues[1] }}</div><br />
+    {%- endif %}
+    
+    <h2><a href="{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
+
+</body>
+</html>

copyparty/web/md.css

@@ -50,6 +50,9 @@ pre code:last-child {
 pre code::before {
 	content: counter(precode);
 	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
 	display: inline-block;
 	text-align: right;
 	font-size: .75em;
@@ -591,12 +594,3 @@ blink {
 		color: #940;
 	}
 }
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-*/

copyparty/web/md.html

@@ -138,16 +138,16 @@ var md_opt = {
         document.documentElement.setAttribute("class", dark ? "dark" : "");
         btn.innerHTML = "go " + (dark ? "light" : "dark");
         if (window.localStorage)
-            localStorage.setItem('darkmode', dark ? 1 : 0);
+            localStorage.setItem('lightmode', dark ? 0 : 1);
     };
     btn.onclick = toggle;
-    if (window.localStorage && localStorage.getItem('darkmode') == 1)
+    if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		toggle();
 })();
 
 	</script>
     <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.full.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}
 	<script src="/.cpr/md2.js"></script>

copyparty/web/md.js

@@ -46,7 +46,7 @@ function statify(obj) {
     var ua = navigator.userAgent;
     if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
         // necessary on ff-68.7 at least
-        var s = document.createElement('style');
+        var s = mknod('style');
         s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
         console.log(s.innerHTML);
         document.head.appendChild(s);
@@ -65,7 +65,7 @@ function statify(obj) {
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = hesc(decodeURIComponent(n[a]));
+        var dec = hesc(uricom_dec(n[a])[0]);
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -175,12 +175,12 @@ function md_plug_err(ex, js) {
         msg = "Line " + ln + ", " + msg;
         var lns = js.split('\n');
         if (ln < lns.length) {
-            o = document.createElement('span');
+            o = mknod('span');
             o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
             o.textContent = lns[ln - 1];
         }
     }
-    errbox = document.createElement('div');
+    errbox = mknod('div');
     errbox.setAttribute('id', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;

copyparty/web/md2.css

@@ -1,126 +1,125 @@
 #toc {
-    display: none;
+	display: none;
 }
 #mtw {
-    display: block;
-    position: fixed;
-    left: .5em;
-    bottom: 0;
-    width: calc(100% - 56em);
+	display: block;
+	position: fixed;
+	left: .5em;
+	bottom: 0;
+	width: calc(100% - 56em);
 }
 #mw {
-    left: calc(100% - 55em);
-    overflow-y: auto;
-    position: fixed;
-    bottom: 0;
+	left: calc(100% - 55em);
+	overflow-y: auto;
+	position: fixed;
+	bottom: 0;
 }
 
 
 /* single-screen */
 #mtw.preview,
 #mw.editor {
-    opacity: 0;
-    z-index: 1;
+	opacity: 0;
+	z-index: 1;
 }
 #mw.preview,
 #mtw.editor {
-    z-index: 5;
+	z-index: 5;
 }
 #mtw.single,
 #mw.single {
-    margin: 0;
-    left: 1em;
-    left: max(1em, calc((100% - 56em) / 2));
+	margin: 0;
+	left: 1em;
+	left: max(1em, calc((100% - 56em) / 2));
 }
 #mtw.single {
-    width: 55em;
-    width: min(55em, calc(100% - 2em));
+	width: 55em;
+	width: min(55em, calc(100% - 2em));
 }
 
 
 #mp {
-    position: relative;
+	position: relative;
 }
 #mt, #mtr {
-    width: 100%;
-    height: calc(100% - 1px);
-    color: #444;
-    background: #f7f7f7;
-    border: 1px solid #999;
-    outline: none;
-    padding: 0;
-    margin: 0;
-    font-family: 'consolas', monospace, monospace;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    word-wrap: break-word; /*ie*/
-    overflow-y: scroll;
-    line-height: 1.3em;
-    font-size: .9em;
-    position: relative;
-    scrollbar-color: #eb0 #f7f7f7;
+	width: 100%;
+	height: calc(100% - 1px);
+	color: #444;
+	background: #f7f7f7;
+	border: 1px solid #999;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	font-family: 'consolas', monospace, monospace;
+	white-space: pre-wrap;
+	word-break: break-word;
+	overflow-wrap: break-word;
+	word-wrap: break-word; /*ie*/
+	overflow-y: scroll;
+	line-height: 1.3em;
+	font-size: .9em;
+	position: relative;
+	scrollbar-color: #eb0 #f7f7f7;
 }
 html.dark #mt {
-    color: #eee;
-    background: #222;
-    border: 1px solid #777;
-    scrollbar-color: #b80 #282828;
+	color: #eee;
+	background: #222;
+	border: 1px solid #777;
+	scrollbar-color: #b80 #282828;
 }
 #mtr {
-    position: absolute;
-    top: 0;
-    left: 0;
+	position: absolute;
+	top: 0;
+	left: 0;
 }
 #save.force-save {
-    color: #400;
-    background: #f97;
-    border-radius: .15em;
+	color: #400;
+	background: #f97;
+	border-radius: .15em;
 }
 html.dark #save.force-save {
-    color: #fca;
-    background: #720;
+	color: #fca;
+	background: #720;
 }
 #save.disabled {
-    opacity: .4;
+	opacity: .4;
 }
 #helpbox,
 #toast {
-    background: #f7f7f7;
-    border-radius: .4em;
-    z-index: 9001;
+	background: #f7f7f7;
+	border-radius: .4em;
+	z-index: 9001;
 }
 #helpbox {
-    display: none;
-    position: fixed;
-    padding: 2em;
-    top: 4em;
-    overflow-y: auto;
-    box-shadow: 0 .5em 2em #777;
-    height: calc(100% - 12em);
-    left: calc(50% - 15em);
-    right: 0;
-    width: 30em;
+	display: none;
+	position: fixed;
+	padding: 2em;
+	top: 4em;
+	overflow-y: auto;
+	box-shadow: 0 .5em 2em #777;
+	height: calc(100% - 12em);
+	left: calc(50% - 15em);
+	right: 0;
+	width: 30em;
 }
 #helpclose {
-    display: block;
+	display: block;
 }
 html.dark #helpbox {
-    box-shadow: 0 .5em 2em #444;
+	box-shadow: 0 .5em 2em #444;
 }
 html.dark #helpbox,
 html.dark #toast {
-    background: #222;
-    border: 1px solid #079;
-    border-width: 1px 0;
+	background: #222;
+	border: 1px solid #079;
+	border-width: 1px 0;
 }
 #toast {
-    font-weight: bold;
-    text-align: center;
-    padding: .6em 0;
-    position: fixed;
-    z-index: 9001;
-    top: 30%;
-    transition: opacity 0.2s ease-in-out;
-    opacity: 1;
+	font-weight: bold;
+	text-align: center;
+	padding: .6em 0;
+	position: fixed;
+	top: 30%;
+	transition: opacity 0.2s ease-in-out;
+	opacity: 1;
 }

copyparty/web/md2.js

@@ -16,7 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = document.createElement('div');
+    var d = mknod('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
@@ -71,7 +71,7 @@ var map_src = [];
 var map_pre = [];
 function genmap(dom, oldmap) {
     var find = nlines;
-    while (oldmap && find --> 0) {
+    while (oldmap && find-- > 0) {
         var tmap = genmapq(dom, '*[data-ln="' + find + '"]');
         if (!tmap || !tmap.length)
             continue;
@@ -94,7 +94,7 @@ var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
     function draw_md() {
-        var t0 = new Date().getTime();
+        var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
 
@@ -110,7 +110,7 @@ var draw_md = (function () {
 
         cls(ebi('save'), 'disabled', src == server_md);
 
-        var t1 = new Date().getTime();
+        var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
     }
 
@@ -252,7 +252,7 @@ function Modpoll() {
         }
 
         console.log('modpoll...');
-        var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
         var xhr = new XMLHttpRequest();
         xhr.modpoll = this;
         xhr.open('GET', url, true);
@@ -399,7 +399,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
@@ -455,7 +455,7 @@ function toast(autoclose, style, width, msg) {
         ok.parentNode.removeChild(ok);
 
     style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
-    ok = document.createElement('div');
+    ok = mknod('div');
     ok.setAttribute('id', 'toast');
     ok.setAttribute('style', style);
     ok.innerHTML = msg;
@@ -1049,7 +1049,7 @@ action_stack = (function () {
         var p1 = from.length,
             p2 = to.length;
 
-        while (p1 --> 0 && p2 --> 0)
+        while (p1-- > 0 && p2-- > 0)
             if (from[p1] != to[p2])
                 break;
 
@@ -1142,14 +1142,3 @@ action_stack = (function () {
         _ref: ref
     }
 })();
-
-/*
-ebi('help').onclick = function () {
-    var c1 = getComputedStyle(dom_src).cssText.split(';');
-    var c2 = getComputedStyle(dom_ref).cssText.split(';');
-    var max = Math.min(c1.length, c2.length);
-    for (var a = 0; a < max; a++)
-        if (c1[a] !== c2[a])
-            console.log(c1[a] + '\n' + c2[a]);
-}
-*/

copyparty/web/mde.css

@@ -8,68 +8,58 @@ html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
 html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
 
 html {
-    line-height: 1.5em;
+	line-height: 1.5em;
 }
 html, body {
-    margin: 0;
-    padding: 0;
-    min-height: 100%;
-    font-family: sans-serif;
-    background: #f7f7f7;
-    color: #333;
+	margin: 0;
+	padding: 0;
+	min-height: 100%;
+	font-family: sans-serif;
+	background: #f7f7f7;
+	color: #333;
 }
 #mn {
-    font-weight: normal;
-    margin: 1.3em 0 .7em 1em;
+	font-weight: normal;
+	margin: 1.3em 0 .7em 1em;
 }
 #mn a {
-    color: #444;
-    margin: 0 0 0 -.2em;
-    padding: 0 0 0 .4em;
-    text-decoration: none;
-    /* ie: */
-    border-bottom: .1em solid #777\9;
-    margin-right: 1em\9;
+	color: #444;
+	margin: 0 0 0 -.2em;
+	padding: 0 0 0 .4em;
+	text-decoration: none;
+	/* ie: */
+	border-bottom: .1em solid #777\9;
+	margin-right: 1em\9;
 }
 #mn a:first-child {
-    padding-left: .5em;
+	padding-left: .5em;
 }
 #mn a:last-child {
-    padding-right: .5em;
+	padding-right: .5em;
 }
 #mn a:not(:last-child):after {
-    content: '';
-    width: 1.05em;
-    height: 1.05em;
-    margin: -.2em .3em -.2em -.4em;
-    display: inline-block;
-    border: 1px solid rgba(0,0,0,0.2);
-    border-width: .2em .2em 0 0;
-    transform: rotate(45deg);
+	content: '';
+	width: 1.05em;
+	height: 1.05em;
+	margin: -.2em .3em -.2em -.4em;
+	display: inline-block;
+	border: 1px solid rgba(0,0,0,0.2);
+	border-width: .2em .2em 0 0;
+	transform: rotate(45deg);
 }
 #mn a:hover {
-    color: #000;
-    text-decoration: underline;
+	color: #000;
+	text-decoration: underline;
 }
 
 html .editor-toolbar>button.disabled {
-    opacity: .35;
-    pointer-events: none;
+	opacity: .35;
+	pointer-events: none;
 }
 html .editor-toolbar>button.save.force-save {
-    background: #f97;
+	background: #f97;
 }
 
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-.cm-header { font-size: .4em !important }
-*/
-
 
 
 
@@ -101,29 +91,29 @@ html .editor-toolbar>button.save.force-save {
 	line-height: 1.1em;
 }
 .mdo a {
-    color: #fff;
-    background: #39b;
-    text-decoration: none;
-    padding: 0 .3em;
-    border: none;
-    border-bottom: .07em solid #079;
+	color: #fff;
+	background: #39b;
+	text-decoration: none;
+	padding: 0 .3em;
+	border: none;
+	border-bottom: .07em solid #079;
 }
 .mdo h2 {
-    color: #fff;
-    background: #555;
-    margin-top: 2em;
-    border-bottom: .22em solid #999;
-    border-top: none;
+	color: #fff;
+	background: #555;
+	margin-top: 2em;
+	border-bottom: .22em solid #999;
+	border-top: none;
 }
 .mdo h1 {
-    color: #fff;
-    background: #444;
-    font-weight: normal;
-    border-top: .4em solid #fb0;
-    border-bottom: .4em solid #777;
-    border-radius: 0 1em 0 1em;
-    margin: 3em 0 1em 0;
-    padding: .5em 0;
+	color: #fff;
+	background: #444;
+	font-weight: normal;
+	border-top: .4em solid #fb0;
+	border-bottom: .4em solid #777;
+	border-radius: 0 1em 0 1em;
+	margin: 3em 0 1em 0;
+	padding: .5em 0;
 }
 h1, h2 {
 	line-height: 1.5em;
@@ -197,14 +187,14 @@ th {
 
 /* mde support */
 .mdo {
-    padding: 1em;
-    background: #f7f7f7;
+	padding: 1em;
+	background: #f7f7f7;
 }
 html.dark .mdo {
-    background: #1c1c1c;
+	background: #1c1c1c;
 }
 .CodeMirror {
-    background: #f7f7f7;
+	background: #f7f7f7;
 }
 
 
@@ -214,108 +204,108 @@ html.dark .mdo {
 /* darkmode */
 html.dark .mdo,
 html.dark .CodeMirror {
-    border-color: #222;
+	border-color: #222;
 }
 html.dark,
 html.dark body,
 html.dark .CodeMirror {
-    background: #222;
-    color: #ccc;
+	background: #222;
+	color: #ccc;
 }
 html.dark .CodeMirror-cursor {
-    border-color: #fff;
+	border-color: #fff;
 }
 html.dark .CodeMirror-selected {
-    box-shadow: 0 0 1px #0cf inset;
+	box-shadow: 0 0 1px #0cf inset;
 }
 html.dark .CodeMirror-selected,
 html.dark .CodeMirror-selectedtext {
-    border-radius: .1em;
-    background: #246;
-    color: #fff;
+	border-radius: .1em;
+	background: #246;
+	color: #fff;
 }
 html.dark .mdo a {
-    background: #057;
+	background: #057;
 }
 html.dark .mdo h1 a, html.dark .mdo h4 a,
 html.dark .mdo h2 a, html.dark .mdo h5 a,
 html.dark .mdo h3 a, html.dark .mdo h6 a {
-    color: inherit;
-    background: none;
+	color: inherit;
+	background: none;
 }
 html.dark pre,
 html.dark code {
-    color: #8c0;
-    background: #1a1a1a;
-    border: .07em solid #333;
+	color: #8c0;
+	background: #1a1a1a;
+	border: .07em solid #333;
 }
 html.dark .mdo ul,
 html.dark .mdo ol {
-    border-color: #444;
+	border-color: #444;
 }
 html.dark .mdo>ul,
 html.dark .mdo>ol {
-    border-color: #555;
+	border-color: #555;
 }
 html.dark strong {
-    color: #fff;
+	color: #fff;
 }
 html.dark p>em,
 html.dark li>em,
 html.dark td>em {
-    color: #f94;
-    border-color: #666;
+	color: #f94;
+	border-color: #666;
 }
 html.dark h1 {
-    background: #383838;
-    border-top: .4em solid #b80;
-    border-bottom: .4em solid #4c4c4c;
+	background: #383838;
+	border-top: .4em solid #b80;
+	border-bottom: .4em solid #4c4c4c;
 }
 html.dark h2 {
-    background: #444;
-    border-bottom: .22em solid #555;
+	background: #444;
+	border-bottom: .22em solid #555;
 }
 html.dark td,
 html.dark th {
-    border-color: #444;
+	border-color: #444;
 }
 html.dark blockquote {
-    background: #282828;
-    border: .07em dashed #444;
+	background: #282828;
+	border: .07em dashed #444;
 }
 
 
 
 html.dark #mn a {
-    color: #ccc;
+	color: #ccc;
 }
 html.dark #mn a:not(:last-child):after {
-    border-color: rgba(255,255,255,0.3);
+	border-color: rgba(255,255,255,0.3);
 }
 html.dark .editor-toolbar {
-    border-color: #2c2c2c;
-    background: #1c1c1c;
+	border-color: #2c2c2c;
+	background: #1c1c1c;
 }
 html.dark .editor-toolbar>i.separator {
-    border-left: 1px solid #444;
-    border-right: 1px solid #111;
+	border-left: 1px solid #444;
+	border-right: 1px solid #111;
 }
 html.dark .editor-toolbar>button {
-    margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
-    color: #aaa;
+	margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
+	color: #aaa;
 }
 
 
 
 html.dark .editor-toolbar>button:hover {
-    color: #333;
+	color: #333;
 }
 html.dark .editor-toolbar>button.active {
-    color: #333;
-    border-color: #ec1;
-    background: #c90;
+	color: #333;
+	border-color: #ec1;
+	background: #c90;
 }
 html.dark .editor-toolbar::after,
 html.dark .editor-toolbar::before {
-    background: none;
+	background: none;
 }

copyparty/web/mde.html

@@ -31,12 +31,12 @@ var md_opt = {
 
 var lightswitch = (function () {
 	var fun = function () {
-		var dark = !!!document.documentElement.getAttribute("class");
+		var dark = !document.documentElement.getAttribute("class");
 		document.documentElement.setAttribute("class", dark ? "dark" : "");
 		if (window.localStorage)
-			localStorage.setItem('darkmode', dark ? 1 : 0);
+			localStorage.setItem('lightmode', dark ? 0 : 1);
 	};
-	if (window.localStorage && localStorage.getItem('darkmode') == 1)
+	if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		fun();
 	
 	return fun;

copyparty/web/mde.js

@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = decodeURIComponent(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+        var dec = uricom_dec(n[a])[0].replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -71,7 +71,7 @@ var mde = (function () {
 })();
 
 function set_jumpto() {
-    document.querySelector('.editor-preview-side').onclick = jumpto;
+    QS('.editor-preview-side').onclick = jumpto;
 }
 
 function jumpto(ev) {
@@ -94,7 +94,7 @@ function md_changed(mde, on_srv) {
         window.md_saved = mde.value();
 
     var md_now = mde.value();
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
 
     if (md_now == window.md_saved)
         save_btn.classList.add('disabled');
@@ -105,7 +105,7 @@ function md_changed(mde, on_srv) {
 }
 
 function save(mde) {
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
     if (save_btn.classList.contains('disabled')) {
         alert('there is nothing to save');
         return;
@@ -212,7 +212,7 @@ function save_chk() {
     last_modified = this.lastmod;
     md_changed(this.mde, true);
 
-    var ok = document.createElement('div');
+    var ok = mknod('div');
     ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
     ok.innerHTML = 'OK✔️';
     var parent = ebi('m');

copyparty/web/msg.css

@@ -3,7 +3,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -20,8 +20,8 @@ body {
 	padding-bottom: 5em;
 }
 #box {
-    padding: .5em 1em;
-    background: #2c2c2c;
+	padding: .5em 1em;
+	background: #2c2c2c;
 }
 pre {
 	font-family: monospace, monospace;

copyparty/web/splash.html

@@ -13,23 +13,27 @@
     <div id="wrap">
         <p>hello {{ this.uname }}</p>
 
+        {%- if rvol %}
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
+        {%- if wvol %}
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/">
+            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />
@@ -38,7 +42,7 @@
     </div>
     <script>
 
-if (window.localStorage && localStorage.getItem('darkmode') == 1)
+if (window.localStorage && localStorage.getItem('lightmode') != 1)
     document.documentElement.setAttribute("class", "dark");
 
 </script>

copyparty/web/upload.css

@@ -19,6 +19,11 @@
 	color: #f87;
 	padding: .5em;
 }
+#u2err.msg {
+	color: #999;
+	padding: .5em;
+	font-size: .9em;
+}
 #u2btn {
 	color: #eee;
 	background: #555;
@@ -47,6 +52,11 @@
 	margin: -1.5em 0;
 	padding: .8em 0;
 	width: 100%;
+	max-width: 12em;
+	display: inline-block;
+}
+#u2conf #u2btn_cw {
+	text-align: right;
 }
 #u2notbtn {
 	display: none;
@@ -62,7 +72,7 @@
 	width: calc(100% - 2em);
 	max-width: 100em;
 }
-#u2form.srch #u2tab {
+#op_up2k.srch #u2tab {
 	max-width: none;
 }
 #u2tab td {
@@ -72,23 +82,60 @@
 }
 #u2tab td:nth-child(2) {
 	width: 5em;
+	white-space: nowrap;
 }
 #u2tab td:nth-child(3) {
 	width: 40%;
 }
-#u2form.srch #u2tab td:nth-child(3) {
+#op_up2k.srch #u2tab td:nth-child(3) {
 	font-family: sans-serif;
 	width: auto;
 }
-#u2tab tr+tr:hover td {
+#u2tab tbody tr:hover td {
 	background: #222;
 }
+#u2cards {
+	padding: 1em 0 .3em 1em;
+	margin: 1.5em auto -2.5em auto;
+	text-align: center;
+	overflow: hidden;
+}
+#u2cards.w {
+	width: 45em;
+	text-align: left;
+}
+#u2cards a {
+	padding: .2em 1em;
+	border: 1px solid #777;
+	border-width: 0 0 1px 0;
+	background: linear-gradient(to bottom, #333, #222);
+}
+#u2cards a:first-child {
+	border-radius: .4em 0 0 0;
+}
+#u2cards a:last-child {
+	border-radius: 0 .4em 0 0;
+}
+#u2cards a.act {
+	padding-bottom: .5em;
+	border-width: 1px 1px .1em 1px;
+	border-radius: .3em .3em 0 0;
+	margin-left: -1px;
+	background: linear-gradient(to bottom, #464, #333 80%);
+	box-shadow: 0 -.17em .67em #280;
+	border-color: #7c5 #583 #333 #583;
+	position: relative;
+	color: #fd7;
+}
+#u2cards span {
+	color: #fff;
+}
 #u2conf {
 	margin: 1em auto;
 	width: 30em;
 }
 #u2conf.has_btn {
-	width: 46em;
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;
@@ -99,12 +146,16 @@
 	outline: none;
 }
 #u2conf .txtbox {
-	width: 4em;
+	width: 3em;
 	color: #fff;
 	background: #444;
 	border: 1px solid #777;
 	font-size: 1.2em;
 	padding: .15em 0;
+	height: 1.05em;
+}
+#u2conf .txtbox.err {
+	background: #922;
 }
 #u2conf a {
 	color: #fff;
@@ -113,13 +164,12 @@
 	border-radius: .1em;
 	font-size: 1.5em;
 	padding: .1em 0;
-	margin: 0 -.25em;
+	margin: 0 -1px;
 	width: 1.5em;
 	height: 1em;
 	display: inline-block;
 	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
+	bottom: -0.08em;
 }
 #u2conf input+a {
 	background: #d80;
@@ -130,7 +180,6 @@
 	height: 1em;
 	padding: .4em 0;
 	display: block;
-	user-select: none;
 	border-radius: .25em;
 }
 #u2conf input[type="checkbox"] {
@@ -170,12 +219,13 @@
 	text-align: center;
 	overflow: hidden;
 	margin: 0 -2em;
-	height: 0;
 	padding: 0 1em;
+	height: 0;
 	opacity: .1;
-    transition: all 0.14s ease-in-out;
-	border-radius: .4em;
+	transition: all 0.14s ease-in-out;
 	box-shadow: 0 .2em .5em #222;
+	border-radius: .4em;
+	z-index: 1;
 }
 #u2cdesc.show {
 	padding: 1em;
@@ -193,24 +243,6 @@
 .prog {
 	font-family: monospace;
 }
-.prog>div {
-	display: inline-block;
-	position: relative;
-	overflow: hidden;
-	margin: 0;
-	padding: 0;
-	height: 1.1em;
-	margin-bottom: -.15em;
-	box-shadow: -1px -1px 0 inset rgba(255,255,255,0.1);
-}
-.prog>div>div {
-	width: 0%;
-	position: absolute;
-	left: 0;
-	top: 0;
-	bottom: 0;
-	background: #0a0;
-}
 #u2tab a>span {
 	font-weight: bold;
 	font-style: italic;
@@ -221,3 +253,44 @@
 	float: right;
 	margin-bottom: -.3em;
 }
+
+
+
+
+
+html.light #u2btn {
+	box-shadow: .4em .4em 0 #ccc;
+}
+html.light #u2cards span {
+	color: #000;
+}
+html.light #u2cards a {
+	background: linear-gradient(to bottom, #eee, #fff);
+}
+html.light #u2cards a.act {
+	color: #037;
+	background: inherit;
+	box-shadow: 0 -.17em .67em #0ad;
+	border-color: #09c #05a #eee #05a;
+}
+html.light #u2conf .txtbox {
+	background: #fff;
+	color: #444;
+}
+html.light #u2conf .txtbox.err {
+	background: #f96;
+	color: #300;
+}
+html.light #u2cdesc {
+	background: #fff;
+	border: none;
+}
+html.light #op_up2k.srch #u2btn {
+	border-color: #a80;
+}
+html.light #u2foot {
+	color: #000;
+}
+html.light #u2tab tbody tr:hover td {
+	background: #fff;
+}

copyparty/web/upload.html

@@ -1,7 +1,7 @@
 
     <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
@@ -9,7 +9,7 @@
     </div>
 
     <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="mkdir" />
             <input type="text" name="name" size="30">
             <input type="submit" value="mkdir">
@@ -17,15 +17,15 @@
     </div>
 
     <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="new_md" />
             <input type="text" name="name" size="30">
             <input type="submit" value="create doc">
         </form>
     </div>
 
-    <div id="op_msg" class="opview opbox">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
+    <div id="op_msg" class="opview opbox act">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="text" name="msg" size="30">
             <input type="submit" value="send msg">
         </form>
@@ -36,7 +36,7 @@
 
             <table id="u2conf">
                 <tr>
-                    <td>parallel uploads</td>
+                    <td><br />parallel uploads:</td>
                     <td rowspan="2">
                         <input type="checkbox" id="multitask" />
                         <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
@@ -59,9 +59,9 @@
                 </tr>
                 <tr>
                     <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
+                        <a href="#" id="nthread_sub">&ndash;</a><input
+                            class="txtbox" id="nthread" value="2"/><a
+                            href="#" id="nthread_add">+</a><br />&nbsp;
                     </td>
                 </tr>
             </table>
@@ -73,19 +73,31 @@
             <div id="u2btn_ct">
                 <div id="u2btn">
                     <span id="u2bm"></span><br />
-                    drop files here<br />
+                    drag/drop files<br />
+                    and folders here<br />
                     (or click me)
                 </div>
             </div>
 
+            <div id="u2cards">
+                <a href="#" act="ok">ok <span>0</span></a><a
+                href="#" act="ng">ng <span>0</span></a><a
+                href="#" act="done">done <span>0</span></a><a
+                href="#" act="bz" class="act">busy <span>0</span></a><a
+                href="#" act="q">que <span>0</span></a>
+            </div>
+
             <table id="u2tab">
-                <tr>
-                    <td>filename</td>
-                    <td>status</td>
-                    <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
-                </tr>
+                <thead>
+                    <tr>
+                        <td>filename</td>
+                        <td>status</td>
+                        <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
+                    </tr>
+                </thead>
+                <tbody></tbody>
             </table>
 
             <p id="u2foot"></p>
-            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
+            <p id="u2footfoot">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
     </div>

copyparty/web/util.js

@@ -1,5 +1,15 @@
 "use strict";
 
+if (!window['console'])
+    window['console'] = {
+        "log": function (msg) { }
+    };
+
+
+var clickev = window.Touch ? 'touchstart' : 'click',
+    ANDROID = /(android)/i.test(navigator.userAgent);
+
+
 // error handler for mobile devices
 function hcroak(msg) {
     document.body.innerHTML = msg;
@@ -23,6 +33,7 @@ function esc(txt) {
 }
 function vis_exh(msg, url, lineNo, columnNo, error) {
     window.onerror = undefined;
+    window['vis_exh'] = null;
     var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
         esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
 
@@ -39,9 +50,11 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
 }
 
 
-function ebi(id) {
-    return document.getElementById(id);
-}
+var ebi = document.getElementById.bind(document),
+    QS = document.querySelector.bind(document),
+    QSA = document.querySelectorAll.bind(document),
+    mknod = document.createElement.bind(document);
+
 
 function ev(e) {
     e = e || window.event;
@@ -79,7 +92,7 @@ if (!String.startsWith) {
 // https://stackoverflow.com/a/950146
 function import_js(url, cb) {
     var head = document.head || document.getElementsByTagName('head')[0];
-    var script = document.createElement('script');
+    var script = mknod('script');
     script.type = 'text/javascript';
     script.src = url;
 
@@ -90,7 +103,107 @@ function import_js(url, cb) {
 }
 
 
-function sortTable(table, col) {
+var crctab = (function () {
+    var c, tab = [];
+    for (var n = 0; n < 256; n++) {
+        c = n;
+        for (var k = 0; k < 8; k++) {
+            c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+        }
+        tab[n] = c;
+    }
+    return tab;
+})();
+
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+        crc = (crc >>> 8) ^ crctab[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return ((crc ^ (-1)) >>> 0).toString(16);
+}
+
+
+function clmod(obj, cls, add) {
+    var re = new RegExp('\\s*\\b' + cls + '\\s*\\b', 'g');
+    if (add == 't')
+        add = !re.test(obj.className);
+
+    obj.className = obj.className.replace(re, ' ') + (add ? ' ' + cls : '');
+}
+
+
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["lead", -1, ""], ["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v)
+                        v = uricom_dec(v)[0]
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map(function (x) { return x; });
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+    }
+    return nodes;
+}
+
+
+function sortTable(table, col, cb) {
     var tb = table.tBodies[0],
         th = table.tHead.rows[0].cells,
         tr = Array.prototype.slice.call(tb.rows, 0),
@@ -99,6 +212,27 @@ function sortTable(table, col) {
         th[a].className = th[a].className.replace(/ *sort-?1 */, " ");
     th[col].className += ' sort' + reverse;
     var stype = th[col].getAttribute('sort');
+    try {
+        var nrules = [], rules = jread("fsort", []);
+        rules.unshift([th[col].getAttribute('name'), reverse, stype || '']);
+        for (var a = 0; a < rules.length; a++) {
+            var add = true;
+            for (var b = 0; b < a; b++)
+                if (rules[a][0] == rules[b][0])
+                    add = false;
+
+            if (add)
+                nrules.push(rules[a]);
+
+            if (nrules.length >= 10)
+                break;
+        }
+        jwrite("fsort", nrules);
+    }
+    catch (ex) {
+        console.log("failed to persist sort rules, resetting: " + ex);
+        jwrite("fsort", null);
+    }
     var vl = [];
     for (var a = 0; a < tr.length; a++) {
         var cell = tr[a].cells[col];
@@ -126,8 +260,9 @@ function sortTable(table, col) {
         return reverse * (a.localeCompare(b));
     });
     for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
+    if (cb) cb();
 }
-function makeSortable(table) {
+function makeSortable(table, cb) {
     var th = table.tHead, i;
     th && (th = th.rows[0]) && (th = th.cells);
     if (th) i = th.length;
@@ -135,15 +270,14 @@ function makeSortable(table) {
     while (--i >= 0) (function (i) {
         th[i].onclick = function (e) {
             ev(e);
-            sortTable(table, i);
+            sortTable(table, i, cb);
         };
     }(i));
 }
 
 
-
 (function () {
-    var ops = document.querySelectorAll('#ops>a');
+    var ops = QSA('#ops>a');
     for (var a = 0; a < ops.length; a++) {
         ops[a].onclick = opclick;
     }
@@ -156,32 +290,35 @@ function opclick(e) {
     var dest = this.getAttribute('data-dest');
     goto(dest);
 
-    swrite('opmode', dest || undefined);
+    swrite('opmode', dest || null);
 
-    var input = document.querySelector('.opview.act input:not([type="hidden"])')
+    var input = QS('.opview.act input:not([type="hidden"])')
     if (input)
         input.focus();
 }
 
 
 function goto(dest) {
-    var obj = document.querySelectorAll('.opview.act');
+    var obj = QSA('.opview.act');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
-    obj = document.querySelectorAll('#ops>a');
+    obj = QSA('#ops>a');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     if (dest) {
         var ui = ebi('op_' + dest);
-        ui.classList.add('act');
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+        clmod(ui, 'act', true);
+        QS('#ops>a[data-dest=' + dest + ']').className += " act";
 
         var fn = window['goto_' + dest];
         if (fn)
             fn();
     }
+
+    if (window['treectl'])
+        treectl.onscroll();
 }
 
 
@@ -189,7 +326,10 @@ function goto(dest) {
     goto();
     var op = sread('opmode');
     if (op !== null && op !== '.')
-        goto(op);
+        try {
+            goto(op);
+        }
+        catch (ex) { }
 })();
 
 
@@ -220,6 +360,31 @@ function linksplit(rp) {
 }
 
 
+function uricom_enc(txt, do_fb_enc) {
+    try {
+        return encodeURIComponent(txt);
+    }
+    catch (ex) {
+        console.log("uce-err [" + txt + "]");
+        if (do_fb_enc)
+            return esc(txt);
+
+        return txt;
+    }
+}
+
+
+function uricom_dec(txt) {
+    try {
+        return [decodeURIComponent(txt), true];
+    }
+    catch (ex) {
+        console.log("ucd-err [" + txt + "]");
+        return [txt, false];
+    }
+}
+
+
 function get_evpath() {
     var ret = document.location.pathname;
 
@@ -234,7 +399,7 @@ function get_evpath() {
 
 
 function get_vpath() {
-    return decodeURIComponent(get_evpath());
+    return uricom_dec(get_evpath())[0];
 }
 
 
@@ -244,6 +409,7 @@ function unix2iso(ts) {
 
 
 function s2ms(s) {
+    s = Math.floor(s);
     var m = Math.floor(s / 60);
     return m + ":" + ("0" + (s - m * 60)).slice(-2);
 }
@@ -262,12 +428,12 @@ function sread(key) {
     if (window.localStorage)
         return localStorage.getItem(key);
 
-    return '';
+    return null;
 }
 
 function swrite(key, val) {
     if (window.localStorage) {
-        if (val === undefined)
+        if (val === undefined || val === null)
             localStorage.removeItem(key);
         else
             localStorage.setItem(key, val);
@@ -293,7 +459,7 @@ function icfg_get(name, defval) {
     var o = ebi(name);
 
     var val = parseInt(sread(name));
-    if (val === null)
+    if (isNaN(val))
         return parseInt(o ? o.value : defval);
 
     if (o)
@@ -330,19 +496,18 @@ function bcfg_upd_ui(name, val) {
 
     if (o.getAttribute('type') == 'checkbox')
         o.checked = val;
-    else if (o)
-        o.setAttribute('class', val ? 'on' : '');
+    else if (o) {
+        clmod(o, 'on', val);
+    }
 }
 
 
-function hist_push(html, url) {
-    var key = new Date().getTime();
-    sessionStorage.setItem(key, html);
-    history.pushState(key, url, url);
+function hist_push(url) {
+    console.log("h-push " + url);
+    history.pushState(url, url, url);
 }
 
-function hist_replace(html, url) {
-    var key = new Date().getTime();
-    sessionStorage.setItem(key, html);
-    history.replaceState(key, url, url);
+function hist_replace(url) {
+    console.log("h-repl " + url);
+    history.replaceState(url, url, url);
 }

docs/example.conf

@@ -32,9 +32,13 @@ r
 
 # and a folder where anyone can upload
 # but nobody can see the contents
+# and set the e2d flag to enable the uploads database
+# and set the nodupe flag to reject duplicate uploads
 /home/ed/inc
 /dump
 w
+c e2d
+c nodupe
 
 # this entire config file can be replaced with these arguments:
 # -u ed:123 -u k:k -v .::r:aed -v priv:priv:rk:aed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w

docs/minimal-up2k.html

@@ -0,0 +1,31 @@
+<!--
+  save this as .epilogue.html inside a write-only folder to declutter the UI
+-->
+
+<style>
+
+    /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
+
+    #ops, #tree, #path,  /* main tabs and navigators (tree/breadcrumbs) */
+
+    #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
+
+    #u2cards  /* and the upload progress tabs */
+
+    {display: none !important}  /* do it! */
+
+
+
+    /* add some margins because now it's weird */
+    .opview {margin-top: 2.5em}
+    #op_up2k {margin-top: 3em}
+
+    /* and embiggen the upload button */
+    #u2conf #u2btn, #u2btn {padding:1.5em 0}
+
+    /* adjust the button area a bit */
+    #u2conf.has_btn {width: 35em !important; margin: 5em auto}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

docs/music-analysis.sh

@@ -0,0 +1,242 @@
+#!/bin/bash
+echo please dont actually run this as a scriopt
+exit 1
+
+
+# dependency-heavy, not particularly good fit
+pacman -S llvm10
+python3 -m pip install --user librosa
+git clone https://github.com/librosa/librosa.git
+
+
+# correct bpm for tracks with bad tags
+br='
+/Trip Trip Trip\(Hardcore Edit\).mp3/ {v=176}
+/World!!.BIG_SOS/ {v=175}
+/\/08\..*\(BIG_SOS Bootleg\)\.mp3/ {v=175}
+/もってけ！セーラ服.Asterisk DnB/ {v=175}
+/Rondo\(Asterisk DnB Re.mp3/ {v=175}
+/Ray Nautica 175 Edit/ {v=175;x="thunk"}
+/TOKIMEKI Language.Jauz/ {v=174}
+/YUPPUN Hardcore Remix\).mp3/ {v=174;x="keeps drifting"}
+/(èâAâï.î╧ûδ|バーチャリアル.狐耶)J-Core Remix\).mp3/ {v=172;x="hard"}
+/lucky train..Freezer/ {v=170}
+/Alf zero Bootleg ReMix/ {v=170}
+/Prisoner of Love.Kacky/ {v=170}
+/火炎 .Qota/ {v=170}
+/\(hu-zin Bootleg\)\.mp3/ {v=170}
+/15. STRAIGHT BET\(Milynn Bootleg\)\.mp3/ {v=170}
+/\/13.*\(Milynn Bootleg\)\.mp3/ {v=167;x="way hard"}
+/COLOR PLANET .10SAI . nijikon Remix\)\.mp3/ {v=165}
+/11\. (朝はご飯派|Æ⌐é═é▓ö╤öh)\.mp3/ {v=162}
+/09\. Where.s the core/ {v=160}
+/PLANET\(Koushif Jersey Club Bootleg\)remaster.mp3/ {v=160;x="starts ez turns bs"}
+/kened Soul - Madeon x Angel Beats!.mp3/ {v=160}
+/Dear Moments\(Mother Harlot Bootleg\)\.mp3/ {v=150}
+/POWER.Ringos UKG/ {v=140}
+/ブルー・フィールド\(Ringos UKG Remix\).mp3/ {v=135}
+/プラチナジェット.Ringo Remix..mp3/ {v=131.2}
+/Mirrorball Love \(TKM Bootleg Mix\).mp3/ {v=130}
+/Photon Melodies \(TKM Bootleg Mix\).mp3/ {v=128}
+/Trap of Love \(TKM Bootleg Mix\).mp3/ {v=128}
+/One Step \(TKM Bootleg Mix\)\.mp3/ {v=126}
+/04 (トリカムイ岩|âgâèâJâÇâCèΓ).mp3/ {v=125}
+/Get your Wish \(NAWN REMIX\)\.mp3/ {v=95}
+/Flicker .Nitro Fun/ {v=92}
+/\/14\..*suicat Remix/ {v=85.5;x="tricky"}
+/Yanagi Nagi - Harumodoki \(EO Remix\)\.mp3/ {v=150}
+/Azure - Nicology\.mp3/ {v=128;x="off by 5 how"}
+'
+
+
+# afun host, collects/grades the results
+runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | awk '{v=$1;sub(/[^ ]+ /,"")} '"$br"' {printf "%s %s\n",v,$0}' | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk 'function prt(c) {printf "\033[3%sm%s\033[0m\n",c,$0} $8!="div,"{next} $5!~/^[0-9\.]+/{next} {meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;prt(6);next} idiff==1{n1++;prt(3);next} idiff>10{nx++;prt(1);next} {n10++;prt(5)} END {printf "ok: %d   1off: %2s   (%3s)   10off: %2s   (%3s)   fail: %2s\n",n0,n1,n0+n1,n10,n0+n1+n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; }
+
+
+# ok:   8   1off: 62   ( 70)   10off: 86   (156)   fail: 25   # 105 sec,  librosa @ 8c archvm on 3700x w10
+# ok:   4   1off: 59   ( 63)   10off: 65   (128)   fail: 53   # using original tags (bad)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } runfun
+
+
+# ok: 119   1off:  5   (124)   10off:  8   (132)   fail: 49   # 51 sec,  vamp-example-fixedtempo
+# ok: 109   1off:  4   (113)   10off:  9   (122)   fail: 59   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; }; runfun
+
+
+# ok: 102   1off: 61   (163)   10off: 12   (175)   fail:  6   # 61 sec,  vamp-qm-tempotracker
+# ok:  80   1off: 48   (128)   10off: 11   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; }; runfun
+
+
+# ok: 133   1off: 32   (165)   10off: 12   (177)   fail:  3   # 51 sec,  vamp-beatroot
+# ok: 101   1off: 22   (123)   10off: 16   (139)   fail: 39   # bad-tags
+# note: some tracks fully fail to analyze (unlike the others which always provide a guess)
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; }; runfun
+
+
+# ok: 124   1off:  9   (133)   10off: 40   (173)   fail:  8   # 231 sec,  essentia/full
+# ok: 109   1off:  8   (117)   10off: 22   (139)   fail: 42   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'import essentia; import essentia.standard as es; fe, fef = es.MusicExtractor(lowlevelStats=["mean", "stdev"], rhythmStats=["mean", "stdev"], tonalStats=["mean", "stdev"])("/dev/shm/'$core'.wav"); print("{:.2f}".format(fe["rhythm.bpm"]))')"; }; runfun
+
+
+# ok: 113   1off: 18   (131)   10off: 46   (177)   fail:  4   # 134 sec,  essentia/re2013
+# ok: 101   1off: 15   (116)   10off: 26   (142)   fail: 39   # bad-tags
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'from essentia.standard import *; a=MonoLoader(filename="/dev/shm/'$core'.wav")(); bpm,beats,confidence,_,intervals=RhythmExtractor2013(method="multifeature")(a); print("{:.2f}".format(bpm))')"; }; runfun
+
+
+
+########################################################################
+##
+##  key detectyion
+##
+########################################################################
+
+
+
+# console scriptlet reusing keytabs from browser.js
+var m=''; for (var a=0; a<24; a++) m += 's/\\|(' + maps["traktor_sharps"][a].trim() + "|" + maps["rekobo_classic"][a].trim() + "|" + maps["traktor_musical"][a].trim() + "|" + maps["traktor_open"][a].trim() + ')$/|' + maps["rekobo_alnum"][a].trim() + '/;'; console.log(m);
+
+
+# translate to camelot
+re='s/\|(B|B|B|6d)$/|1B/;s/\|(F#|F#|Gb|7d)$/|2B/;s/\|(C#|Db|Db|8d)$/|3B/;s/\|(G#|Ab|Ab|9d)$/|4B/;s/\|(D#|Eb|Eb|10d)$/|5B/;s/\|(A#|Bb|Bb|11d)$/|6B/;s/\|(F|F|F|12d)$/|7B/;s/\|(C|C|C|1d)$/|8B/;s/\|(G|G|G|2d)$/|9B/;s/\|(D|D|D|3d)$/|10B/;s/\|(A|A|A|4d)$/|11B/;s/\|(E|E|E|5d)$/|12B/;s/\|(G#m|Abm|Abm|6m)$/|1A/;s/\|(D#m|Ebm|Ebm|7m)$/|2A/;s/\|(A#m|Bbm|Bbm|8m)$/|3A/;s/\|(Fm|Fm|Fm|9m)$/|4A/;s/\|(Cm|Cm|Cm|10m)$/|5A/;s/\|(Gm|Gm|Gm|11m)$/|6A/;s/\|(Dm|Dm|Dm|12m)$/|7A/;s/\|(Am|Am|Am|1m)$/|8A/;s/\|(Em|Em|Em|2m)$/|9A/;s/\|(Bm|Bm|Bm|3m)$/|10A/;s/\|(F#m|F#m|Gbm|4m)$/|11A/;s/\|(C#m|Dbm|Dbm|5m)$/|12A/;'
+
+
+# runner/wrapper
+runfun() { cores=8; touch run; tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, key.v from mt key join mt dur on key.w = dur.w where key.k = "key" and dur.k = ".dur" order by dur.w' | uniq -w16 | grep -vE '(Off-Key|None)$' | sed -r "s/ //g;$re" | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done| grep mir/cr | tr \| / | while read key fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); [ "$key" = "$py" ] && c=2 || c=5; printf '%4s sec, %4s orig, \033[3%dm%4s py,\033[0m %s\n' $td "$key" $c "$py" "$fn") || break; done & done; time wait 2>/dev/null; }
+
+
+# ok: 26   1off: 10   2off: 1   fail: 3   #  15 sec, keyfinder
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 -c 'import sys; import keyfinder; print(keyfinder.key(sys.argv[1]).camelot())' "/dev/shm/$core.wav")"; }; runfun
+
+
+# https://github.com/MTG/essentia/raw/master/src/examples/tutorial/example_key_by_steps_streaming.py
+# https://essentia.upf.edu/reference/std_Key.html  # edma edmm braw bgate
+sed -ri 's/^(key = Key\().*/\1profileType="bgate")/' example_key_by_steps_streaming.py
+afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 -t 60 /dev/shm/$core.wav || break; py="$(python3 example_key_by_steps_streaming.py /dev/shm/$core.{wav,yml} 2>/dev/null | sed -r "s/ major//;s/ minor/m/;s/^/|/;$re;s/.//")"; }; runfun
+
+
+
+########################################################################
+##
+##  misc
+##
+########################################################################
+
+
+
+python3 -m pip install --user vamp
+
+import librosa
+d, r = librosa.load('/dev/shm/0.wav')
+d.dtype
+# dtype('float32')
+d.shape
+# (1323000,)
+d
+# array([-1.9614939e-08,  1.8037968e-08, -1.4106059e-08, ...,
+#         1.2024145e-01,  2.7462116e-01,  1.6202132e-01], dtype=float32)
+
+
+
+import vamp
+c = vamp.collect(d, r, "vamp-example-plugins:fixedtempo")
+c
+# {'list': [{'timestamp':  0.005804988, 'duration':  9.999092971, 'label': '110.0 bpm', 'values': array([109.98116], dtype=float32)}]}
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+d
+array([-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+       -0.0267825 , -0.03564296], dtype=float32)
+
+d = np.reshape(d, [1, -1])
+d
+array([[-0.17803933, -0.27206388, -0.41586545, ..., -0.04940119,
+        -0.0267825 , -0.03564296]], dtype=float32)
+
+
+
+import vampyhost
+print("\n".join(vampyhost.list_plugins()))
+
+mvamp:marsyas_bextract_centroid
+mvamp:marsyas_bextract_lpcc
+mvamp:marsyas_bextract_lsp
+mvamp:marsyas_bextract_mfcc
+mvamp:marsyas_bextract_rolloff
+mvamp:marsyas_bextract_scf
+mvamp:marsyas_bextract_sfm
+mvamp:marsyas_bextract_zero_crossings
+mvamp:marsyas_ibt
+mvamp:zerocrossing
+qm-vamp-plugins:qm-adaptivespectrogram
+qm-vamp-plugins:qm-barbeattracker
+qm-vamp-plugins:qm-chromagram
+qm-vamp-plugins:qm-constantq
+qm-vamp-plugins:qm-dwt
+qm-vamp-plugins:qm-keydetector
+qm-vamp-plugins:qm-mfcc
+qm-vamp-plugins:qm-onsetdetector
+qm-vamp-plugins:qm-segmenter
+qm-vamp-plugins:qm-similarity
+qm-vamp-plugins:qm-tempotracker
+qm-vamp-plugins:qm-tonalchange
+qm-vamp-plugins:qm-transcription
+vamp-aubio:aubiomelenergy
+vamp-aubio:aubiomfcc
+vamp-aubio:aubionotes
+vamp-aubio:aubioonset
+vamp-aubio:aubiopitch
+vamp-aubio:aubiosilence
+vamp-aubio:aubiospecdesc
+vamp-aubio:aubiotempo
+vamp-example-plugins:amplitudefollower
+vamp-example-plugins:fixedtempo
+vamp-example-plugins:percussiononsets
+vamp-example-plugins:powerspectrum
+vamp-example-plugins:spectralcentroid
+vamp-example-plugins:zerocrossing
+vamp-rubberband:rubberband
+
+
+
+plug = vampyhost.load_plugin("vamp-example-plugins:fixedtempo", 22050, 0)
+plug.info
+{'apiVersion': 2, 'pluginVersion': 1, 'identifier': 'fixedtempo', 'name': 'Simple Fixed Tempo Estimator', 'description': 'Study a short section of audio and estimate its tempo, assuming the tempo is constant', 'maker': 'Vamp SDK Example Plugins', 'copyright': 'Code copyright 2008 Queen Mary, University of London.  Freely redistributable (BSD license)'}
+plug = vampyhost.load_plugin("qm-vamp-plugins:qm-tempotracker", 22050, 0)
+from pprint import pprint; pprint(plug.parameters)
+
+
+
+for c in plug.parameters: print("{} \033[36m{}  [\033[33m{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], "\033[36m, \033[33m".join(c["valueNames"]), c["valueNames"][int(c["defaultValue"])])) if "valueNames" in c else print("{} \033[36m{}  [\033[33m{}..{}\033[36m] = {}\033[0m".format(c["identifier"], c["name"], c["minValue"], c["maxValue"], c["defaultValue"]))
+
+
+
+beatroot-vamp:beatroot
+cl=c["list"]; 60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"])))
+
+
+
+ffmpeg -ss 48 -i /mnt/Users/ed/Music/mir/cr-a/'I Beg You(ths Bootleg).wav' -ac 1 -ar 22050 -f f32le -t 60 /dev/shm/f32.pcm
+# 128 bpm, key 5A Cm
+
+import vamp
+import numpy as np
+f = open('/dev/shm/f32.pcm', 'rb')
+d = np.fromfile(f, dtype=np.float32)
+c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40})
+c["list"][0]["label"]
+# 127.6 bpm
+
+c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150})
+print("\n".join([v["label"] for v in c["list"] if v["label"]]))
+v = [float(x["label"].split(' ')[0]) for x in c["list"] if x["label"]]
+v = list(sorted(v))[len(v)//4:-len(v)//4]
+v = sum(v) / len(v)
+# 128.1 bpm
+

docs/notes.sh

@@ -11,6 +11,13 @@ gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f --
 gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done
 
 
+##
+## detect partial uploads based on file contents
+##  (in case of context loss or old copyparties)
+
+echo; find -type f | while IFS= read -r x; do printf '\033[A\033[36m%s\033[K\033[0m\n' "$x"; tail -c$((1024*1024)) <"$x" | xxd -a | awk 'NR==1&&/^[0: ]+.{16}$/{next} NR==2&&/^\*$/{next} NR==3&&/^[0f]+: [0 ]+65 +.{16}$/{next} {e=1} END {exit e}' || continue; printf '\033[A\033[31msus:\033[33m %s \033[0m\n\n' "$x"; done
+
+
 ##
 ## create a test payload
 
@@ -60,6 +67,43 @@ wget -S --header='Accept-Encoding: gzip' -U 'MSIE 6.0; SV1' http://127.0.0.1:392
 shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*1024*1024)); printf $(tail -c +$((v+1)) "$f" | head -c $((w-v)) | sha512sum | cut -c-64 | sed -r 's/ .*//;s/(..)/\\x\1/g') | base64 -w0 | cut -c-43 | tr '+/' '-_'; v=$w; [ $v -lt $sz ] || break; done; }
 
 
+##
+## poll url for performance issues
+
+command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
+
+
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+
+##
+## sqlite3 stuff
+
+# find dupe metadata keys
+sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = mt2.k and mt1.rowid != mt2.rowid'
+
+# partial reindex by deleting all tags for a list of files
+time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
+cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
+
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
+
+##
+## media
+
+# split track into test files
+e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
+
+-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
+sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
+
+
 ##
 ## vscode
 
@@ -89,6 +133,19 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 brew install python@2
 pip install virtualenv
 
+# readme toc
+cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
+
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
 
 ##
 ## http 206
@@ -114,7 +171,7 @@ Range: bytes=26-         Content-Range: bytes */26
 
 var tsh = [];
 function convert_markdown(md_text, dest_dom) {
-    tsh.push(new Date().getTime());
+    tsh.push(Date.now());
     while (tsh.length > 10)
         tsh.shift();
     if (tsh.length > 1) {

scripts/copyparty-repack.sh

@@ -20,6 +20,7 @@ set -e
 # -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py
 
 
+command -v gnutar && tar() { gnutar "$@"; }
 command -v gtar && tar() { gtar "$@"; }
 command -v gsed && sed() { gsed "$@"; }
 td="$(mktemp -d)"
@@ -29,11 +30,11 @@ pwd
 
 
 dl_text() {
-	command -v curl && exec curl "$@"
+	command -v curl >/dev/null && exec curl "$@"
 	exec wget -O- "$@"
 }
 dl_files() {
-	command -v curl && exec curl -L --remote-name-all "$@"
+	command -v curl >/dev/null && exec curl -L --remote-name-all "$@"
 	exec wget "$@"
 }
 export -f dl_files

scripts/make-sfx.sh

@@ -28,6 +28,13 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	unexpand() { gunexpand "$@"; }
 	command -v grealpath >/dev/null &&
 		realpath() { grealpath "$@"; }
+
+	[ -e /opt/local/bin/bzip2 ] &&
+		bzip2() { /opt/local/bin/bzip2 "$@"; }
+}
+pybin=$(command -v python3 || command -v python) || {
+	echo need python
+	exit 1
 }
 
 [ -e copyparty/__main__.py ] || cd ..
@@ -38,11 +45,17 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	exit 1
 }
 
+use_gz=
+do_sh=1
+do_py=1
 while [ ! -z "$1" ]; do
 	[ "$1" = clean  ] && clean=1  && shift && continue
 	[ "$1" = re     ] && repack=1 && shift && continue
+	[ "$1" = gz     ] && use_gz=1 && shift && continue
 	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
 	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
+	[ "$1" = no-sh  ] && do_sh=   && shift && continue
+	[ "$1" = no-py  ] && do_py=   && shift && continue
 	break
 done
 
@@ -104,7 +117,7 @@ cd sfx
 ver=
 git describe --tags >/dev/null 2>/dev/null && {
 	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
-	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//; s/-g?/./g')";
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//')";
 	t_ver=
 
 	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
@@ -150,7 +163,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -169,10 +182,11 @@ done
 	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
 }
 
+[ $repack ] ||
 find | grep -E '\.py$' |
   grep -vE '__version__' |
   tr '\n' '\0' |
-  xargs -0 python ../scripts/uncomment.py
+  xargs -0 $pybin ../scripts/uncomment.py
 
 f=dep-j2/jinja2/constants.py
 awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
@@ -180,39 +194,74 @@ tmv "$f"
 
 # up2k goes from 28k to 22k laff
 echo entabbening
-find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
+find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
 
+echo gen tarlist
+for d in copyparty dep-j2; do find $d -type f; done |
+sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
+sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
+
+(grep -vE 'gz$' list1; grep -E 'gz$' list1) >list
+
 echo creating tar
 args=(--owner=1000 --group=1000)
 [ "$OSTYPE" = msys ] &&
 	args=()
 
-tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
+tar -cf tar "${args[@]}" --numeric-owner -T list
+
+pc=bzip2
+pe=bz2
+[ $use_gz ] && pc=gzip && pe=gz
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2
-for n in {2..9}; do cp tar t.$n;  xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz
-rm t.*
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2; }
+[ $do_sh ] && { for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
+rm t.* || true
+exts=()
 
+
+[ $do_sh ] && {
+exts+=(.sh)
 echo creating unix sfx
 (
 	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
 	grep -E '^sfx_eof$' -B 9001;
 	cat tar.xz
 ) >$sfx_out.sh
+}
 
-echo creating generic sfx
-python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
-mv sfx.out $sfx_out.py
-chmod 755 $sfx_out.*
+
+[ $do_py ] && {
+	echo creating generic sfx
+
+	py=../scripts/sfx.py
+	suf=
+	[ $use_gz ] && {
+		sed -r 's/"r:bz2"/"r:gz"/' <$py >$py.t
+		py=$py.t
+		suf=-gz
+	}
+
+	$pybin $py --sfx-make tar.bz2 $ver $ts
+	mv sfx.out $sfx_out$suf.py
+	
+	exts+=($suf.py)
+	[ $use_gz ] &&
+		rm $py
+}
+
+
+chmod 755 $sfx_out*
 
 printf "done:\n"
-printf "  %s\n" "$(realpath $sfx_out)."{sh,py}
-# rm -rf *
+for ext in ${exts[@]}; do
+	printf "  %s\n" "$(realpath $sfx_out)"$ext
+done
 
-# tar -tvf ../sfx/tar | sed -r 's/(.* ....-..-.. ..:.. )(.*)/\2 `` \1/' | sort | sed -r 's/(.*) `` (.*)/\2 \1/'| less
-# for n in {1..9}; do tar -tf tar | grep -vE '/$' | sed -r 's/(.*)\.(.*)/\2.\1/' | sort | sed -r 's/([^\.]+)\.(.*)/\2.\1/' | tar -cT- | bzip2 -c$n | wc -c; done 
+# apk add bash python3 tar xz bzip2
+# while true; do ./make-sfx.sh; for f in ..//dist/copyparty-sfx.{sh,py}; do mv $f $f.$(wc -c <$f | awk '{print$1}'); done; done

scripts/sfx.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
-# coding: utf-8
+# coding: latin-1
 from __future__ import print_function, unicode_literals
 
-import os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
+import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform, tempfile, traceback
 import subprocess as sp
 
 """
@@ -27,21 +27,21 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
+WINDOWS = sys.platform in ["win32", "msys"]
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
-cpp = None
 
 
-def eprint(*args, **kwargs):
-    kwargs["file"] = sys.stderr
-    print(*args, **kwargs)
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    print(*a, **ka)
 
 
-def msg(*args, **kwargs):
-    if args:
-        args = ["[SFX]", args[0]] + list(args[1:])
+def msg(*a, **ka):
+    if a:
+        a = ["[SFX]", a[0]] + list(a[1:])
 
-    eprint(*args, **kwargs)
+    eprint(*a, **ka)
 
 
 # skip 1
@@ -156,6 +156,9 @@ def encode(data, size, cksum, ver, ts):
                 skip = True
                 continue
 
+            if ln.strip().startswith("# fmt: "):
+                continue
+
             unpk += ln + "\n"
 
         for k, v in [
@@ -209,11 +212,11 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    hasher = hashlib.md5()
+    h = hashlib.md5()
     for block in yieldfile(fn):
-        hasher.update(block)
+        h.update(block)
 
-    return hasher.hexdigest()
+    return h.hexdigest()
 
 
 def unpack():
@@ -222,9 +225,10 @@ def unpack():
     tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
-    final = os.path.join(top, name)
-    mine = os.path.join(top, withpid)
-    tar = os.path.join(mine, "tar")
+    opj = os.path.join
+    final = opj(top, name)
+    mine = opj(top, withpid)
+    tar = opj(mine, "tar")
 
     try:
         if tag in os.listdir(final):
@@ -233,28 +237,24 @@ def unpack():
     except:
         pass
 
-    nwrite = 0
+    sz = 0
     os.mkdir(mine)
     with open(tar, "wb") as f:
         for buf in get_payload():
-            nwrite += len(buf)
+            sz += len(buf)
             f.write(buf)
 
-    if nwrite != SIZE:
-        t = "\n\n  bad file:\n    expected {} bytes, got {}\n".format(SIZE, nwrite)
-        raise Exception(t)
-
-    cksum = hashfile(tar)
-    if cksum != CKSUM:
-        t = "\n\n  bad file:\n    {} expected,\n    {} obtained\n".format(CKSUM, cksum)
-        raise Exception(t)
+    ck = hashfile(tar)
+    if ck != CKSUM:
+        t = "\n\nexpected {} ({} byte)\nobtained {} ({} byte)\nsfx corrupt"
+        raise Exception(t.format(CKSUM, SIZE, ck, sz))
 
     with tarfile.open(tar, "r:bz2") as tf:
         tf.extractall(mine)
 
     os.remove(tar)
 
-    with open(os.path.join(mine, tag), "wb") as f:
+    with open(opj(mine, tag), "wb") as f:
         f.write(b"h\n")
 
     try:
@@ -272,25 +272,25 @@ def unpack():
     except:
         pass
 
+    for fn in u8(os.listdir(top)):
+        if fn.startswith(name) and fn != withpid:
+            try:
+                old = opj(top, fn)
+                if time.time() - os.path.getmtime(old) > 86400:
+                    shutil.rmtree(old)
+            except:
+                pass
+
     try:
         os.symlink(mine, final)
     except:
         try:
             os.rename(mine, final)
+            return final
         except:
             msg("reloc fail,", mine)
-            return mine
 
-    for fn in u8(os.listdir(top)):
-        if fn.startswith(name) and fn not in [name, withpid]:
-            try:
-                old = os.path.join(top, fn)
-                if time.time() - os.path.getmtime(old) > 10:
-                    shutil.rmtree(old)
-            except:
-                pass
-
-    return final
+    return mine
 
 
 def get_payload():
@@ -307,96 +307,119 @@ def get_payload():
         if ofs < 0:
             raise Exception("could not find archive marker")
 
-        # start reading from the final b"\n"
+        # start at final b"\n"
         fpos = ofs + len(ptn) - 3
-        # msg("tar found at", fpos)
         f.seek(fpos)
         dpos = 0
-        leftovers = b""
+        rem = b""
         while True:
             rbuf = f.read(1024 * 32)
             if rbuf:
-                buf = leftovers + rbuf
+                buf = rem + rbuf
                 ofs = buf.rfind(b"\n")
                 if len(buf) <= 4:
-                    leftovers = buf
+                    rem = buf
                     continue
 
                 if ofs >= len(buf) - 4:
-                    leftovers = buf[ofs:]
+                    rem = buf[ofs:]
                     buf = buf[:ofs]
                 else:
-                    leftovers = b"\n# "
+                    rem = b"\n# "
             else:
-                buf = leftovers
+                buf = rem
 
             fpos += len(buf) + 1
-            buf = (
-                buf.replace(b"\n# ", b"")
-                .replace(b"\n#r", b"\r")
-                .replace(b"\n#n", b"\n")
-            )
-            dpos += len(buf) - 1
+            for a, b in [[b"\n# ", b""], [b"\n#r", b"\r"], [b"\n#n", b"\n"]]:
+                buf = buf.replace(a, b)
 
+            dpos += len(buf) - 1
             yield buf
 
             if not rbuf:
                 break
 
 
-def confirm():
+def utime(top):
+    i = 0
+    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    while WINDOWS:
+        t = int(time.time())
+        if i:
+            msg("utime {}, {}".format(i, t))
+
+        for f in files:
+            os.utime(f, (t, t))
+
+        i += 1
+        time.sleep(78123)
+
+
+def confirm(rv):
     msg()
+    msg("retcode", rv if rv else traceback.format_exc())
     msg("*** hit enter to exit ***")
     try:
         raw_input() if PY2 else input()
     except:
         pass
 
+    sys.exit(rv)
 
-def run(tmp, j2ver):
-    global cpp
 
-    msg("jinja2:", j2ver or "bundled")
+def run(tmp, j2):
+    msg("jinja2:", j2 or "bundled")
     msg("sfxdir:", tmp)
+    msg()
 
-    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    # block systemd-tmpfiles-clean.timer
     try:
         import fcntl
 
         fd = os.open(tmp, os.O_RDONLY)
         fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
-    except:
-        pass
+    except Exception as ex:
+        if not WINDOWS:
+            msg("\033[31mflock:", repr(ex))
+
+    t = threading.Thread(target=utime, args=(tmp,))
+    t.daemon = True
+    t.start()
 
     ld = [tmp, os.path.join(tmp, "dep-j2")]
-    if j2ver:
+    if j2:
         del ld[-1]
 
-    cmd = (
-        "import sys, runpy; "
-        + "".join(['sys.path.insert(0, r"' + x + '"); ' for x in ld])
-        + 'runpy.run_module("copyparty", run_name="__main__")'
-    )
-    cmd = [sys.executable, "-c", cmd] + list(sys.argv[1:])
-
-    cmd = [str(x) for x in cmd]
-    msg("\n", cmd, "\n")
-    cpp = sp.Popen(cmd)
-    try:
-        cpp.wait()
-    except:
-        cpp.wait()
-
-    if cpp.returncode != 0:
-        confirm()
-
-    sys.exit(cpp.returncode)
+    if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
+        run_s(ld)
+    else:
+        run_i(ld)
 
 
-def bye(sig, frame):
-    if cpp is not None:
-        cpp.terminate()
+def run_i(ld):
+    for x in ld:
+        sys.path.insert(0, x)
+
+    from copyparty.__main__ import main as p
+
+    p()
+
+
+def run_s(ld):
+    # fmt: off
+    c = "import sys,runpy;" + "".join(['sys.path.insert(0,r"' + x + '");' for x in ld]) + 'runpy.run_module("copyparty",run_name="__main__")'
+    c = [str(x) for x in [sys.executable, "-c", c] + list(sys.argv[1:])]
+    # fmt: on
+    msg("\n", c, "\n")
+    p = sp.Popen(c)
+
+    def bye(*a):
+        p.send_signal(signal.SIGINT)
+
+    signal.signal(signal.SIGTERM, bye)
+    p.wait()
+
+    raise SystemExit(p.returncode)
 
 
 def main():
@@ -430,16 +453,23 @@ def main():
 
     # skip 0
 
-    signal.signal(signal.SIGTERM, bye)
-
-    tmp = unpack()
+    tmp = os.path.realpath(unpack())
 
     try:
-        from jinja2 import __version__ as j2ver
+        from jinja2 import __version__ as j2
     except:
-        j2ver = None
+        j2 = None
 
-    return run(tmp, j2ver)
+    try:
+        run(tmp, j2)
+    except SystemExit as ex:
+        c = ex.code
+        if c not in [0, -15]:
+            confirm(ex.code)
+    except KeyboardInterrupt:
+        pass
+    except:
+        confirm(0)
 
 
 if __name__ == "__main__":

tests/run.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import sys
+import runpy
+
+host = sys.argv[1]
+sys.argv = sys.argv[:1] + sys.argv[2:]
+sys.path.insert(0, ".")
+
+
+def rp():
+    runpy.run_module("unittest", run_name="__main__")
+
+
+if host == "vmprof":
+    rp()
+
+elif host == "cprofile":
+    import cProfile
+    import pstats
+
+    log_fn = "cprofile.log"
+    cProfile.run("rp()", log_fn)
+    p = pstats.Stats(log_fn)
+    p.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(64)
+
+
+"""
+python3.9 tests/run.py cprofile -v tests/test_httpcli.py
+
+python3.9 -m pip install --user vmprof
+python3.9 -m vmprof --lines -o vmprof.log tests/run.py vmprof -v tests/test_httpcli.py
+"""

tests/test_httpcli.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import io
+import os
+import time
+import shutil
+import pprint
+import tarfile
+import unittest
+
+from argparse import Namespace
+from copyparty.authsrv import AuthSrv
+from copyparty.httpcli import HttpCli
+
+from tests import util as tu
+
+
+def hdr(query):
+    h = "GET /{} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\n\r\n"
+    return h.format(query).encode("utf-8")
+
+
+class Cfg(Namespace):
+    def __init__(self, a=[], v=[], c=None):
+        super(Cfg, self).__init__(
+            a=a,
+            v=v,
+            c=c,
+            ed=False,
+            no_zip=False,
+            no_scandir=False,
+            no_sendfile=True,
+            nih=True,
+            mtp=[],
+            mte="a",
+            **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        )
+
+
+class TestHttpCli(unittest.TestCase):
+    def test(self):
+        td = os.path.join(tu.get_ramdisk(), "vfs")
+        try:
+            shutil.rmtree(td)
+        except OSError:
+            pass
+
+        os.mkdir(td)
+        os.chdir(td)
+
+        self.dtypes = ["ra", "ro", "rx", "wa", "wo", "wx", "aa", "ao", "ax"]
+        self.can_read = ["ra", "ro", "aa", "ao"]
+        self.can_write = ["wa", "wo", "aa", "ao"]
+        self.fn = "g{:x}g".format(int(time.time() * 3))
+
+        allfiles = []
+        allvols = []
+        for top in self.dtypes:
+            allvols.append(top)
+            allfiles.append("/".join([top, self.fn]))
+            for s1 in self.dtypes:
+                p = "/".join([top, s1])
+                allvols.append(p)
+                allfiles.append(p + "/" + self.fn)
+                allfiles.append(p + "/n/" + self.fn)
+                for s2 in self.dtypes:
+                    p = "/".join([top, s1, "n", s2])
+                    os.makedirs(p)
+                    allvols.append(p)
+                    allfiles.append(p + "/" + self.fn)
+
+        for fp in allfiles:
+            with open(fp, "w") as f:
+                f.write("ok {}\n".format(fp))
+
+        for top in self.dtypes:
+            vcfg = []
+            for vol in allvols:
+                if not vol.startswith(top):
+                    continue
+
+                mode = vol[-2]
+                usr = vol[-1]
+                if usr == "a":
+                    usr = ""
+
+                if "/" not in vol:
+                    vol += "/"
+
+                top, sub = vol.split("/", 1)
+                vcfg.append("{0}/{1}:{1}:{2}{3}".format(top, sub, mode, usr))
+
+            pprint.pprint(vcfg)
+
+            self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
+            self.auth = AuthSrv(self.args, self.log)
+            vfiles = [x for x in allfiles if x.startswith(top)]
+            for fp in vfiles:
+                rok, wok = self.can_rw(fp)
+                furl = fp.split("/", 1)[1]
+                durl = furl.rsplit("/", 1)[0] if "/" in furl else ""
+
+                # file download
+                h, ret = self.curl(furl)
+                res = "ok " + fp in ret
+                print("[{}] {} {} = {}".format(fp, rok, wok, res))
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, furl))
+                    self.fail()
+
+                # file browser: html
+                h, ret = self.curl(durl)
+                res = "'{}'".format(self.fn) in ret
+                print(res)
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, durl))
+                    self.fail()
+
+                # file browser: json
+                url = durl + "?ls"
+                h, ret = self.curl(url)
+                res = '"{}"'.format(self.fn) in ret
+                print(res)
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, url))
+                    self.fail()
+
+                # tar
+                url = durl + "?tar"
+                h, b = self.curl(url, True)
+                # with open(os.path.join(td, "tar"), "wb") as f:
+                #    f.write(b)
+                try:
+                    tar = tarfile.open(fileobj=io.BytesIO(b)).getnames()
+                except:
+                    tar = []
+                tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
+                tar = [[x] + self.can_rw(x) for x in tar]
+                tar_ok = [x[0] for x in tar if x[1]]
+                tar_ng = [x[0] for x in tar if not x[1]]
+                self.assertEqual([], tar_ng)
+
+                if durl.split("/")[-1] in self.can_read:
+                    ref = [x for x in vfiles if self.in_dive(top + "/" + durl, x)]
+                    for f in ref:
+                        print("{}: {}".format("ok" if f in tar_ok else "NG", f))
+                    ref.sort()
+                    tar_ok.sort()
+                    self.assertEqual(ref, tar_ok)
+
+                # stash
+                h, ret = self.put(url)
+                res = h.startswith("HTTP/1.1 200 ")
+                self.assertEqual(res, wok)
+
+    def can_rw(self, fp):
+        # lowest non-neutral folder declares permissions
+        expect = fp.split("/")[:-1]
+        for x in reversed(expect):
+            if x != "n":
+                expect = x
+                break
+
+        return [expect in self.can_read, expect in self.can_write]
+
+    def in_dive(self, top, fp):
+        # archiver bails at first inaccessible subvolume
+        top = top.strip("/").split("/")
+        fp = fp.split("/")
+        for f1, f2 in zip(top, fp):
+            if f1 != f2:
+                return False
+
+        for f in fp[len(top) :]:
+            if f == self.fn:
+                return True
+            if f not in self.can_read and f != "n":
+                return False
+
+        return True
+
+    def put(self, url):
+        buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
+        buf = buf.format(url, len(url) + 4).encode("utf-8")
+        conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
+        HttpCli(conn).run()
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def curl(self, url, binary=False):
+        conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
+        HttpCli(conn).run()
+        if binary:
+            h, b = conn.s._reply.split(b"\r\n\r\n", 1)
+            return [h.decode("utf-8"), b]
+
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def log(self, src, msg, c=0):
+        # print(repr(msg))
+        pass

tests/test_vfs.py

@@ -3,22 +3,24 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import time
 import json
 import shutil
 import tempfile
 import unittest
-import subprocess as sp  # nosec
 
 from textwrap import dedent
 from argparse import Namespace
 from copyparty.authsrv import AuthSrv
 from copyparty import util
 
+from tests import util as tu
+
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
 
 
@@ -49,52 +51,11 @@ class TestVFS(unittest.TestCase):
         real = [x[0] for x in real]
         return fsdir, real, virt
 
-    def runcmd(self, *argv):
-        p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
-        stdout, stderr = p.communicate()
-        stdout = stdout.decode("utf-8")
-        stderr = stderr.decode("utf-8")
-        return [p.returncode, stdout, stderr]
-
-    def chkcmd(self, *argv):
-        ok, sout, serr = self.runcmd(*argv)
-        if ok != 0:
-            raise Exception(serr)
-
-        return sout, serr
-
-    def get_ramdisk(self):
-        for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
-            if os.path.exists(vol):
-                return vol
-
-        if os.path.exists("/Volumes"):
-            devname, _ = self.chkcmd("hdiutil", "attach", "-nomount", "ram://8192")
-            devname = devname.strip()
-            print("devname: [{}]".format(devname))
-            for _ in range(10):
-                try:
-                    _, _ = self.chkcmd(
-                        "diskutil", "eraseVolume", "HFS+", "cptd", devname
-                    )
-                    return "/Volumes/cptd"
-                except Exception as ex:
-                    print(repr(ex))
-                    time.sleep(0.25)
-
-            raise Exception("ramdisk creation failed")
-
-        ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
-        try:
-            os.mkdir(ret)
-        finally:
-            return ret
-
-    def log(self, src, msg):
+    def log(self, src, msg, c=0):
         pass
 
     def test(self):
-        td = os.path.join(self.get_ramdisk(), "vfs")
+        td = os.path.join(tu.get_ramdisk(), "vfs")
         try:
             shutil.rmtree(td)
         except OSError:
@@ -266,7 +227,7 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(list(v1), list(v2))
 
         # config file parser
-        cfg_path = os.path.join(self.get_ramdisk(), "test.cfg")
+        cfg_path = os.path.join(tu.get_ramdisk(), "test.cfg")
         with open(cfg_path, "wb") as f:
             f.write(
                 dedent(

tests/util.py

@@ -0,0 +1,97 @@
+import os
+import time
+import jinja2
+import tempfile
+import subprocess as sp
+
+from copyparty.util import Unrecv
+
+
+J2_ENV = jinja2.Environment(loader=jinja2.BaseLoader)
+J2_FILES = J2_ENV.from_string("{{ files|join('\n') }}")
+
+
+def runcmd(*argv):
+    p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode("utf-8")
+    stderr = stderr.decode("utf-8")
+    return [p.returncode, stdout, stderr]
+
+
+def chkcmd(*argv):
+    ok, sout, serr = runcmd(*argv)
+    if ok != 0:
+        raise Exception(serr)
+
+    return sout, serr
+
+
+def get_ramdisk():
+    for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
+        if os.path.exists(vol):
+            return vol
+
+    if os.path.exists("/Volumes"):
+        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://32768")
+        devname = devname.strip()
+        print("devname: [{}]".format(devname))
+        for _ in range(10):
+            try:
+                _, _ = chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
+                return "/Volumes/cptd"
+            except Exception as ex:
+                print(repr(ex))
+                time.sleep(0.25)
+
+        raise Exception("ramdisk creation failed")
+
+    ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
+    try:
+        os.mkdir(ret)
+    finally:
+        return ret
+
+
+class NullBroker(object):
+    def put(*args):
+        pass
+
+
+class VSock(object):
+    def __init__(self, buf):
+        self._query = buf
+        self._reply = b""
+        self.sendall = self.send
+
+    def recv(self, sz):
+        ret = self._query[:sz]
+        self._query = self._query[sz:]
+        return ret
+
+    def send(self, buf):
+        self._reply += buf
+        return len(buf)
+
+
+class VHttpSrv(object):
+    def __init__(self):
+        self.broker = NullBroker()
+
+        aliases = ["splash", "browser", "browser2", "msg", "md", "mde"]
+        self.j2 = {x: J2_FILES for x in aliases}
+
+
+class VHttpConn(object):
+    def __init__(self, args, auth, log, buf):
+        self.s = VSock(buf)
+        self.sr = Unrecv(self.s)
+        self.addr = ("127.0.0.1", "42069")
+        self.args = args
+        self.auth = auth
+        self.log_func = log
+        self.log_src = "a"
+        self.hsrv = VHttpSrv()
+        self.nbyte = 0
+        self.workload = 0
+        self.t0 = time.time()