v0.10.19

.vscode/launch.json

@@ -14,6 +14,8 @@
                 "-emp",
                 "-e2dsa",
                 "-e2ts",
+                "-mtp",
+                ".bpm=f,bin/mtag/audio-bpm.py",
                 "-a",
                 "ed:wark",
                 "-v",

.vscode/launch.py

@@ -0,0 +1,35 @@
+# takes arguments from launch.json
+# is used by no_dbg in tasks.json
+# launches 10x faster than mspython debugpy
+# and is stoppable with ^C
+
+import os
+import sys
+import shlex
+
+sys.path.insert(0, os.getcwd())
+
+import jstyleson
+from copyparty.__main__ import main as copyparty
+
+with open(".vscode/launch.json", "r", encoding="utf-8") as f:
+    tj = f.read()
+
+oj = jstyleson.loads(tj)
+argv = oj["configurations"][0]["args"]
+
+try:
+    sargv = " ".join([shlex.quote(x) for x in argv])
+    print(sys.executable + " -m copyparty " + sargv + "\n")
+except:
+    pass
+
+argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
+try:
+    copyparty(["a"] + argv)
+except SystemExit as ex:
+    if ex.code:
+        raise
+
+print("\n\033[32mokke\033[0m")
+sys.exit(1)

.vscode/tasks.json

@@ -9,9 +9,7 @@
         {
             "label": "no_dbg",
             "type": "shell",
-            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -e2ts -a ed:wark -v srv::r:aed:cnodupe -v dist:dist:r ;exit 1"
-            // -v ~/Music/mt:mt:r:cmtp=.bpm=~/dev/copyparty/bin/mtag/audio-bpm.py:cmtp=key=~/dev/copyparty/bin/mtag/audio-key.py:ce2tsr 
-            // -v ~/Music/mt:mt:r:cmtp=.bpm=~/dev/copyparty/bin/mtag/audio-bpm.py:ce2tsr
+            "command": "${config:python.pythonPath} .vscode/launch.py"
         }
     ]
 }

README.md

@@ -12,6 +12,8 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
 * code standard: `black`
 
+📷 screenshots: [browser](#the-browser) // [upload](#uploading) // [md-viewer](#markdown-viewer) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [ie4](#browser-support)
+
 
 ## readme toc
 
@@ -20,13 +22,24 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [notes](#notes)
     * [status](#status)
 * [bugs](#bugs)
-* [usage](#usage)
+    * [not my bugs](#not-my-bugs)
+* [the browser](#the-browser)
+    * [tabs](#tabs)
+    * [hotkeys](#hotkeys)
+    * [tree-mode](#tree-mode)
+    * [zip downloads](#zip-downloads)
+    * [uploading](#uploading)
+        * [file-search](#file-search)
+    * [markdown viewer](#markdown-viewer)
+    * [other tricks](#other-tricks)
 * [searching](#searching)
     * [search configuration](#search-configuration)
     * [metadata from audio files](#metadata-from-audio-files)
     * [file parser plugins](#file-parser-plugins)
     * [complete examples](#complete-examples)
+* [browser support](#browser-support)
 * [client examples](#client-examples)
+* [up2k](#up2k)
 * [dependencies](#dependencies)
     * [optional gpl stuff](#optional-gpl-stuff)
 * [sfx](#sfx)
@@ -51,9 +64,9 @@ you may also want these, especially on servers:
 ## notes
 
 * iPhone/iPad: use Firefox to download files
-* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
-* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
-* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
+* Android-Firefox: takes a while to select files (their fix for ☝️)
+* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
 * paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
   * because no browsers currently implement the media-query to do this properly orz
 
@@ -72,7 +85,7 @@ you may also want these, especially on servers:
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ folders as zip / tar files
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view
@@ -95,24 +108,140 @@ summary: it works! you can use it! (but technically not even close to beta)
 
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
+* Windows: python 2.7 cannot handle filenames with mojibake
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
+## not my bugs
 
-# usage
+* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
+  * this is an msys2 bug, the regular windows edition of python is fine
+
+
+# the browser
+
+![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/115978054-65106380-a57d-11eb-98f8-59e3dee73557.png)
+
+
+## tabs
+
+* `[🔎]` search by size, date, path/name, mp3-tags ... see [searching](#searching)
+* `[🚀]` and `[🎈]` are the uploaders, see [uploading](#uploading)
+* `[📂]` mkdir, create directories
+* `[📝]` new-md, create a new markdown document
+* `[📟]` send-msg, either to server-log or into textfiles if `--urlform save`
+* `[⚙️]` client configuration options
+
+
+## hotkeys
 
 the browser has the following hotkeys
-* `0..9` jump to 10%..90%
-* `U/O` skip 10sec back/forward
-* `J/L` prev/next song
 * `I/K` prev/next folder
 * `P` parent folder
+* when playing audio:
+  * `0..9` jump to 10%..90%
+  * `U/O` skip 10sec back/forward
+  * `J/L` prev/next song
+    * `J` also starts playing the folder
+
+
+## tree-mode
+
+by default there's a breadcrumbs path; you can replace this with a tree-browser sidebar thing by clicking the 🌲
+
+click `[-]` and `[+]` to adjust the size, and the `[a]` toggles if the tree should widen dynamically as you go deeper or stay fixed-size
+
+
+## zip downloads
+
+the `zip` link next to folders can produce various types of zip/tar files using these alternatives in the browser settings tab:
+
+| name | url-suffix | description |
+|--|--|--|
+| `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
+| `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
+| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
+
+* hidden files (dotfiles) are excluded unless `-ed`
+  * the up2k.db is always excluded
+* `zip_crc` will take longer to download since the server has to read each file twice
+  * please let me know if you find a program old enough to actually need this
+
+you can also zip a selection of files or folders by clicking them in the browser, that brings up a selection editor and zip button in the bottom right
+
+![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/116008321-372a2e00-a614-11eb-9a4a-4a1fd9074224.png)
+
+## uploading
+
+two upload methods are available in the html client:
+* 🎈 bup, the basic uploader, supports almost every browser since netscape 4.0
+* 🚀 up2k, the fancy one
+
+up2k has several advantages:
+* you can drop folders into the browser (files are added recursively)
+* files are processed in chunks, and each chunk is checksummed
+  * uploads resume if they are interrupted (for example by a reboot)
+  * server detects any corruption; the client reuploads affected chunks
+  * the client doesn't upload anything that already exists on the server
+* the last-modified timestamp of the file is preserved
+
+see [up2k](#up2k) for details on how it works
+
+![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/115978061-680b5400-a57d-11eb-9ef6-cbb5f60aeccc.png)
+
+the up2k UI is the epitome of polished inutitive experiences:
+* "parallel uploads" specifies how many chunks to upload at the same time
+* `[🏃]` analysis of other files should continue while one is uploading
+* `[💭]` ask for confirmation before files are added to the list
+* `[💤]` sync uploading between other copyparty tabs so only one is active
+* `[🔎]` switch between upload and file-search mode
+
+and then theres the tabs below it,
+* `[ok]` is uploads which completed successfully
+* `[ng]` is the uploads which failed / got rejected (already exists, ...)
+* `[done]` shows a combined list of `[ok]` and `[ng]`, chronological order
+* `[busy]` files which are currently hashing, pending-upload, or uploading
+  * plus up to 3 entries each from `[done]` and `[que]` for context
+* `[que]` is all the files that are still queued
+
+protip: you can avoid scaring away users by hiding some of the UI with hacks like [docs/minimal-up2k.html](docs/minimal-up2k.html)
+
+### file-search
+
+![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/116008320-36919780-a614-11eb-803f-04162326a700.png)
+
+in the 🚀 up2k tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere already
+
+files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
+* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else
+
+adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files
+
+note that since up2k has to read the file twice, 🎈 bup can be up to 2x faster if your internet connection is faster than the read-speed of your HDD
+
+up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well thanks to tls also functioning as an integrity check
+
+
+## markdown viewer
+
+![copyparty-md-read-fs8](https://user-images.githubusercontent.com/241032/115978057-66419080-a57d-11eb-8539-d2be843991aa.png)
+
+* the document preview has a max-width which is the same as an A4 paper when printed
+
+
+## other tricks
+
+* you can link a particular timestamp in an audio file by adding it to the URL, such as `&20` / `&20s` / `&1m20` / `&t=1:20` after the `.../#af-c8960dab`
 
 
 # searching
 
+![copyparty-search-fs8](https://user-images.githubusercontent.com/241032/115978060-6772bd80-a57d-11eb-81d3-174e869b72c3.png)
+
 when started with `-e2dsa` copyparty will scan/index all your files. This avoids duplicates on upload, and also makes the volumes searchable through the web-ui:
 * make search queries by `size`/`date`/`directory-path`/`filename`, or...
-* drag/drop a local file to see if the same contents exist somewhere on the server (you get the URL if it does)
+* drag/drop a local file to see if the same contents exist somewhere on the server, see [file-search](#file-search)
 
 path/name queries are space-separated, AND'ed together, and words are negated with a `-` prefix, so for example:
 * path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
@@ -176,6 +305,43 @@ copyparty can invoke external programs to collect additional metadata for files
   `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
 
 
+# browser support
+
+![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/118192791-fb31fe00-b446-11eb-9647-898ea8efc1f7.png)
+
+`ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
+
+| feature         | ie6 | ie9 | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
+| --------------- | --- | --- | ---- | ---- | ----- | ---- | --- | ---- |
+| browse files    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| basic uploader  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| make directory  | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| send message    | yep | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| set sort order  |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| zip selection   |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| directory tree  |  -  |  -  | `*1` | yep  | yep   | yep  | yep | yep  |
+| up2k            |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| icons work      |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown editor |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| markdown viewer |  -  |  -  | yep  | yep  | yep   | yep  | yep | yep  |
+| play mp3/m4a    |  -  | yep | yep  | yep  | yep   | yep  | yep | yep  |
+| play ogg/opus   |  -  |  -  |  -   |  -   | yep   | yep  | `*2` | yep |
+
+* internet explorer 6 to 8 behave the same
+* firefox 52 and chrome 49 are the last winxp versions
+* `*1` only public folders (login session is dropped) and no history / back-button
+* `*2` using a wasm decoder which can sometimes get stuck and consumes a bit more power
+
+quick summary of more eccentric web-browsers trying to view a directory index:
+* safari (14.0.3/macos) is chrome with janky wasm, so playing opus can deadlock the javascript engine
+* safari (14.0.1/iOS) same as macos, except it recovers from the deadlocks if you poke it a bit
+* links (2.21/macports) can browse, login, upload/mkdir/msg
+* lynx (2.8.9/macports) can browse, login, upload/mkdir/msg
+* w3m (0.5.3/macports) can browse, login, upload at 100kB/s, mkdir/msg
+* netsurf (3.10/arch) is basically ie6 with much better css (javascript has almost no effect)
+* ie4 and netscape 4.0 can browse (text is yellow on white), upload with `?b=u`
+* SerenityOS (22d13d8) hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying
+
 # client examples
 
 * javascript: dump some state into a file (two separate examples)
@@ -200,6 +366,22 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
     b512 <movie.mkv
 
 
+# up2k
+
+quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+* the up2k client splits a file into an "optimal" number of chunks
+  * 1 MiB each, unless that becomes more than 256 chunks
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256# or chunksize >= 32M
+* client posts the list of hashes, filename, size, last-modified
+* server creates the `wark`, an identifier for this upload
+  * `sha512( salt + filesize + chunk_hashes )`
+  * and a sparse file is created for the chunks to drop into
+* client uploads each chunk
+  * header entries for the chunk-hash and wark
+  * server writes chunks into place based on the hash
+* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+
+
 # dependencies
 
 * `jinja2` (is built into the SFX)
@@ -216,12 +398,12 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
 
 some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
 
-these are standalone and will never be imported / evaluated by copyparty
+these are standalone programs and will never be imported / evaluated by copyparty
 
 
 # sfx
 
-currently there are two self-contained binaries:
+currently there are two self-contained "binaries":
 * [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
 * [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
 
@@ -283,15 +465,20 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
+* separate sqlite table per tag
+* audio fingerprinting
+* readme.md as epilogue
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* drop onto folders
 * `os.copy_file_range` for up2k cloning
-* up2k partials ui
 * support pillow-simd
-* cache sha512 chunks on client
-* comment field
-* ~~look into android thumbnail cache file format~~ bad idea
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||
+
+discarded ideas
+
+* up2k partials ui
+* cache sha512 chunks on client
+* comment field
+* look into android thumbnail cache file format

copyparty/__init__.py

@@ -16,12 +16,17 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 
 class EnvParams(object):
     def __init__(self):
         self.mod = os.path.dirname(os.path.realpath(__file__))
+        if self.mod.endswith("__init__"):
+            self.mod = os.path.dirname(self.mod)
+
         if sys.platform == "win32":
             self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
         elif sys.platform == "darwin":

copyparty/__main__.py

@@ -12,7 +12,6 @@ import re
 import os
 import sys
 import time
-import signal
 import shutil
 import filecmp
 import locale
@@ -56,6 +55,12 @@ class RiceFormatter(argparse.HelpFormatter):
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
+class Dodge11874(RiceFormatter):
+    def __init__(self, *args, **kwargs):
+        kwargs["width"] = 9003
+        super(Dodge11874, self).__init__(*args, **kwargs)
+
+
 def warn(msg):
     print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
@@ -167,7 +172,7 @@ def configure_ssl_ciphers(al):
         sys.exit(0)
 
 
-def sighandler(signal=None, frame=None):
+def sighandler(sig=None, frame=None):
     msg = [""] * 5
     for th in threading.enumerate():
         msg.append(str(th))
@@ -177,34 +182,9 @@ def sighandler(signal=None, frame=None):
     print("\n".join(msg))
 
 
-def main():
-    time.strptime("19970815", "%Y%m%d")  # python#7980
-    if WINDOWS:
-        os.system("rem")  # enables colors
-
-    desc = py_desc().replace("[", "\033[1;30m[")
-
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
-
-    ensure_locale()
-    if HAVE_SSL:
-        ensure_cert()
-
-    deprecated = [["-e2s", "-e2ds"]]
-    for dk, nk in deprecated:
-        try:
-            idx = sys.argv.index(dk)
-        except:
-            continue
-
-        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
-        sys.argv[idx] = nk
-        time.sleep(2)
-
+def run_argparse(argv, formatter):
     ap = argparse.ArgumentParser(
-        formatter_class=RiceFormatter,
+        formatter_class=formatter,
         prog="copyparty",
         description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
         epilog=dedent(
@@ -216,6 +196,9 @@ def main():
             
             list of cflags:
               "cnodupe" rejects existing files (instead of symlinking them)
+              "ce2d" sets -e2d (all -e2* args can be set using ce2* cflags)
+              "cd2t" disables metadata collection, overrides -e2t*
+              "cd2d" disables all database stuff, overrides -e2*
 
             example:\033[35m
               -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
@@ -261,8 +244,11 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
     ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
+    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 
@@ -289,9 +275,44 @@ def main():
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     
-    al = ap.parse_args()
+    return ap.parse_args(args=argv[1:])
     # fmt: on
 
+
+def main(argv=None):
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if WINDOWS:
+        os.system("rem")  # enables colors
+
+    if argv is None:
+        argv = sys.argv
+
+    desc = py_desc().replace("[", "\033[1;30m[")
+
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
+    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+
+    ensure_locale()
+    if HAVE_SSL:
+        ensure_cert()
+
+    deprecated = [["-e2s", "-e2ds"]]
+    for dk, nk in deprecated:
+        try:
+            idx = argv.index(dk)
+        except:
+            continue
+
+        msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
+        print(msg.format(dk, nk))
+        argv[idx] = nk
+        time.sleep(2)
+
+    try:
+        al = run_argparse(argv, RiceFormatter)
+    except AssertionError:
+        al = run_argparse(argv, Dodge11874)
+
     # propagate implications
     for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 9, 13)
-CODENAME = "the strongest music server"
-BUILD_DT = (2021, 3, 23)
+VERSION = (0, 10, 19)
+CODENAME = "zip it"
+BUILD_DT = (2021, 5, 14)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import os
 import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
@@ -53,6 +54,7 @@ class VFS(object):
                 self.uwrite,
                 self.flags,
             )
+            self._trk(vn)
             self.nodes[name] = vn
             return self._trk(vn.add(src, dst))
 
@@ -109,7 +111,27 @@ class VFS(object):
         if rem:
             rp += "/" + rem
 
+        try:
             return fsdec(os.path.realpath(fsenc(rp)))
+        except:
+            if not WINDOWS:
+                raise
+
+            # cpython bug introduced in 3.8, still exists in 3.9.1;
+            # some win7sp1 and win10:20H2 boxes cannot realpath a
+            # networked drive letter such as b"n:" or b"n:\\"
+            #
+            # requirements to trigger:
+            #  * bytestring (not unicode str)
+            #  * just the drive letter (subfolders are ok)
+            #  * networked drive (regular disks and vmhgfs are ok)
+            #  * on an enterprise network (idk, cannot repro with samba)
+            #
+            # hits the following exceptions in succession:
+            #  * access denied at L601: "path = _getfinalpathname(path)"
+            #  * "cant concat str to bytes" at L621: "return path + tail"
+            #
+            return os.path.realpath(rp)
 
     def ls(self, rem, uname, scandir, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
@@ -119,7 +141,12 @@ class VFS(object):
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
-                if uname in vn2.uread or "*" in vn2.uread:
+                if (
+                    uname in vn2.uread
+                    or "*" in vn2.uread
+                    or uname in vn2.uwrite
+                    or "*" in vn2.uwrite
+                ):
                     virt_vis[name] = vn2
 
             # no vfs nodes in the list of real inodes
@@ -127,6 +154,78 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, vrem, flt, uname, dots, scandir):
+        if flt:
+            flt = {k: True for k in flt}
+
+        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+            if flt:
+                files = [x for x in files if x[0] in flt]
+
+                rm = [x for x in rd if x[0] not in flt]
+                [rd.remove(x) for x in rm]
+
+                rm = [x for x in vd.keys() if x not in flt]
+                [vd.pop(x) for x in rm]
+
+                flt = None
+
+            # print(repr([vpath, apath, [x[0] for x in files]]))
+            fnames = [n[0] for n in files]
+            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+            apaths = [os.path.join(apath, n) for n in fnames]
+            files = list(zip(vpaths, apaths, files))
+
+            if not dots:
+                # dotfile filtering based on vpath (intended visibility)
+                files = [x for x in files if "/." not in "/" + x[0]]
+
+                rm = [x for x in rd if x[0].startswith(".")]
+                for x in rm:
+                    rd.remove(x)
+
+                rm = [k for k in vd.keys() if k.startswith(".")]
+                for x in rm:
+                    del vd[x]
+
+            # up2k filetring based on actual abspath
+            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+
+            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
+                yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)
@@ -147,6 +246,7 @@ class AuthSrv(object):
         self.args = args
         self.log_func = log_func
         self.warn_anonwrite = warn_anonwrite
+        self.line_ctr = 0
 
         if WINDOWS:
             self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -159,12 +259,6 @@ class AuthSrv(object):
     def log(self, msg, c=0):
         self.log_func("auth", msg, c)
 
-    def invert(self, orig):
-        if PY2:
-            return {v: k for k, v in orig.iteritems()}
-        else:
-            return {v: k for k, v in orig.items()}
-
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
         it = iter(iterable)
@@ -178,7 +272,9 @@ class AuthSrv(object):
     def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount):
         vol_src = None
         vol_dst = None
+        self.line_ctr = 0
         for ln in [x.decode("utf-8").strip() for x in fd]:
+            self.line_ctr += 1
             if not ln and vol_src is not None:
                 vol_src = None
                 vol_dst = None
@@ -208,7 +304,12 @@ class AuthSrv(object):
                 mflags[vol_dst] = {}
                 continue
 
+            if len(ln) > 1:
                 lvl, uname = ln.split(" ")
+            else:
+                lvl = ln
+                uname = "*"
+
             self._read_vol_str(
                 lvl, uname, mread[vol_dst], mwrite[vol_dst], mflags[vol_dst]
             )
@@ -286,7 +387,12 @@ class AuthSrv(object):
         if self.args.c:
             for cfg_fn in self.args.c:
                 with open(cfg_fn, "rb") as f:
+                    try:
                         self._parse_config_file(f, user, mread, mwrite, mflags, mount)
+                    except:
+                        m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
+                        print(m.format(cfg_fn, self.line_ctr))
+                        raise
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
@@ -421,7 +527,7 @@ class AuthSrv(object):
         with self.mutex:
             self.vfs = vfs
             self.user = user
-            self.iuser = self.invert(user)
+            self.iuser = {v: k for k, v in user.items()}
 
         # import pprint
         # pprint.pprint({"usr": user, "rd": mread, "wr": mwrite, "mnt": mount})

copyparty/broker_mp.py

@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()

copyparty/httpcli.py

@@ -7,13 +7,16 @@ import gzip
 import time
 import copy
 import json
+import string
 import socket
 import ctypes
 from datetime import datetime
 import calendar
 
-from .__init__ import E, PY2, WINDOWS
+from .__init__ import E, PY2, WINDOWS, ANYWIN
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .szip import StreamZip
+from .star import StreamTar
 
 if not PY2:
     unicode = str
@@ -52,6 +55,10 @@ class HttpCli(object):
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def j2(self, name, **kwargs):
+        tpl = self.conn.hsrv.j2[name]
+        return tpl.render(**kwargs) if kwargs else tpl
+
     def run(self):
         """returns true if connection can be reused"""
         self.keepalive = False
@@ -67,7 +74,7 @@ class HttpCli(object):
                 headerlines.pop(0)
 
             try:
-                self.mode, self.req, _ = headerlines[0].split(" ")
+                self.mode, self.req, self.http_ver = headerlines[0].split(" ")
             except:
                 raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
 
@@ -86,30 +93,13 @@ class HttpCli(object):
             self.headers[k.lower()] = v.strip()
 
         v = self.headers.get("connection", "").lower()
-        self.keepalive = not v.startswith("close")
+        self.keepalive = not v.startswith("close") and self.http_ver != "HTTP/1.0"
 
         v = self.headers.get("x-forwarded-for", None)
         if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
             self.ip = v.split(",")[0]
             self.log_src = self.conn.set_rproxy(self.ip)
 
-        self.uname = "*"
-        if "cookie" in self.headers:
-            cookies = self.headers["cookie"].split(";")
-            for k, v in [x.split("=", 1) for x in cookies]:
-                if k.strip() != "cppwd":
-                    continue
-
-                v = unescape_cookie(v)
-                if v in self.auth.iuser:
-                    self.uname = self.auth.iuser[v]
-
-                break
-
-        if self.uname:
-            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
-            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
-
         # split req into vpath + uparam
         uparam = {}
         if "?" not in self.req:
@@ -130,13 +120,33 @@ class HttpCli(object):
                 else:
                     uparam[k.lower()] = False
 
+        self.ouparam = {k: v for k, v in uparam.items()}
+
+        cookies = self.headers.get("cookie") or {}
+        if cookies:
+            cookies = [x.split("=", 1) for x in cookies.split(";") if "=" in x]
+            cookies = {k.strip(): unescape_cookie(v) for k, v in cookies}
+            for kc, ku in [["cppwd", "pw"], ["b", "b"]]:
+                if kc in cookies and ku not in uparam:
+                    uparam[ku] = cookies[kc]
+
         self.uparam = uparam
+        self.cookies = cookies
         self.vpath = unquotep(vpath)
 
+        pwd = uparam.get("pw")
+        self.uname = self.auth.iuser.get(pwd, "*")
+        if self.uname:
+            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
+            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
+
         ua = self.headers.get("user-agent", "")
-        if ua.startswith("rclone/"):
+        self.is_rclone = ua.startswith("rclone/")
+        if self.is_rclone:
             uparam["raw"] = False
             uparam["dots"] = False
+            uparam["b"] = False
+            cookies["b"] = False
 
         try:
             if self.mode in ["GET", "HEAD"]:
@@ -153,14 +163,18 @@ class HttpCli(object):
         except Pebkac as ex:
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
-                self.keepalive = self._check_nonfatal(ex)
-                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
+                if not self._check_nonfatal(ex):
+                    self.keepalive = False
+
+                self.log("{}\033[0m, {}".format(str(ex), self.vpath), 3)
+                msg = "<pre>{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                self.reply(msg.encode("utf-8", "replace"), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
 
     def send_headers(self, length, status=200, mime=None, headers={}):
-        response = ["HTTP/1.1 {} {}".format(status, HTTPCODE[status])]
+        response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
 
         if length is not None:
             response.append("Content-Length: " + unicode(length))
@@ -172,10 +186,8 @@ class HttpCli(object):
         self.out_headers.update(headers)
 
         # default to utf8 html if no content-type is set
-        try:
-            mime = mime or self.out_headers["Content-Type"]
-        except KeyError:
-            mime = "text/html; charset=UTF-8"
+        if not mime:
+            mime = self.out_headers.get("Content-Type", "text/html; charset=UTF-8")
 
         self.out_headers["Content-Type"] = mime
 
@@ -204,6 +216,43 @@ class HttpCli(object):
         self.log(body.rstrip())
         self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
+    def urlq(self, add={}, rm=[]):
+        """
+        generates url query based on uparam (b, pw, all others)
+        removing anything in rm, adding pairs in add
+        """
+
+        if self.is_rclone:
+            return ""
+
+        kv = {
+            k: v
+            for k, v in self.uparam.items()
+            if k not in rm and self.cookies.get(k) != v
+        }
+        kv.update(add)
+        if not kv:
+            return ""
+
+        r = ["{}={}".format(k, quotep(v)) if v else k for k, v in kv.items()]
+        return "?" + "&amp;".join(r)
+
+    def redirect(self, vpath, suf="", msg="aight", flavor="go to", use302=False):
+        html = self.j2(
+            "msg",
+            h2='<a href="/{}">{} /{}</a>'.format(
+                quotep(vpath) + suf, flavor, html_escape(vpath, crlf=True) + suf
+            ),
+            pre=msg,
+            click=True,
+        ).encode("utf-8", "replace")
+
+        if use302:
+            h = {"Location": "/" + vpath, "Cache-Control": "no-cache"}
+            self.reply(html, status=302, headers=h)
+        else:
+            self.reply(html)
+
     def handle_get(self):
         logmsg = "{:4} {}".format(self.mode, self.req)
 
@@ -226,23 +275,27 @@ class HttpCli(object):
             return self.tx_tree()
 
         # conditional redirect to single volumes
-        if self.vpath == "" and not self.uparam:
+        if self.vpath == "" and not self.ouparam:
             nread = len(self.rvol)
             nwrite = len(self.wvol)
             if nread + nwrite == 1 or (self.rvol == self.wvol and nread == 1):
                 if nread == 1:
-                    self.vpath = self.rvol[0]
+                    vpath = self.rvol[0]
                 else:
-                    self.vpath = self.wvol[0]
+                    vpath = self.wvol[0]
 
-                self.absolute_urls = True
+                if self.vpath != vpath:
+                    self.redirect(vpath, flavor="redirecting to", use302=True)
+                    return True
 
-        # go home if verboten
         self.readable, self.writable = self.conn.auth.vfs.can_access(
             self.vpath, self.uname
         )
         if not self.readable and not self.writable:
+            if self.vpath:
                 self.log("inaccessible: [{}]".format(self.vpath))
+                raise Pebkac(404)
+
             self.uparam = {"h": False}
 
         if "h" in self.uparam:
@@ -312,8 +365,19 @@ class HttpCli(object):
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
                 for buf in reader:
-                    buf = buf.decode("utf-8", "replace")
-                    self.log("urlform @ {}\n  {}\n".format(self.vpath, buf))
+                    orig = buf.decode("utf-8", "replace")
+                    m = "urlform_raw {} @ {}\n  {}\n"
+                    self.log(m.format(len(orig), self.vpath, orig))
+                    try:
+                        plain = unquote(buf.replace(b"+", b" "))
+                        plain = plain.decode("utf-8", "replace")
+                        if buf.startswith(b"msg="):
+                            plain = plain[4:]
+
+                        m = "urlform_dec {} @ {}\n  {}\n"
+                        self.log(m.format(len(plain), self.vpath, plain))
+                    except Exception as ex:
+                        self.log(repr(ex))
 
             if "get" in opt:
                 return self.handle_get()
@@ -388,8 +452,30 @@ class HttpCli(object):
         if act == "tput":
             return self.handle_text_upload()
 
+        if act == "zip":
+            return self.handle_zip_post()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
+    def handle_zip_post(self):
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                break
+
+        if v is None:
+            raise Pebkac(422, "need zip or tar keyword")
+
+        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        items = self.parser.require("files", 1024 * 1024)
+        if not items:
+            raise Pebkac(422, "need files list")
+
+        items = items.replace("\r", "").split("\n")
+        items = [unquotep(x) for x in items if items]
+
+        return self.tx_zip(k, v, vn, rem, items, self.args.ed)
+
     def handle_post_json(self):
         try:
             remains = int(self.headers["content-length"])
@@ -417,15 +503,18 @@ class HttpCli(object):
         if "srch" in self.uparam or "srch" in body:
             return self.handle_search(body)
 
-        # prefer this over undot; no reason to allow traversion
-        if "/" in body["name"]:
-            raise Pebkac(400, "folders verboten")
-
         # up2k-php compat
         for k in "chunkpit.php", "handshake.php":
             if self.vpath.endswith(k):
                 self.vpath = self.vpath[: -len(k)]
 
+        sub = None
+        name = undot(body["name"])
+        if "/" in name:
+            sub, name = name.rsplit("/", 1)
+            self.vpath = "/".join([self.vpath, sub]).strip("/")
+            body["name"] = name
+
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
 
         body["vtop"] = vfs.vpath
@@ -434,12 +523,22 @@ class HttpCli(object):
         body["addr"] = self.ip
         body["vcfg"] = vfs.flags
 
-        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
-        response = x.get()
-        response = json.dumps(response)
+        if sub:
+            try:
+                dst = os.path.join(vfs.realpath, rem)
+                os.makedirs(dst)
+            except:
+                if not os.path.isdir(dst):
+                    raise Pebkac(400, "some file got your folder name")
 
-        self.log(response)
-        self.reply(response.encode("utf-8"), mime="application/json")
+        x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
+        ret = x.get()
+        if sub:
+            ret["name"] = "/".join([sub, ret["name"]])
+
+        ret = json.dumps(ret)
+        self.log(ret)
+        self.reply(ret.encode("utf-8"), mime="application/json")
         return True
 
     def handle_search(self, body):
@@ -464,7 +563,7 @@ class HttpCli(object):
             self.log("qj: " + repr(vbody))
             hits = idx.fsearch(vols, body)
             msg = repr(hits)
-            taglist = []
+            taglist = {}
         else:
             # search by query params
             self.log("qj: " + repr(body))
@@ -556,7 +655,7 @@ class HttpCli(object):
             self.loud_reply(x, status=500)
             return False
 
-        if not WINDOWS and num_left == 0:
+        if not ANYWIN and num_left == 0:
             times = (int(time.time()), int(lastmod))
             self.log("no more chunks, setting times {}".format(times))
             try:
@@ -575,13 +674,16 @@ class HttpCli(object):
 
         if pwd in self.auth.iuser:
             msg = "login ok"
+            dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
+            exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
         else:
             msg = "naw dude"
             pwd = "x"  # nosec
+            exp = "Fri, 15 Aug 1997 01:00:00 GMT"
 
-        h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
-        html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
-        self.reply(html.encode("utf-8"), headers=h)
+        ck = "cppwd={}; Path=/; Expires={}; SameSite=Lax".format(pwd, exp)
+        html = self.j2("msg", h1=msg, h2='<a href="/">ack</a>', redir="/")
+        self.reply(html.encode("utf-8"), headers={"Set-Cookie": ck})
         return True
 
     def handle_mkdir(self):
@@ -610,13 +712,7 @@ class HttpCli(object):
                 raise Pebkac(500, "mkdir failed, check the logs")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        esc_paths = [quotep(vpath), html_escape(vpath)]
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
-            pre="aight",
-            click=True,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(vpath)
         return True
 
     def handle_new_md(self):
@@ -643,14 +739,7 @@ class HttpCli(object):
                 f.write(b"`GRUNNUR`\n")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
-                quotep(vpath), html_escape(vpath)
-            ),
-            pre="aight",
-            click=True,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(vpath, "?edit")
         return True
 
     def handle_plain_upload(self):
@@ -669,7 +758,9 @@ class HttpCli(object):
 
                 if p_file and not nullwrite:
                     fdir = os.path.join(vfs.realpath, rem)
-                    fname = sanitize_fn(p_file)
+                    fname = sanitize_fn(
+                        p_file, bad=[".prologue.html", ".epilogue.html"]
+                    )
 
                     if not os.path.isdir(fsenc(fdir)):
                         raise Pebkac(404, "that folder does not exist")
@@ -698,12 +789,16 @@ class HttpCli(object):
                 except Pebkac:
                     if fname != os.devnull:
                         fp = os.path.join(fdir, fname)
+                        fp2 = fp
+                        if self.args.dotpart:
+                            fp2 = os.path.join(fdir, "." + fname)
+
                         suffix = ".PARTIAL"
                         try:
-                            os.rename(fsenc(fp), fsenc(fp + suffix))
+                            os.rename(fsenc(fp), fsenc(fp2 + suffix))
                         except:
-                            fp = fp[: -len(suffix)]
-                            os.rename(fsenc(fp), fsenc(fp + suffix))
+                            fp2 = fp2[: -len(suffix) - 1]
+                            os.rename(fsenc(fp), fsenc(fp2 + suffix))
 
                     raise
 
@@ -749,13 +844,7 @@ class HttpCli(object):
                     ).encode("utf-8")
                 )
 
-        html = self.conn.tpl_msg.render(
-            h2='<a href="/{}">return to /{}</a>'.format(
-                quotep(self.vpath), html_escape(self.vpath)
-            ),
-            pre=msg,
-        )
-        self.reply(html.encode("utf-8", "replace"))
+        self.redirect(self.vpath, msg=msg, flavor="return to")
         self.parser.drop()
         return True
 
@@ -855,13 +944,14 @@ class HttpCli(object):
         return True
 
     def _chk_lastmod(self, file_ts):
+        date_fmt = "%a, %d %b %Y %H:%M:%S GMT"
         file_dt = datetime.utcfromtimestamp(file_ts)
-        file_lastmod = file_dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
+        file_lastmod = file_dt.strftime(date_fmt)
 
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
-                cli_dt = time.strptime(cli_lastmod, "%a, %d %b %Y %H:%M:%S GMT")
+                cli_dt = time.strptime(cli_lastmod, date_fmt)
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
             except Exception as ex:
@@ -1037,29 +1127,87 @@ class HttpCli(object):
         self.log("{},  {}".format(logmsg, spd))
         return ret
 
+    def tx_zip(self, fmt, uarg, vn, rem, items, dots):
+        if self.args.no_zip:
+            raise Pebkac(400, "not enabled")
+
+        logmsg = "{:4} {} ".format("", self.req)
+        self.keepalive = False
+
+        if not uarg:
+            uarg = ""
+
+        if fmt == "tar":
+            mime = "application/x-tar"
+            packer = StreamTar
+        else:
+            mime = "application/zip"
+            packer = StreamZip
+
+        fn = items[0] if items and items[0] else self.vpath
+        if fn:
+            fn = fn.rstrip("/").split("/")[-1]
+        else:
+            fn = self.headers.get("host", "hey")
+
+        afn = "".join(
+            [x if x in (string.ascii_letters + string.digits) else "_" for x in fn]
+        )
+
+        bascii = unicode(string.ascii_letters + string.digits).encode("utf-8")
+        ufn = fn.encode("utf-8", "xmlcharrefreplace")
+        if PY2:
+            ufn = [unicode(x) if x in bascii else "%{:02x}".format(ord(x)) for x in ufn]
+        else:
+            ufn = [
+                chr(x).encode("utf-8")
+                if x in bascii
+                else "%{:02x}".format(x).encode("ascii")
+                for x in ufn
+            ]
+        ufn = b"".join(ufn).decode("ascii")
+
+        cdis = "attachment; filename=\"{}.{}\"; filename*=UTF-8''{}.{}"
+        cdis = cdis.format(afn, fmt, ufn, fmt)
+        self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis})
+
+        fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
+        # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
+        bsent = 0
+        for buf in bgen.gen():
+            if not buf:
+                break
+
+            try:
+                self.s.sendall(buf)
+                bsent += len(buf)
+            except:
+                logmsg += " \033[31m" + unicode(bsent) + "\033[0m"
+                break
+
+        spd = self._spd(bsent)
+        self.log("{},  {}".format(logmsg, spd))
+        return True
+
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
-        if "edit2" in self.uparam:
-            html_path = "web/mde.html"
-            template = self.conn.tpl_mde
-        else:
-            html_path = "web/md.html"
-            template = self.conn.tpl_md
 
-        html_path = os.path.join(E.mod, html_path)
+        tpl = "mde" if "edit2" in self.uparam else "md"
+        html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
+        template = self.j2(tpl)
 
         st = os.stat(fsenc(fs_path))
-        # sz_md = st.st_size
         ts_md = st.st_mtime
 
         st = os.stat(fsenc(html_path))
         ts_html = st.st_mtime
 
-        # TODO dont load into memory ;_;
-        #   (trivial fix, count the &'s)
-        with open(fsenc(fs_path), "rb") as f:
-            md = f.read().replace(b"&", b"&amp;")
-            sz_md = len(md)
+        sz_md = 0
+        for buf in yieldfile(fs_path):
+            sz_md += len(buf)
+            for c, v in [[b"&", 4], [b"<", 3], [b">", 3]]:
+                sz_md += (len(buf) - len(buf.replace(c, b""))) * v
 
         file_ts = max(ts_md, ts_html)
         file_lastmod, do_send = self._chk_lastmod(file_ts)
@@ -1067,27 +1215,34 @@ class HttpCli(object):
         self.out_headers["Cache-Control"] = "no-cache"
         status = 200 if do_send else 304
 
+        boundary = "\roll\tide"
         targs = {
             "edit": "edit" in self.uparam,
-            "title": html_escape(self.vpath),
+            "title": html_escape(self.vpath, crlf=True),
             "lastmod": int(ts_md * 1000),
             "md_plug": "true" if self.args.emp else "false",
             "md_chk_rate": self.args.mcr,
-            "md": "",
+            "md": boundary,
         }
-        sz_html = len(template.render(**targs).encode("utf-8"))
-        self.send_headers(sz_html + sz_md, status)
+        html = template.render(**targs).encode("utf-8")
+        html = html.split(boundary.encode("utf-8"))
+        if len(html) != 2:
+            raise Exception("boundary appears in " + html_path)
+
+        self.send_headers(sz_md + len(html[0]) + len(html[1]), status)
 
         logmsg += unicode(status)
         if self.mode == "HEAD" or not do_send:
             self.log(logmsg)
             return True
 
-        # TODO jinja2 can stream this right?
-        targs["md"] = md.decode("utf-8", "replace")
-        html = template.render(**targs).encode("utf-8")
         try:
-            self.s.sendall(html)
+            self.s.sendall(html[0])
+            for buf in yieldfile(fs_path):
+                self.s.sendall(html_bescape(buf))
+
+            self.s.sendall(html[1])
+
         except:
             self.log(logmsg + " \033[31md/c\033[0m")
             return False
@@ -1096,9 +1251,10 @@ class HttpCli(object):
         return True
 
     def tx_mounts(self):
+        suf = self.urlq(rm=["h"])
         rvol = [x + "/" if x else x for x in self.rvol]
         wvol = [x + "/" if x else x for x in self.wvol]
-        html = self.conn.tpl_mounts.render(this=self, rvol=rvol, wvol=wvol)
+        html = self.j2("splash", this=self, rvol=rvol, wvol=wvol, url_suf=suf)
         self.reply(html.encode("utf-8"))
         return True
 
@@ -1167,7 +1323,7 @@ class HttpCli(object):
                 else:
                     vpath += "/" + node
 
-                vpnodes.append([quotep(vpath) + "/", html_escape(node)])
+                vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
         vn, rem = self.auth.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
@@ -1178,6 +1334,94 @@ class HttpCli(object):
             # print(abspath)
             raise Pebkac(404)
 
+        srv_info = []
+
+        try:
+            if not self.args.nih:
+                srv_info.append(unicode(socket.gethostname()).split(".")[0])
+        except:
+            self.log("#wow #whoa")
+
+        try:
+            # some fuses misbehave
+            if not self.args.nid:
+                if WINDOWS:
+                    bfree = ctypes.c_ulonglong(0)
+                    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
+                        ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+                    )
+                    srv_info.append(humansize(bfree.value) + " free")
+                else:
+                    sv = os.statvfs(abspath)
+                    free = humansize(sv.f_frsize * sv.f_bfree, True)
+                    total = humansize(sv.f_frsize * sv.f_blocks, True)
+
+                    srv_info.append(free + " free")
+                    srv_info.append(total)
+        except:
+            pass
+
+        srv_info = "</span> /// <span>".join(srv_info)
+
+        perms = []
+        if self.readable:
+            perms.append("read")
+        if self.writable:
+            perms.append("write")
+
+        url_suf = self.urlq()
+        is_ls = "ls" in self.uparam
+        ts = ""  # "?{}".format(time.time())
+
+        tpl = "browser"
+        if "b" in self.uparam:
+            tpl = "browser2"
+
+        logues = ["", ""]
+        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
+            fn = os.path.join(abspath, fn)
+            if os.path.exists(fsenc(fn)):
+                with open(fsenc(fn), "rb") as f:
+                    logues[n] = f.read().decode("utf-8")
+
+        ls_ret = {
+            "dirs": [],
+            "files": [],
+            "taglist": [],
+            "srvinf": srv_info,
+            "perms": perms,
+            "logues": logues,
+        }
+        j2a = {
+            "vdir": quotep(self.vpath),
+            "vpnodes": vpnodes,
+            "files": [],
+            "ts": ts,
+            "perms": json.dumps(perms),
+            "taglist": [],
+            "tag_order": [],
+            "have_up2k_idx": ("e2d" in vn.flags),
+            "have_tags_idx": ("e2t" in vn.flags),
+            "have_zip": (not self.args.no_zip),
+            "have_b_u": (self.writable and self.uparam.get("b") == "u"),
+            "url_suf": url_suf,
+            "logues": logues,
+            "title": html_escape(self.vpath, crlf=True),
+            "srv_info": srv_info,
+        }
+        if not self.readable:
+            if is_ls:
+                ret = json.dumps(ls_ret)
+                self.reply(ret.encode("utf-8", "replace"), mime="application/json")
+                return True
+
+            if not os.path.isdir(fsenc(abspath)):
+                raise Pebkac(404)
+
+            html = self.j2(tpl, **j2a)
+            self.reply(html.encode("utf-8", "replace"))
+            return True
+
         if not os.path.isdir(fsenc(abspath)):
             if abspath.endswith(".md") and "raw" not in self.uparam:
                 return self.tx_md(abspath)
@@ -1187,6 +1431,11 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                return self.tx_zip(k, v, vn, rem, [], self.args.ed)
+
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
@@ -1216,8 +1465,6 @@ class HttpCli(object):
         if rem == ".hist":
             hidden = ["up2k."]
 
-        is_ls = "ls" in self.uparam
-
         icur = None
         if "e2t" in vn.flags:
             idx = self.conn.get_u2idx()
@@ -1247,11 +1494,14 @@ class HttpCli(object):
 
             is_dir = stat.S_ISDIR(inf.st_mode)
             if is_dir:
-                margin = "DIR"
                 href += "/"
+                if self.args.no_zip:
+                    margin = "DIR"
+                else:
+                    margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
-                    base, html_escape(hist[fn][2], quote=True), hist[fn][0]
+                    base, html_escape(hist[fn][2], quote=True, crlf=True), hist[fn][0]
                 )
             else:
                 margin = "-"
@@ -1310,83 +1560,21 @@ class HttpCli(object):
             for f in dirs:
                 f["tags"] = {}
 
-        srv_info = []
-
-        try:
-            if not self.args.nih:
-                srv_info.append(unicode(socket.gethostname()).split(".")[0])
-        except:
-            self.log("#wow #whoa")
-            pass
-
-        try:
-            # some fuses misbehave
-            if not self.args.nid:
-                if WINDOWS:
-                    bfree = ctypes.c_ulonglong(0)
-                    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
-                        ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
-                    )
-                    srv_info.append(humansize(bfree.value) + " free")
-                else:
-                    sv = os.statvfs(abspath)
-                    free = humansize(sv.f_frsize * sv.f_bfree, True)
-                    total = humansize(sv.f_frsize * sv.f_blocks, True)
-
-                    srv_info.append(free + " free")
-                    srv_info.append(total)
-        except:
-            pass
-
-        srv_info = "</span> /// <span>".join(srv_info)
-
-        perms = []
-        if self.readable:
-            perms.append("read")
-        if self.writable:
-            perms.append("write")
-
-        logues = ["", ""]
-        for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
-            fn = os.path.join(abspath, fn)
-            if os.path.exists(fsenc(fn)):
-                with open(fsenc(fn), "rb") as f:
-                    logues[n] = f.read().decode("utf-8")
-
         if is_ls:
             [x.pop(k) for k in ["name", "dt"] for y in [dirs, files] for x in y]
-            ret = {
-                "dirs": dirs,
-                "files": files,
-                "srvinf": srv_info,
-                "perms": perms,
-                "logues": logues,
-                "taglist": taglist,
-            }
-            ret = json.dumps(ret)
+            ls_ret["dirs"] = dirs
+            ls_ret["files"] = files
+            ls_ret["taglist"] = taglist
+            ret = json.dumps(ls_ret)
             self.reply(ret.encode("utf-8", "replace"), mime="application/json")
             return True
 
-        ts = ""
-        # ts = "?{}".format(time.time())
+        j2a["files"] = dirs + files
+        j2a["logues"] = logues
+        j2a["taglist"] = taglist
+        if "mte" in vn.flags:
+            j2a["tag_order"] = json.dumps(vn.flags["mte"].split(","))
 
-        dirs.extend(files)
-
-        html = self.conn.tpl_browser.render(
-            vdir=quotep(self.vpath),
-            vpnodes=vpnodes,
-            files=dirs,
-            ts=ts,
-            perms=json.dumps(perms),
-            taglist=taglist,
-            tag_order=json.dumps(
-                vn.flags["mte"].split(",") if "mte" in vn.flags else []
-            ),
-            have_up2k_idx=("e2d" in vn.flags),
-            have_tags_idx=("e2t" in vn.flags),
-            logues=logues,
-            title=html_escape(self.vpath),
-            srv_info=srv_info,
-        )
+        html = self.j2(tpl, **j2a)
         self.reply(html.encode("utf-8", "replace"))
         return True

copyparty/httpconn.py

@@ -12,23 +12,6 @@ try:
 except:
     HAVE_SSL = False
 
-try:
-    import jinja2
-except ImportError:
-    print(
-        """\033[1;31m
-  you do not have jinja2 installed,\033[33m
-  choose one of these:\033[0m
-   * apt install python-jinja2
-   * {} -m pip install --user jinja2
-   * (try another python version, if you have one)
-   * (try copyparty.sfx instead)
-""".format(
-            os.path.basename(sys.executable)
-        )
-    )
-    sys.exit(1)
-
 from .__init__ import E
 from .util import Unrecv
 from .httpcli import HttpCli
@@ -57,14 +40,6 @@ class HttpConn(object):
         self.log_func = hsrv.log
         self.set_rproxy()
 
-        env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
-        self.tpl_mounts = env.get_template("splash.html")
-        self.tpl_browser = env.get_template("browser.html")
-        self.tpl_msg = env.get_template("msg.html")
-        self.tpl_md = env.get_template("md.html")
-        self.tpl_mde = env.get_template("mde.html")
-
     def set_rproxy(self, ip=None):
         if ip is None:
             color = 36
@@ -112,7 +87,9 @@ class HttpConn(object):
                 err = "need at least 4 bytes in the first packet; got {}".format(
                     len(method)
                 )
+                if method:
                     self.log(err)
+
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 

copyparty/httpsrv.py

@@ -2,10 +2,28 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import sys
 import time
 import socket
 import threading
 
+try:
+    import jinja2
+except ImportError:
+    print(
+        """\033[1;31m
+  you do not have jinja2 installed,\033[33m
+  choose one of these:\033[0m
+   * apt install python-jinja2
+   * {} -m pip install --user jinja2
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            os.path.basename(sys.executable)
+        )
+    )
+    sys.exit(1)
+
 from .__init__ import E, MACOS
 from .httpconn import HttpConn
 from .authsrv import AuthSrv
@@ -30,6 +48,13 @@ class HttpSrv(object):
         self.workload_thr_alive = False
         self.auth = AuthSrv(self.args, self.log)
 
+        env = jinja2.Environment()
+        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        self.j2 = {
+            x: env.get_template(x + ".html")
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+        }
+
         cert_path = os.path.join(E.cfg, "cert.pem")
         if os.path.exists(cert_path):
             self.cert_path = cert_path

copyparty/star.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import threading
+
+from .sutil import errdesc
+from .util import Queue, fsenc
+
+
+class QFile(object):
+    """file-like object which buffers writes into a queue"""
+
+    def __init__(self):
+        self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
+
+    def write(self, buf):
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
+
+
+class StreamTar(object):
+    """construct in-memory tar file from the given path"""
+
+    def __init__(self, fgen, **kwargs):
+        self.ci = 0
+        self.co = 0
+        self.qfile = QFile()
+        self.fgen = fgen
+        self.errf = None
+
+        # python 3.8 changed to PAX_FORMAT as default,
+        # waste of space and don't care about the new features
+        fmt = tarfile.GNU_FORMAT
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+
+        w = threading.Thread(target=self._gen)
+        w.daemon = True
+        w.start()
+
+    def gen(self):
+        while True:
+            buf = self.qfile.q.get()
+            if not buf:
+                break
+
+            self.co += len(buf)
+            yield buf
+
+        yield None
+        if self.errf:
+            os.unlink(self.errf["ap"])
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        fsi = f["st"]
+
+        inf = tarfile.TarInfo(name=name)
+        inf.mode = fsi.st_mode
+        inf.size = fsi.st_size
+        inf.mtime = fsi.st_mtime
+        inf.uid = 0
+        inf.gid = 0
+
+        self.ci += inf.size
+        with open(fsenc(src), "rb", 512 * 1024) as f:
+            self.tar.addfile(inf, f)
+
+    def _gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                self.ser(f)
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            self.errf = errdesc(errors)
+            self.ser(self.errf)
+
+        self.tar.close()
+        self.qfile.write(None)

copyparty/sutil.py

@@ -0,0 +1,25 @@
+import os
+import time
+import tempfile
+from datetime import datetime
+
+
+def errdesc(errors):
+    report = ["copyparty failed to add the following files to the archive:", ""]
+
+    for fn, err in errors:
+        report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
+
+    with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
+        tf_path = tf.name
+        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+
+    dt = datetime.utcfromtimestamp(time.time())
+    dt = dt.strftime("%Y-%m%d-%H%M%S")
+
+    os.chmod(tf_path, 0o444)
+    return {
+        "vp": "archive-errors-{}.txt".format(dt),
+        "ap": tf_path,
+        "st": os.stat(tf_path),
+    }

copyparty/szip.py

@@ -0,0 +1,271 @@
+import os
+import time
+import zlib
+import struct
+from datetime import datetime
+
+from .sutil import errdesc
+from .util import yieldfile, sanitize_fn
+
+
+def dostime2unix(buf):
+    t, d = struct.unpack("<HH", buf)
+
+    ts = (t & 0x1F) * 2
+    tm = (t >> 5) & 0x3F
+    th = t >> 11
+
+    dd = d & 0x1F
+    dm = (d >> 5) & 0xF
+    dy = (d >> 9) + 1980
+
+    tt = (dy, dm, dd, th, tm, ts)
+    tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
+    iso = tf.format(*tt)
+
+    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(dt.timestamp())
+
+
+def unixtime2dos(ts):
+    tt = time.gmtime(ts)
+    dy, dm, dd, th, tm, ts = list(tt)[:6]
+
+    bd = ((dy - 1980) << 9) + (dm << 5) + dd
+    bt = (th << 11) + (tm << 5) + ts // 2
+    return struct.pack("<HH", bt, bd)
+
+
+def gen_fdesc(sz, crc32, z64):
+    ret = b"\x50\x4b\x07\x08"
+    fmt = "<LQQ" if z64 else "<LLL"
+    ret += struct.pack(fmt, crc32, sz, sz)
+    return ret
+
+
+def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+    """
+    does regular file headers
+    and the central directory meme if h_pos is set
+    (h_pos = absolute position of the regular header)
+    """
+
+    # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
+    # extinfo for values which exceed H, but that becomes an off-by-one
+    # (can't tell if it was clamped or exactly maxval), make it obvious
+    z64 = sz >= 0xFFFFFFFF
+    z64v = [sz, sz] if z64 else []
+    if h_pos and h_pos >= 0xFFFFFFFF:
+        # central, also consider ptr to original header
+        z64v.append(h_pos)
+
+    # confusingly this doesn't bump if h_pos
+    req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
+
+    if crc32:
+        crc32 = struct.pack("<L", crc32)
+    else:
+        crc32 = b"\x00" * 4
+
+    if h_pos is None:
+        # 4b magic, 2b min-ver
+        ret = b"\x50\x4b\x03\x04" + req_ver
+    else:
+        # 4b magic, 2b spec-ver, 2b min-ver
+        ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
+
+    ret += b"\x00" if pre_crc else b"\x08"  # streaming
+    ret += b"\x08" if utf8 else b"\x00"  # appnote 6.3.2 (2007)
+
+    # 2b compression, 4b time, 4b crc
+    ret += b"\x00\x00" + unixtime2dos(lastmod) + crc32
+
+    # spec says to put zeros when !crc if bit3 (streaming)
+    # however infozip does actual sz and it even works on winxp
+    # (same reasning for z64 extradata later)
+    vsz = 0xFFFFFFFF if z64 else sz
+    ret += struct.pack("<LL", vsz, vsz)
+
+    # windows support (the "?" replace below too)
+    fn = sanitize_fn(fn, ok="/")
+    bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
+
+    z64_len = len(z64v) * 8 + 4 if z64v else 0
+    ret += struct.pack("<HH", len(bfn), z64_len)
+
+    if h_pos is not None:
+        # 2b comment, 2b diskno
+        ret += b"\x00" * 4
+
+        # 2b internal.attr, 4b external.attr
+        # infozip-macos: 0100 0000 a481 file:644
+        # infozip-macos: 0100 0100 0080 file:000
+        ret += b"\x01\x00\x00\x00\xa4\x81"
+
+        # 4b local-header-ofs
+        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+
+    ret += bfn
+
+    if z64v:
+        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+
+    return ret
+
+
+def gen_ecdr(items, cdir_pos, cdir_end):
+    """
+    summary of all file headers,
+    usually the zipfile footer unless something clamps
+    """
+
+    ret = b"\x50\x4b\x05\x06"
+
+    # 2b ndisk, 2b disk0
+    ret += b"\x00" * 4
+
+    cdir_sz = cdir_end - cdir_pos
+
+    nitems = min(0xFFFF, len(items))
+    csz = min(0xFFFFFFFF, cdir_sz)
+    cpos = min(0xFFFFFFFF, cdir_pos)
+
+    need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
+
+    # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
+    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+
+    # 2b comment length
+    ret += b"\x00\x00"
+
+    return [ret, need_64]
+
+
+def gen_ecdr64(items, cdir_pos, cdir_end):
+    """
+    z64 end of central directory
+    added when numfiles or a headerptr clamps
+    """
+
+    ret = b"\x50\x4b\x06\x06"
+
+    # 8b own length from hereon
+    ret += b"\x2c" + b"\x00" * 7
+
+    # 2b spec-ver, 2b min-ver
+    ret += b"\x1e\x03\x2d\x00"
+
+    # 4b ndisk, 4b disk0
+    ret += b"\x00" * 8
+
+    # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
+    cdir_sz = cdir_end - cdir_pos
+    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+
+    return ret
+
+
+def gen_ecdr64_loc(ecdr64_pos):
+    """
+    z64 end of central directory locator
+    points to ecdr64
+    why
+    """
+
+    ret = b"\x50\x4b\x06\x07"
+
+    # 4b cdisk, 8b start of ecdr64, 4b ndisks
+    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+
+    return ret
+
+
+class StreamZip(object):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
+        self.fgen = fgen
+        self.utf8 = utf8
+        self.pre_crc = pre_crc
+
+        self.pos = 0
+        self.items = []
+
+    def _ct(self, buf):
+        self.pos += len(buf)
+        return buf
+
+    def ser(self, f):
+        name = f["vp"]
+        src = f["ap"]
+        st = f["st"]
+
+        sz = st.st_size
+        ts = st.st_mtime + 1
+
+        crc = None
+        if self.pre_crc:
+            crc = 0
+            for buf in yieldfile(src):
+                crc = zlib.crc32(buf, crc)
+
+            crc &= 0xFFFFFFFF
+
+        h_pos = self.pos
+        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        yield self._ct(buf)
+
+        crc = crc or 0
+        for buf in yieldfile(src):
+            if not self.pre_crc:
+                crc = zlib.crc32(buf, crc)
+
+            yield self._ct(buf)
+
+        crc &= 0xFFFFFFFF
+
+        self.items.append([name, sz, ts, crc, h_pos])
+
+        z64 = sz >= 4 * 1024 * 1024 * 1024
+
+        if z64 or not self.pre_crc:
+            buf = gen_fdesc(sz, crc, z64)
+            yield self._ct(buf)
+
+    def gen(self):
+        errors = []
+        for f in self.fgen:
+            if "err" in f:
+                errors.append([f["vp"], f["err"]])
+                continue
+
+            try:
+                for x in self.ser(f):
+                    yield x
+            except Exception as ex:
+                errors.append([f["vp"], repr(ex)])
+
+        if errors:
+            errf = errdesc(errors)
+            print(repr(errf))
+            for x in self.ser(errf):
+                yield x
+
+        cdir_pos = self.pos
+        for name, sz, ts, crc, h_pos in self.items:
+            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+            yield self._ct(buf)
+        cdir_end = self.pos
+
+        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+        if need_64:
+            ecdir64_pos = self.pos
+            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+            yield self._ct(buf)
+
+            buf = gen_ecdr64_loc(ecdir64_pos)
+            yield self._ct(buf)
+
+        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+        yield self._ct(ecdr)
+
+        if errors:
+            os.unlink(errf["ap"])

copyparty/up2k.py

@@ -16,7 +16,7 @@ import traceback
 import subprocess as sp
 from copy import deepcopy
 
-from .__init__ import WINDOWS
+from .__init__ import WINDOWS, ANYWIN
 from .util import (
     Pebkac,
     Queue,
@@ -79,7 +79,7 @@ class Up2k(object):
             if self.sqlite_ver < (3, 9):
                 self.no_expr_idx = True
 
-        if WINDOWS:
+        if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
             thr = threading.Thread(target=self._lastmodder)
@@ -101,11 +101,12 @@ class Up2k(object):
             thr.daemon = True
             thr.start()
 
-            thr = threading.Thread(target=self._tagger)
+            thr = threading.Thread(target=self._hasher)
             thr.daemon = True
             thr.start()
 
-            thr = threading.Thread(target=self._hasher)
+            if self.mtag:
+                thr = threading.Thread(target=self._tagger)
                 thr.daemon = True
                 thr.start()
 
@@ -232,6 +233,7 @@ class Up2k(object):
                 (ft if v is True else ff if v is False else fv).format(k, str(v))
                 for k, v in flags.items()
             ]
+            if a:
                 self.log(" ".join(sorted(a)) + "\033[0m")
 
             reg = {}
@@ -666,12 +668,6 @@ class Up2k(object):
             cur.close()
 
     def _start_mpool(self):
-        if WINDOWS and False:
-            nah = open(os.devnull, "wb")
-            wmic = "processid={}".format(os.getpid())
-            wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
-            sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
-
         # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
         # both do crazy runahead so lets reinvent another wheel
         nw = os.cpu_count() if hasattr(os, "cpu_count") else 4
@@ -696,12 +692,6 @@ class Up2k(object):
 
         mpool.join()
         done = self._flush_mpool(wcur)
-        if WINDOWS and False:
-            nah = open(os.devnull, "wb")
-            wmic = "processid={}".format(os.getpid())
-            wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
-            sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
-
         return done
 
     def _tag_thr(self, q):
@@ -901,7 +891,7 @@ class Up2k(object):
             if cj["ptop"] not in self.registry:
                 raise Pebkac(410, "location unavailable")
 
-        cj["name"] = sanitize_fn(cj["name"])
+        cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
         wark = self._get_wark(cj)
         now = time.time()
@@ -1067,6 +1057,8 @@ class Up2k(object):
         with self.mutex:
             job = self.registry[ptop].get(wark, None)
             if not job:
+                known = " ".join([x for x in self.registry[ptop].keys()])
+                self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
@@ -1106,8 +1098,9 @@ class Up2k(object):
 
             atomic_move(src, dst)
 
-            if WINDOWS:
-                self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))])
+            if ANYWIN:
+                a = [dst, job["size"], (int(time.time()), int(job["lmod"]))]
+                self.lastmod_q.put(a)
 
             # legit api sware 2 me mum
             if self.idx_wark(
@@ -1205,9 +1198,23 @@ class Up2k(object):
         #    raise Exception("aaa")
 
         tnam = job["name"] + ".PARTIAL"
+        if self.args.dotpart:
+            tnam = "." + tnam
+
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f:
             f, job["tnam"] = f["orz"]
+            if (
+                ANYWIN
+                and self.args.sparse
+                and self.args.sparse * 1024 * 1024 <= job["size"]
+            ):
+                fp = os.path.join(pdir, job["tnam"])
+                try:
+                    sp.check_call(["fsutil", "sparse", "setflag", fp])
+                except:
+                    self.log("could not sparse [{}]".format(fp), 3)
+
             f.seek(job["size"] - 1)
             f.write(b"e")
 
@@ -1219,13 +1226,19 @@ class Up2k(object):
 
             # self.log("lmod: got {}".format(len(ready)))
             time.sleep(5)
-            for path, times in ready:
+            for path, sz, times in ready:
                 self.log("lmod: setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:
                     self.log("lmod: failed to utime ({}, {})".format(path, times))
 
+                if self.args.sparse and self.args.sparse * 1024 * 1024 <= sz:
+                    try:
+                        sp.check_call(["fsutil", "sparse", "setflag", path, "0"])
+                    except:
+                        self.log("could not unsparse [{}]".format(path), 3)
+
     def _snapshot(self):
         persist_interval = 30  # persist unfinished uploads index every 30 sec
         discard_interval = 21600  # drop unfinished uploads after 6 hours inactivity
@@ -1309,6 +1322,7 @@ class Up2k(object):
                     self.log("no cursor to write tags with??", c=1)
                     continue
 
+                # TODO is undef if vol 404 on startup
                 entags = self.entags[ptop]
                 if not entags:
                     self.log("no entags okay.jpg", c=3)

copyparty/util.py

@@ -16,7 +16,7 @@ import mimetypes
 import contextlib
 import subprocess as sp  # nosec
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -49,6 +49,7 @@ HTTPCODE = {
     200: "OK",
     204: "No Content",
     206: "Partial Content",
+    302: "Found",
     304: "Not Modified",
     400: "Bad Request",
     403: "Forbidden",
@@ -576,11 +577,12 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn):
+def sanitize_fn(fn, ok="", bad=[]):
+    if "/" not in ok:
         fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -590,10 +592,11 @@ def sanitize_fn(fn):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ]:
-            fn = fn.replace(bad, good)
+        ]
+        for a, b in [x for x in remap if x[0] not in ok]:
+            fn = fn.replace(a, b)
 
-        bad = ["con", "prn", "aux", "nul"]
+        bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):
             bad += "com{0} lpt{0}".format(n).split(" ")
 
@@ -614,17 +617,24 @@ def exclude_dotfiles(filepaths):
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 
@@ -780,6 +790,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()

copyparty/web/browser.css

@@ -6,7 +6,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {
@@ -68,7 +68,7 @@ a, #files tbody div a:last-child {
 	color: #999;
 	font-weight: normal;
 }
-#files tr+tr:hover {
+#files tr:hover {
 	background: #1c1c1c;
 }
 #files thead th {
@@ -90,8 +90,6 @@ a, #files tbody div a:last-child {
 #files td {
 	margin: 0;
 	padding: 0 .5em;
-}
-#files td {
 	border-bottom: 1px solid #111;
 }
 #files td+td+td {
@@ -182,6 +180,21 @@ a, #files tbody div a:last-child {
 	color: #840;
 	text-shadow: 0 0 .3em #b80;
 }
+#files tbody tr.sel td {
+	color: #fff;
+	background: #925;
+	border-color: #c37;
+}
+#files tr.sel a {
+	color: #fff;
+}
+#files tr.sel a.play {
+	color: #fc5;
+}
+#files tr.sel a.play.act {
+	color: #fff;
+	text-shadow: 0 0 1px #fff;
+}
 #blocked {
 	position: fixed;
 	top: 0;
@@ -238,7 +251,7 @@ a, #files tbody div a:last-child {
 	height: 100%;
 	background: #3c3c3c;
 }
-#wtoggle {
+#wtico {
 	cursor: url(/.cpr/dd/1.png), pointer;
 	animation: cursor 500ms infinite;
 }
@@ -268,6 +281,33 @@ a, #files tbody div a:last-child {
 	padding: .2em 0 0 .07em;
 	color: #fff;
 }
+#wzip {
+	display: none;
+	margin-right: .3em;
+	padding-right: .3em;
+	border-right: .1em solid #555;
+}
+#wtoggle,
+#wtoggle * {
+	line-height: 1em;
+}
+#wtoggle.sel {
+	width: 6.4em;
+}
+#wtoggle.sel #wzip {
+	display: inline-block;
+}
+#wtoggle.sel #wzip a {
+	font-size: .4em;
+	padding: 0 .3em;
+	margin: -.3em .2em;
+	position: relative;
+	display: inline-block;
+}
+#wtoggle.sel #wzip #selzip {
+	top: -.6em;
+	padding: .4em .3em;
+}
 #barpos,
 #barbuf {
 	position: absolute;
@@ -311,10 +351,10 @@ a, #files tbody div a:last-child {
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
-@media (min-width: 90em) {
+@media (min-width: 80em) {
 	#barpos,
 	#barbuf {
-		width: calc(100% - 24em);
+		width: calc(100% - 21em);
 		left: 9.8em;
 		top: .7em;
 		height: 1.6em;
@@ -324,6 +364,9 @@ a, #files tbody div a:last-child {
 		bottom: -3.2em;
 		height: 3.2em;
 	}
+	#pvol {
+		max-width: 9em;
+	}
 }
 
 
@@ -455,15 +498,12 @@ input[type="checkbox"]:checked+label {
 	border-collapse: collapse;
 	width: 100%;
 }
-#files td div a:last-child {
-	width: 100%;
-}
 #wrap {
 	margin-top: 2em;
 }
 #tree {
 	display: none;
-	position: fixed;
+	position: absolute;
 	left: 0;
 	bottom: 0;
 	top: 7em;
@@ -476,9 +516,7 @@ input[type="checkbox"]:checked+label {
 #thx_ff {
 	padding: 5em 0;
 }
-#tree::-webkit-scrollbar-track {
-	background: #333;
-}
+#tree::-webkit-scrollbar-track,
 #tree::-webkit-scrollbar {
 	background: #333;
 }
@@ -598,7 +636,8 @@ input[type="checkbox"]:checked+label {
 #files td.min a {
 	display: none;
 }
-#files tr.play td {
+#files tr.play td,
+#files tr.play div a {
 	background: #fc4;
 	border-color: transparent;
 	color: #400;
@@ -652,3 +691,199 @@ input[type="checkbox"]:checked+label {
 	font-family: monospace, monospace;
 	line-height: 2em;
 }
+#pvol,
+#barbuf,
+#barpos,
+#u2conf label {
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+}
+
+
+
+
+
+html.light {
+	color: #333;
+	background: #eee;
+	text-shadow: none;
+}
+html.light #ops,
+html.light .opbox,
+html.light #srch_form {
+	background: #f7f7f7;
+	box-shadow: 0 0 .3em #ddd;
+	border-color: #f7f7f7;
+}
+html.light #ops a.act {
+	box-shadow: 0 .2em .2em #ccc;
+	background: #fff;
+	border-color: #07a;
+	padding-top: .4em;
+}
+html.light #op_cfg h3 {
+	border-color: #ccc;
+}
+html.light .tglbtn,
+html.light #tree > a + a {
+	color: #666;
+	background: #ddd;
+	box-shadow: none;
+}
+html.light .tglbtn:hover,
+html.light #tree > a + a:hover {
+	background: #caf;
+}
+html.light .tglbtn.on,
+html.light #tree > a + a.on {
+	background: #4a0;
+	color: #fff;
+}
+html.light #srv_info {
+	color: #c83;
+	text-shadow: 1px 1px 0 #fff;
+}
+html.light #srv_info span {
+	color: #000;
+}
+html.light #treeul a+a {
+	background: inherit;
+	color: #06a;
+}
+html.light #treeul a.hl {
+	background: #07a;
+	color: #fff;
+}
+html.light #tree li {
+	border-color: #ddd #fff #f7f7f7 #fff;
+}
+html.light #tree ul {
+	border-color: #ccc;
+}
+html.light a,
+html.light #ops a,
+html.light #files tbody div a:last-child {
+	color: #06a;
+}
+html.light #files tbody {
+	background: #f7f7f7;
+}
+html.light #files {
+	box-shadow: 0 0 .3em #ccc;
+}
+html.light #files thead th {
+	background: #eee;
+}
+html.light #files tr td {
+	border-top: 1px solid #ddd;
+}
+html.light #files td {
+	border-bottom: 1px solid #f7f7f7;
+}
+html.light #files tbody tr:last-child td {
+	border-bottom: .2em solid #ccc;
+}
+html.light #files td:nth-child(2n) {
+	color: #d38;
+}
+html.light #files tr:hover td {
+	background: #fff;
+}
+html.light #files tbody a.play {
+	color: #c0f;
+}
+html.light tr.play td {
+	background: #fc5;
+}
+html.light tr.play a {
+	color: #406;
+}
+html.light #files th:hover .cfg,
+html.light #files th.min .cfg {
+	background: #ccc;
+}
+html.light #files > thead > tr > th.min span {
+	background: linear-gradient(90deg, rgba(204,204,204,0), rgba(204,204,204,0.5) 70%, #ccc);
+}
+html.light #blocked {
+	background: #eee;
+}
+html.light #blk_play a,
+html.light #blk_abrt a {
+	background: #fff;
+	box-shadow: 0 .2em .4em #ddd;
+}
+html.light #widget a {
+	color: #fc5;
+}
+html.light #files tr.sel:hover td {
+	background: #c37;
+}
+html.light #files tr.sel td {
+	color: #fff;
+}
+html.light #files tr.sel a {
+	color: #fff;
+}
+html.light #files tr.sel a.play.act {
+	color: #fb0;
+}
+html.light input[type="checkbox"] + label {
+	color: #333;
+}
+html.light .opview input[type="text"] {
+	background: #fff;
+	color: #333;
+	box-shadow: 0 0 2px #888;
+	border-color: #38d;
+}
+html.light #ops:hover #opdesc {
+	background: #fff;
+	box-shadow: 0 .3em 1em #ccc;
+}
+html.light #opdesc code {
+	background: #060;
+	color: #fff;
+}
+html.light #u2tab a>span,
+html.light #files td div span {
+	color: #000;
+}
+html.light #path {
+	background: #f7f7f7;
+	text-shadow: none;
+	box-shadow: 0 0 .3em #bbb;
+}
+html.light #path a {
+	color: #333;
+}
+html.light #path a:not(:last-child)::after {
+	border-color: #ccc;
+	background: none;
+	border-width: .1em .1em 0 0;
+	margin: -.2em .3em -.2em -.3em;
+}
+html.light #path a:hover {
+	background: none;
+	color: #60a;
+}
+html.light #files tbody div a {
+	color: #d38;
+}
+html.light #files a:hover,
+html.light #files tr.sel a:hover {
+	color: #000;
+	background: #fff;
+}
+html.light #tree {
+	scrollbar-color: #a70 #ddd;
+}
+html.light #tree::-webkit-scrollbar-track,
+html.light #tree::-webkit-scrollbar {
+	background: #ddd;
+}
+#tree::-webkit-scrollbar-thumb {
+	background: #da0;
+}

copyparty/web/browser.html

@@ -13,15 +13,15 @@
 <body>
     <div id="ops">
         <a href="#" data-dest="" data-desc="close submenu">---</a>
-        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
         {%- if have_up2k_idx %}
+        <a href="#" data-perm="read" data-dest="search" data-desc="search for files by attributes, path/name, music tags, or any combination of those.&lt;br /&gt;&lt;br /&gt;&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,&lt;br /&gt;&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,&lt;br /&gt;&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>
         <a href="#" data-dest="up2k" data-desc="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>
         {%- else %}
         <a href="#" data-perm="write" data-dest="up2k" data-desc="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>
         {%- endif %}
         <a href="#" data-perm="write" data-dest="bup" data-desc="bup: basic uploader, even supports netscape 4.0">🎈</a>
         <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
-        <a href="#" data-perm="write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
+        <a href="#" data-perm="read write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
         <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
         <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
         <div id="opdesc"></div>
@@ -39,12 +39,17 @@
     {%- include 'upload.html' %}
 
     <div id="op_cfg" class="opview opbox">
+        <h3>switches</h3>
+        <div>
+            <a id="tooltips" class="tglbtn" href="#">tooltips</a>
+            <a id="lightmode" class="tglbtn" href="#">lightmode</a>
+        </div>
+        {%- if have_zip %}
+        <h3>folder download</h3>
+        <div id="arc_fmt"></div>
+        {%- endif %}
         <h3>key notation</h3>
         <div id="key_notation"></div>
-        <h3>tooltips</h3>
-        <div>
-            <a id="tooltips" class="tglbtn" href="#">enable</a>
-        </div>
     </div>
     
     <h1 id="path">
@@ -70,7 +75,7 @@
     <table id="files">
         <thead>
             <tr>
-                <th></th>
+                <th name="lead"><span>c</span></th>
                 <th name="href"><span>File Name</span></th>
                 <th name="sz" sort="int"><span>Size</span></th>
                 {%- for k in taglist %}
@@ -110,7 +115,14 @@
     {%- endif %}
 
     <div id="widget">
-        <div id="wtoggle">♫</div>
+        <div id="wtoggle">
+            <span id="wzip">
+                <a href="#" id="selall">sel.<br />all</a>
+                <a href="#" id="selinv">sel.<br />inv.</a>
+                <a href="#" id="selzip">zip</a>
+            </span><a
+                href="#" id="wtico">♫</a>
+        </div>
         <div id="widgeti">
             <div id="pctl"><a href="#" id="bprev">⏮</a><a href="#" id="bplay">▶</a><a href="#" id="bnext">⏭</a></div>
             <canvas id="pvol" width="288" height="38"></canvas>

copyparty/web/browser2.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>{{ title }}</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+    <style>
+        html{font-family:sans-serif}
+        td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
+        a{display:block}
+    </style>
+</head>
+
+<body>
+    {%- if srv_info %}
+    <p><span>{{ srv_info }}</span></p>
+    {%- endif %}
+
+    {%- if have_b_u %}
+    <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
+        <input type="hidden" name="act" value="bput" />
+        <input type="file" name="f" multiple /><br />
+        <input type="submit" value="start upload" />
+    </form>
+    <br />
+    {%- endif %}
+
+    {%- if logues[0] %}
+    <div>{{ logues[0] }}</div><br />
+    {%- endif %}
+
+    <table id="files">
+        <thead>
+            <tr>
+                <th name="lead"><span>c</span></th>
+                <th name="href"><span>File Name</span></th>
+                <th name="sz" sort="int"><span>Size</span></th>
+                <th name="ts"><span>Date</span></th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
+
+{%- for f in files %}
+    <tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+{%- endfor %}
+
+        </tbody>
+    </table>
+    
+    {%- if logues[1] %}
+    <div>{{ logues[1] }}</div><br />
+    {%- endif %}
+    
+    <h2><a href="{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
+
+</body>
+</html>

copyparty/web/md.css

@@ -50,6 +50,9 @@ pre code:last-child {
 pre code::before {
 	content: counter(precode);
 	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
 	display: inline-block;
 	text-align: right;
 	font-size: .75em;
@@ -591,12 +594,3 @@ blink {
 		color: #940;
 	}
 }
-
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-*/

copyparty/web/md.html

@@ -138,16 +138,16 @@ var md_opt = {
         document.documentElement.setAttribute("class", dark ? "dark" : "");
         btn.innerHTML = "go " + (dark ? "light" : "dark");
         if (window.localStorage)
-            localStorage.setItem('darkmode', dark ? 1 : 0);
+            localStorage.setItem('lightmode', dark ? 0 : 1);
     };
     btn.onclick = toggle;
-    if (window.localStorage && localStorage.getItem('darkmode') == 1)
+    if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		toggle();
 })();
 
 	</script>
     <script src="/.cpr/util.js"></script>
-	<script src="/.cpr/deps/marked.full.js"></script>
+	<script src="/.cpr/deps/marked.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}
 	<script src="/.cpr/md2.js"></script>

copyparty/web/md.js

@@ -46,7 +46,7 @@ function statify(obj) {
     var ua = navigator.userAgent;
     if (ua.indexOf(') Gecko/') !== -1 && /Linux| Mac /.exec(ua)) {
         // necessary on ff-68.7 at least
-        var s = document.createElement('style');
+        var s = mknod('style');
         s.innerHTML = '@page { margin: .5in .6in .8in .6in; }';
         console.log(s.innerHTML);
         document.head.appendChild(s);
@@ -175,12 +175,12 @@ function md_plug_err(ex, js) {
         msg = "Line " + ln + ", " + msg;
         var lns = js.split('\n');
         if (ln < lns.length) {
-            o = document.createElement('span');
+            o = mknod('span');
             o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
             o.textContent = lns[ln - 1];
         }
     }
-    errbox = document.createElement('div');
+    errbox = mknod('div');
     errbox.setAttribute('id', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;

copyparty/web/md2.css

@@ -119,7 +119,6 @@ html.dark #toast {
 	text-align: center;
 	padding: .6em 0;
 	position: fixed;
-    z-index: 9001;
 	top: 30%;
 	transition: opacity 0.2s ease-in-out;
 	opacity: 1;

copyparty/web/md2.js

@@ -16,7 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = document.createElement('div');
+    var d = mknod('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
@@ -71,7 +71,7 @@ var map_src = [];
 var map_pre = [];
 function genmap(dom, oldmap) {
     var find = nlines;
-    while (oldmap && find --> 0) {
+    while (oldmap && find-- > 0) {
         var tmap = genmapq(dom, '*[data-ln="' + find + '"]');
         if (!tmap || !tmap.length)
             continue;
@@ -94,7 +94,7 @@ var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
     function draw_md() {
-        var t0 = new Date().getTime();
+        var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
 
@@ -110,7 +110,7 @@ var draw_md = (function () {
 
         cls(ebi('save'), 'disabled', src == server_md);
 
-        var t1 = new Date().getTime();
+        var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
     }
 
@@ -252,7 +252,7 @@ function Modpoll() {
         }
 
         console.log('modpoll...');
-        var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
         var xhr = new XMLHttpRequest();
         xhr.modpoll = this;
         xhr.open('GET', url, true);
@@ -399,7 +399,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
@@ -455,7 +455,7 @@ function toast(autoclose, style, width, msg) {
         ok.parentNode.removeChild(ok);
 
     style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
-    ok = document.createElement('div');
+    ok = mknod('div');
     ok.setAttribute('id', 'toast');
     ok.setAttribute('style', style);
     ok.innerHTML = msg;
@@ -1049,7 +1049,7 @@ action_stack = (function () {
         var p1 = from.length,
             p2 = to.length;
 
-        while (p1 --> 0 && p2 --> 0)
+        while (p1-- > 0 && p2-- > 0)
             if (from[p1] != to[p2])
                 break;
 
@@ -1142,14 +1142,3 @@ action_stack = (function () {
         _ref: ref
     }
 })();
-
-/*
-ebi('help').onclick = function () {
-    var c1 = getComputedStyle(dom_src).cssText.split(';');
-    var c2 = getComputedStyle(dom_ref).cssText.split(';');
-    var max = Math.min(c1.length, c2.length);
-    for (var a = 0; a < max; a++)
-        if (c1[a] !== c2[a])
-            console.log(c1[a] + '\n' + c2[a]);
-}
-*/

copyparty/web/mde.css

@@ -60,16 +60,6 @@ html .editor-toolbar>button.save.force-save {
 	background: #f97;
 }
 
-/*
-*[data-ln]:before {
-	content: attr(data-ln);
-	font-size: .8em;
-	margin: 0 .4em;
-	color: #f0c;
-}
-.cm-header { font-size: .4em !important }
-*/
-
 
 
 

copyparty/web/mde.html

@@ -31,12 +31,12 @@ var md_opt = {
 
 var lightswitch = (function () {
 	var fun = function () {
-		var dark = !!!document.documentElement.getAttribute("class");
+		var dark = !document.documentElement.getAttribute("class");
 		document.documentElement.setAttribute("class", dark ? "dark" : "");
 		if (window.localStorage)
-			localStorage.setItem('darkmode', dark ? 1 : 0);
+			localStorage.setItem('lightmode', dark ? 0 : 1);
 	};
-	if (window.localStorage && localStorage.getItem('darkmode') == 1)
+	if (window.localStorage && localStorage.getItem('lightmode') != 1)
 		fun();
 	
 	return fun;

copyparty/web/mde.js

@@ -71,7 +71,7 @@ var mde = (function () {
 })();
 
 function set_jumpto() {
-    document.querySelector('.editor-preview-side').onclick = jumpto;
+    QS('.editor-preview-side').onclick = jumpto;
 }
 
 function jumpto(ev) {
@@ -94,7 +94,7 @@ function md_changed(mde, on_srv) {
         window.md_saved = mde.value();
 
     var md_now = mde.value();
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
 
     if (md_now == window.md_saved)
         save_btn.classList.add('disabled');
@@ -105,7 +105,7 @@ function md_changed(mde, on_srv) {
 }
 
 function save(mde) {
-    var save_btn = document.querySelector('.editor-toolbar button.save');
+    var save_btn = QS('.editor-toolbar button.save');
     if (save_btn.classList.contains('disabled')) {
         alert('there is nothing to save');
         return;
@@ -212,7 +212,7 @@ function save_chk() {
     last_modified = this.lastmod;
     md_changed(this.mde, true);
 
-    var ok = document.createElement('div');
+    var ok = mknod('div');
     ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
     ok.innerHTML = 'OK✔️';
     var parent = ebi('m');

copyparty/web/msg.css

@@ -3,7 +3,7 @@ html,body,tr,th,td,#files,a {
 	background: none;
 	font-weight: inherit;
 	font-size: inherit;
-	padding: none;
+	padding: 0;
 	border: none;
 }
 html {

copyparty/web/splash.html

@@ -13,23 +13,27 @@
     <div id="wrap">
         <p>hello {{ this.uname }}</p>
 
+        {%- if rvol %}
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
+        {%- if wvol %}
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}">/{{ mp }}</a></li>
+            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
             {% endfor %}
         </ul>
+        {%- endif %}
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/">
+            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />
@@ -38,7 +42,7 @@
     </div>
     <script>
 
-if (window.localStorage && localStorage.getItem('darkmode') == 1)
+if (window.localStorage && localStorage.getItem('lightmode') != 1)
     document.documentElement.setAttribute("class", "dark");
 
 </script>

copyparty/web/upload.css

@@ -19,6 +19,11 @@
 	color: #f87;
 	padding: .5em;
 }
+#u2err.msg {
+	color: #999;
+	padding: .5em;
+	font-size: .9em;
+}
 #u2btn {
 	color: #eee;
 	background: #555;
@@ -47,6 +52,11 @@
 	margin: -1.5em 0;
 	padding: .8em 0;
 	width: 100%;
+	max-width: 12em;
+	display: inline-block;
+}
+#u2conf #u2btn_cw {
+	text-align: right;
 }
 #u2notbtn {
 	display: none;
@@ -72,6 +82,7 @@
 }
 #u2tab td:nth-child(2) {
 	width: 5em;
+	white-space: nowrap;
 }
 #u2tab td:nth-child(3) {
 	width: 40%;
@@ -80,15 +91,51 @@
 	font-family: sans-serif;
 	width: auto;
 }
-#u2tab tr+tr:hover td {
+#u2tab tbody tr:hover td {
 	background: #222;
 }
+#u2cards {
+	padding: 1em 0 .3em 1em;
+	margin: 1.5em auto -2.5em auto;
+	text-align: center;
+	overflow: hidden;
+}
+#u2cards.w {
+	width: 45em;
+	text-align: left;
+}
+#u2cards a {
+	padding: .2em 1em;
+	border: 1px solid #777;
+	border-width: 0 0 1px 0;
+	background: linear-gradient(to bottom, #333, #222);
+}
+#u2cards a:first-child {
+	border-radius: .4em 0 0 0;
+}
+#u2cards a:last-child {
+	border-radius: 0 .4em 0 0;
+}
+#u2cards a.act {
+	padding-bottom: .5em;
+	border-width: 1px 1px .1em 1px;
+	border-radius: .3em .3em 0 0;
+	margin-left: -1px;
+	background: linear-gradient(to bottom, #464, #333 80%);
+	box-shadow: 0 -.17em .67em #280;
+	border-color: #7c5 #583 #333 #583;
+	position: relative;
+	color: #fd7;
+}
+#u2cards span {
+	color: #fff;
+}
 #u2conf {
 	margin: 1em auto;
 	width: 30em;
 }
 #u2conf.has_btn {
-	width: 46em;
+	width: 48em;
 }
 #u2conf * {
 	text-align: center;
@@ -99,12 +146,16 @@
 	outline: none;
 }
 #u2conf .txtbox {
-	width: 4em;
+	width: 3em;
 	color: #fff;
 	background: #444;
 	border: 1px solid #777;
 	font-size: 1.2em;
 	padding: .15em 0;
+	height: 1.05em;
+}
+#u2conf .txtbox.err {
+	background: #922;
 }
 #u2conf a {
 	color: #fff;
@@ -113,13 +164,12 @@
 	border-radius: .1em;
 	font-size: 1.5em;
 	padding: .1em 0;
-	margin: 0 -.25em;
+	margin: 0 -1px;
 	width: 1.5em;
 	height: 1em;
 	display: inline-block;
 	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
+	bottom: -0.08em;
 }
 #u2conf input+a {
 	background: #d80;
@@ -130,7 +180,6 @@
 	height: 1em;
 	padding: .4em 0;
 	display: block;
-	user-select: none;
 	border-radius: .25em;
 }
 #u2conf input[type="checkbox"] {
@@ -170,12 +219,13 @@
 	text-align: center;
 	overflow: hidden;
 	margin: 0 -2em;
-	height: 0;
 	padding: 0 1em;
+	height: 0;
 	opacity: .1;
 	transition: all 0.14s ease-in-out;
-	border-radius: .4em;
 	box-shadow: 0 .2em .5em #222;
+	border-radius: .4em;
+	z-index: 1;
 }
 #u2cdesc.show {
 	padding: 1em;
@@ -193,24 +243,6 @@
 .prog {
 	font-family: monospace;
 }
-.prog>div {
-	display: inline-block;
-	position: relative;
-	overflow: hidden;
-	margin: 0;
-	padding: 0;
-	height: 1.1em;
-	margin-bottom: -.15em;
-	box-shadow: -1px -1px 0 inset rgba(255,255,255,0.1);
-}
-.prog>div>div {
-	width: 0%;
-	position: absolute;
-	left: 0;
-	top: 0;
-	bottom: 0;
-	background: #0a0;
-}
 #u2tab a>span {
 	font-weight: bold;
 	font-style: italic;
@@ -221,3 +253,44 @@
 	float: right;
 	margin-bottom: -.3em;
 }
+
+
+
+
+
+html.light #u2btn {
+	box-shadow: .4em .4em 0 #ccc;
+}
+html.light #u2cards span {
+	color: #000;
+}
+html.light #u2cards a {
+	background: linear-gradient(to bottom, #eee, #fff);
+}
+html.light #u2cards a.act {
+	color: #037;
+	background: inherit;
+	box-shadow: 0 -.17em .67em #0ad;
+	border-color: #09c #05a #eee #05a;
+}
+html.light #u2conf .txtbox {
+	background: #fff;
+	color: #444;
+}
+html.light #u2conf .txtbox.err {
+	background: #f96;
+	color: #300;
+}
+html.light #u2cdesc {
+	background: #fff;
+	border: none;
+}
+html.light #op_up2k.srch #u2btn {
+	border-color: #a80;
+}
+html.light #u2foot {
+	color: #000;
+}
+html.light #u2tab tbody tr:hover td {
+	background: #fff;
+}

copyparty/web/upload.html

@@ -1,7 +1,7 @@
 
     <div id="op_bup" class="opview opbox act">
         <div id="u2err"></div>
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="bput" />
             <input type="file" name="f" multiple><br />
             <input type="submit" value="start upload">
@@ -9,7 +9,7 @@
     </div>
 
     <div id="op_mkdir" class="opview opbox act">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="mkdir" />
             <input type="text" name="name" size="30">
             <input type="submit" value="mkdir">
@@ -17,15 +17,15 @@
     </div>
 
     <div id="op_new_md" class="opview opbox">
-        <form method="post" enctype="multipart/form-data" accept-charset="utf-8">
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="hidden" name="act" value="new_md" />
             <input type="text" name="name" size="30">
             <input type="submit" value="create doc">
         </form>
     </div>
 
-    <div id="op_msg" class="opview opbox">
-        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8">
+    <div id="op_msg" class="opview opbox act">
+        <form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
             <input type="text" name="msg" size="30">
             <input type="submit" value="send msg">
         </form>
@@ -36,7 +36,7 @@
 
             <table id="u2conf">
                 <tr>
-                    <td>parallel uploads</td>
+                    <td><br />parallel uploads:</td>
                     <td rowspan="2">
                         <input type="checkbox" id="multitask" />
                         <label for="multitask" alt="continue hashing other files while uploading">🏃</label>
@@ -59,9 +59,9 @@
                 </tr>
                 <tr>
                     <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
+                        <a href="#" id="nthread_sub">&ndash;</a><input
+                            class="txtbox" id="nthread" value="2"/><a
+                            href="#" id="nthread_add">+</a><br />&nbsp;
                     </td>
                 </tr>
             </table>
@@ -73,19 +73,31 @@
             <div id="u2btn_ct">
                 <div id="u2btn">
                     <span id="u2bm"></span><br />
-                    drop files here<br />
+                    drag/drop files<br />
+                    and folders here<br />
                     (or click me)
                 </div>
             </div>
 
+            <div id="u2cards">
+                <a href="#" act="ok">ok <span>0</span></a><a
+                href="#" act="ng">ng <span>0</span></a><a
+                href="#" act="done">done <span>0</span></a><a
+                href="#" act="bz" class="act">busy <span>0</span></a><a
+                href="#" act="q">que <span>0</span></a>
+            </div>
+
             <table id="u2tab">
+                <thead>
                     <tr>
                         <td>filename</td>
                         <td>status</td>
                         <td>progress<a href="#" id="u2cleanup">cleanup</a></td>
                     </tr>
+                </thead>
+                <tbody></tbody>
             </table>
 
             <p id="u2foot"></p>
-            <p id="u2footfoot">( if you don't need lastmod timestamps, resumable uploads or progress bars just use the <a href="#" id="u2nope">basic uploader</a>)</p>
+            <p id="u2footfoot">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
     </div>

copyparty/web/util.js

@@ -1,5 +1,15 @@
 "use strict";
 
+if (!window['console'])
+    window['console'] = {
+        "log": function (msg) { }
+    };
+
+
+var clickev = window.Touch ? 'touchstart' : 'click',
+    ANDROID = /(android)/i.test(navigator.userAgent);
+
+
 // error handler for mobile devices
 function hcroak(msg) {
     document.body.innerHTML = msg;
@@ -40,9 +50,11 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
 }
 
 
-function ebi(id) {
-    return document.getElementById(id);
-}
+var ebi = document.getElementById.bind(document),
+    QS = document.querySelector.bind(document),
+    QSA = document.querySelectorAll.bind(document),
+    mknod = document.createElement.bind(document);
+
 
 function ev(e) {
     e = e || window.event;
@@ -80,7 +92,7 @@ if (!String.startsWith) {
 // https://stackoverflow.com/a/950146
 function import_js(url, cb) {
     var head = document.head || document.getElementsByTagName('head')[0];
-    var script = document.createElement('script');
+    var script = mknod('script');
     script.type = 'text/javascript';
     script.src = url;
 
@@ -110,7 +122,85 @@ function crc32(str) {
         crc = (crc >>> 8) ^ crctab[(crc ^ str.charCodeAt(i)) & 0xFF];
     }
     return ((crc ^ (-1)) >>> 0).toString(16);
-};
+}
+
+
+function clmod(obj, cls, add) {
+    var re = new RegExp('\\s*\\b' + cls + '\\s*\\b', 'g');
+    if (add == 't')
+        add = !re.test(obj.className);
+
+    obj.className = obj.className.replace(re, ' ') + (add ? ' ' + cls : '');
+}
+
+
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["lead", -1, ""], ["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v)
+                        v = uricom_dec(v)[0]
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map(function (x) { return x; });
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+    }
+    return nodes;
+}
 
 
 function sortTable(table, col, cb) {
@@ -186,9 +276,8 @@ function makeSortable(table, cb) {
 }
 
 
-
 (function () {
-    var ops = document.querySelectorAll('#ops>a');
+    var ops = QSA('#ops>a');
     for (var a = 0; a < ops.length; a++) {
         ops[a].onclick = opclick;
     }
@@ -203,25 +292,25 @@ function opclick(e) {
 
     swrite('opmode', dest || null);
 
-    var input = document.querySelector('.opview.act input:not([type="hidden"])')
+    var input = QS('.opview.act input:not([type="hidden"])')
     if (input)
         input.focus();
 }
 
 
 function goto(dest) {
-    var obj = document.querySelectorAll('.opview.act');
+    var obj = QSA('.opview.act');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
-    obj = document.querySelectorAll('#ops>a');
+    obj = QSA('#ops>a');
     for (var a = obj.length - 1; a >= 0; a--)
-        obj[a].classList.remove('act');
+        clmod(obj[a], 'act');
 
     if (dest) {
         var ui = ebi('op_' + dest);
-        ui.classList.add('act');
-        document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
+        clmod(ui, 'act', true);
+        QS('#ops>a[data-dest=' + dest + ']').className += " act";
 
         var fn = window['goto_' + dest];
         if (fn)
@@ -237,7 +326,10 @@ function goto(dest) {
     goto();
     var op = sread('opmode');
     if (op !== null && op !== '.')
+        try {
             goto(op);
+        }
+        catch (ex) { }
 })();
 
 
@@ -405,8 +497,7 @@ function bcfg_upd_ui(name, val) {
     if (o.getAttribute('type') == 'checkbox')
         o.checked = val;
     else if (o) {
-        var fun = val ? 'add' : 'remove';
-        o.classList[fun]('on');
+        clmod(o, 'on', val);
     }
 }
 

docs/example.conf

@@ -32,9 +32,13 @@ r
 
 # and a folder where anyone can upload
 # but nobody can see the contents
+# and set the e2d flag to enable the uploads database
+# and set the nodupe flag to reject duplicate uploads
 /home/ed/inc
 /dump
 w
+c e2d
+c nodupe
 
 # this entire config file can be replaced with these arguments:
 # -u ed:123 -u k:k -v .::r:aed -v priv:priv:rk:aed -v /home/ed/Music:music:r -v /home/ed/inc:dump:w

docs/minimal-up2k.html

@@ -0,0 +1,31 @@
+<!--
+  save this as .epilogue.html inside a write-only folder to declutter the UI
+-->
+
+<style>
+
+    /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
+
+    #ops, #tree, #path,  /* main tabs and navigators (tree/breadcrumbs) */
+
+    #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
+
+    #u2cards  /* and the upload progress tabs */
+
+    {display: none !important}  /* do it! */
+
+
+
+    /* add some margins because now it's weird */
+    .opview {margin-top: 2.5em}
+    #op_up2k {margin-top: 3em}
+
+    /* and embiggen the upload button */
+    #u2conf #u2btn, #u2btn {padding:1.5em 0}
+
+    /* adjust the button area a bit */
+    #u2conf.has_btn {width: 35em !important; margin: 5em auto}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

docs/notes.sh

@@ -73,6 +73,13 @@ shab64() { sp=$1; f="$2"; v=0; sz=$(stat -c%s "$f"); while true; do w=$((v+sp*10
 command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (time wget http://127.0.0.1:3923/?ls -qO- | jq -C '.files[]|{sz:.sz,ta:.tags.artist,tb:.tags.".bpm"}|del(.[]|select(.==null))' | awk -F\" '/"/{t[$2]++} END {for (k in t){v=t[k];p=sprintf("%" (v+1) "s",v);gsub(/ /,"#",p);printf "\033[36m%s\033[33m%s   ",k,p}}') 2>&1 | awk -v ts=$t 'NR==1{t1=$0} NR==2{sub(/.*0m/,"");sub(/s$/,"");t2=$0;c=2; if(t2>0.3){c=3} if(t2>0.8){c=1} } END{sub(/[0-9]{6}$/,"",ts);printf "%s   \033[3%dm%s   %s\033[0m\n",ts,c,t2,t1}'; sleep 0.1 || break; done
 
 
+##
+## js oneliners
+
+# get all up2k search result URLs
+var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
+
+
 ##
 ## sqlite3 stuff
 
@@ -83,6 +90,9 @@ sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2
 time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
 cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
 
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
 
 ##
 ## media
@@ -126,6 +136,16 @@ pip install virtualenv
 # readme toc
 cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
 
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
+
 
 ##
 ## http 206
@@ -151,7 +171,7 @@ Range: bytes=26-         Content-Range: bytes */26
 
 var tsh = [];
 function convert_markdown(md_text, dest_dom) {
-    tsh.push(new Date().getTime());
+    tsh.push(Date.now());
     while (tsh.length > 10)
         tsh.shift();
     if (tsh.length > 1) {

scripts/make-sfx.sh

@@ -45,11 +45,13 @@ pybin=$(command -v python3 || command -v python) || {
 	exit 1
 }
 
+use_gz=
 do_sh=1
 do_py=1
 while [ ! -z "$1" ]; do
 	[ "$1" = clean  ] && clean=1  && shift && continue
 	[ "$1" = re     ] && repack=1 && shift && continue
+	[ "$1" = gz     ] && use_gz=1 && shift && continue
 	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
 	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
 	[ "$1" = no-sh  ] && do_sh=   && shift && continue
@@ -115,7 +117,7 @@ cd sfx
 ver=
 git describe --tags >/dev/null 2>/dev/null && {
 	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
-	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//; s/-g?/./g')";
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//')";
 	t_ver=
 
 	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
@@ -161,7 +163,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -197,23 +199,34 @@ find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
 	tmv "$f"
 done
 
+echo gen tarlist
+for d in copyparty dep-j2; do find $d -type f; done |
+sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
+sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
+
+(grep -vE 'gz$' list1; grep -E 'gz$' list1) >list
+
 echo creating tar
 args=(--owner=1000 --group=1000)
 [ "$OSTYPE" = msys ] &&
 	args=()
 
-tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
+tar -cf tar "${args[@]}" --numeric-owner -T list
+
+pc=bzip2
+pe=bz2
+[ $use_gz ] && pc=gzip && pe=gz
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-[ $do_py ] && { for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2; }
+[ $do_py ] && { for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2; }
 [ $do_sh ] && { for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz  | tail -n 1) tar.xz; }
 rm t.* || true
 exts=()
 
 
 [ $do_sh ] && {
-exts+=(sh)
+exts+=(.sh)
 echo creating unix sfx
 (
 	sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
@@ -224,17 +237,30 @@ echo creating unix sfx
 
 
 [ $do_py ] && {
-exts+=(py)
-echo creating generic sfx
-$pybin ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
-mv sfx.out $sfx_out.py
-chmod 755 $sfx_out.*
+	echo creating generic sfx
+
+	py=../scripts/sfx.py
+	suf=
+	[ $use_gz ] && {
+		sed -r 's/"r:bz2"/"r:gz"/' <$py >$py.t
+		py=$py.t
+		suf=-gz
+	}
+
+	$pybin $py --sfx-make tar.bz2 $ver $ts
+	mv sfx.out $sfx_out$suf.py
+	
+	exts+=($suf.py)
+	[ $use_gz ] &&
+		rm $py
 }
 
 
+chmod 755 $sfx_out*
+
 printf "done:\n"
 for ext in ${exts[@]}; do
-	printf "  %s\n" "$(realpath $sfx_out)."$ext
+	printf "  %s\n" "$(realpath $sfx_out)"$ext
 done
 
 # apk add bash python3 tar xz bzip2

scripts/sfx.py

@@ -2,7 +2,8 @@
 # coding: latin-1
 from __future__ import print_function, unicode_literals
 
-import os, sys, time, shutil, runpy, tarfile, hashlib, platform, tempfile, traceback
+import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform, tempfile, traceback
+import subprocess as sp
 
 """
 run me with any version of python, i will unpack and run copyparty
@@ -26,21 +27,21 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
+WINDOWS = sys.platform in ["win32", "msys"]
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
-cpp = None
 
 
-def eprint(*args, **kwargs):
-    kwargs["file"] = sys.stderr
-    print(*args, **kwargs)
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    print(*a, **ka)
 
 
-def msg(*args, **kwargs):
-    if args:
-        args = ["[SFX]", args[0]] + list(args[1:])
+def msg(*a, **ka):
+    if a:
+        a = ["[SFX]", a[0]] + list(a[1:])
 
-    eprint(*args, **kwargs)
+    eprint(*a, **ka)
 
 
 # skip 1
@@ -155,6 +156,9 @@ def encode(data, size, cksum, ver, ts):
                 skip = True
                 continue
 
+            if ln.strip().startswith("# fmt: "):
+                continue
+
             unpk += ln + "\n"
 
         for k, v in [
@@ -208,11 +212,11 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    hasher = hashlib.md5()
+    h = hashlib.md5()
     for block in yieldfile(fn):
-        hasher.update(block)
+        h.update(block)
 
-    return hasher.hexdigest()
+    return h.hexdigest()
 
 
 def unpack():
@@ -221,9 +225,10 @@ def unpack():
     tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
-    final = os.path.join(top, name)
-    mine = os.path.join(top, withpid)
-    tar = os.path.join(mine, "tar")
+    opj = os.path.join
+    final = opj(top, name)
+    mine = opj(top, withpid)
+    tar = opj(mine, "tar")
 
     try:
         if tag in os.listdir(final):
@@ -232,28 +237,24 @@ def unpack():
     except:
         pass
 
-    nwrite = 0
+    sz = 0
     os.mkdir(mine)
     with open(tar, "wb") as f:
         for buf in get_payload():
-            nwrite += len(buf)
+            sz += len(buf)
             f.write(buf)
 
-    if nwrite != SIZE:
-        t = "\n\n  bad file:\n    expected {} bytes, got {}\n".format(SIZE, nwrite)
-        raise Exception(t)
-
-    cksum = hashfile(tar)
-    if cksum != CKSUM:
-        t = "\n\n  bad file:\n    {} expected,\n    {} obtained\n".format(CKSUM, cksum)
-        raise Exception(t)
+    ck = hashfile(tar)
+    if ck != CKSUM:
+        t = "\n\nexpected {} ({} byte)\nobtained {} ({} byte)\nsfx corrupt"
+        raise Exception(t.format(CKSUM, SIZE, ck, sz))
 
     with tarfile.open(tar, "r:bz2") as tf:
         tf.extractall(mine)
 
     os.remove(tar)
 
-    with open(os.path.join(mine, tag), "wb") as f:
+    with open(opj(mine, tag), "wb") as f:
         f.write(b"h\n")
 
     try:
@@ -271,26 +272,26 @@ def unpack():
     except:
         pass
 
+    for fn in u8(os.listdir(top)):
+        if fn.startswith(name) and fn != withpid:
+            try:
+                old = opj(top, fn)
+                if time.time() - os.path.getmtime(old) > 86400:
+                    shutil.rmtree(old)
+            except:
+                pass
+
     try:
         os.symlink(mine, final)
     except:
         try:
             os.rename(mine, final)
+            return final
         except:
             msg("reloc fail,", mine)
+
     return mine
 
-    for fn in u8(os.listdir(top)):
-        if fn.startswith(name) and fn not in [name, withpid]:
-            try:
-                old = os.path.join(top, fn)
-                if time.time() - os.path.getmtime(old) > 10:
-                    shutil.rmtree(old)
-            except:
-                pass
-
-    return final
-
 
 def get_payload():
     """yields the binary data attached to script"""
@@ -306,46 +307,57 @@ def get_payload():
         if ofs < 0:
             raise Exception("could not find archive marker")
 
-        # start reading from the final b"\n"
+        # start at final b"\n"
         fpos = ofs + len(ptn) - 3
-        # msg("tar found at", fpos)
         f.seek(fpos)
         dpos = 0
-        leftovers = b""
+        rem = b""
         while True:
             rbuf = f.read(1024 * 32)
             if rbuf:
-                buf = leftovers + rbuf
+                buf = rem + rbuf
                 ofs = buf.rfind(b"\n")
                 if len(buf) <= 4:
-                    leftovers = buf
+                    rem = buf
                     continue
 
                 if ofs >= len(buf) - 4:
-                    leftovers = buf[ofs:]
+                    rem = buf[ofs:]
                     buf = buf[:ofs]
                 else:
-                    leftovers = b"\n# "
+                    rem = b"\n# "
             else:
-                buf = leftovers
+                buf = rem
 
             fpos += len(buf) + 1
-            buf = (
-                buf.replace(b"\n# ", b"")
-                .replace(b"\n#r", b"\r")
-                .replace(b"\n#n", b"\n")
-            )
-            dpos += len(buf) - 1
+            for a, b in [[b"\n# ", b""], [b"\n#r", b"\r"], [b"\n#n", b"\n"]]:
+                buf = buf.replace(a, b)
 
+            dpos += len(buf) - 1
             yield buf
 
             if not rbuf:
                 break
 
 
+def utime(top):
+    i = 0
+    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    while WINDOWS:
+        t = int(time.time())
+        if i:
+            msg("utime {}, {}".format(i, t))
+
+        for f in files:
+            os.utime(f, (t, t))
+
+        i += 1
+        time.sleep(78123)
+
+
 def confirm(rv):
     msg()
-    msg(traceback.format_exc())
+    msg("retcode", rv if rv else traceback.format_exc())
     msg("*** hit enter to exit ***")
     try:
         raw_input() if PY2 else input()
@@ -355,37 +367,59 @@ def confirm(rv):
     sys.exit(rv)
 
 
-def run(tmp, j2ver):
-    global cpp
-
-    msg("jinja2:", j2ver or "bundled")
+def run(tmp, j2):
+    msg("jinja2:", j2 or "bundled")
     msg("sfxdir:", tmp)
     msg()
 
-    # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
+    # block systemd-tmpfiles-clean.timer
     try:
         import fcntl
 
         fd = os.open(tmp, os.O_RDONLY)
         fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        tmp = os.readlink(tmp)  # can't flock a symlink, even with O_NOFOLLOW
-    except:
-        pass
+    except Exception as ex:
+        if not WINDOWS:
+            msg("\033[31mflock:", repr(ex))
+
+    t = threading.Thread(target=utime, args=(tmp,))
+    t.daemon = True
+    t.start()
 
     ld = [tmp, os.path.join(tmp, "dep-j2")]
-    if j2ver:
+    if j2:
         del ld[-1]
 
+    if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
+        run_s(ld)
+    else:
+        run_i(ld)
+
+
+def run_i(ld):
     for x in ld:
         sys.path.insert(0, x)
 
-    try:
-        runpy.run_module(str("copyparty"), run_name=str("__main__"))
-    except SystemExit as ex:
-        if ex.code:
-            confirm(ex.code)
-    except:
-        confirm(1)
+    from copyparty.__main__ import main as p
+
+    p()
+
+
+def run_s(ld):
+    # fmt: off
+    c = "import sys,runpy;" + "".join(['sys.path.insert(0,r"' + x + '");' for x in ld]) + 'runpy.run_module("copyparty",run_name="__main__")'
+    c = [str(x) for x in [sys.executable, "-c", c] + list(sys.argv[1:])]
+    # fmt: on
+    msg("\n", c, "\n")
+    p = sp.Popen(c)
+
+    def bye(*a):
+        p.send_signal(signal.SIGINT)
+
+    signal.signal(signal.SIGTERM, bye)
+    p.wait()
+
+    raise SystemExit(p.returncode)
 
 
 def main():
@@ -419,14 +453,23 @@ def main():
 
     # skip 0
 
-    tmp = unpack()
+    tmp = os.path.realpath(unpack())
 
     try:
-        from jinja2 import __version__ as j2ver
+        from jinja2 import __version__ as j2
     except:
-        j2ver = None
+        j2 = None
 
-    run(tmp, j2ver)
+    try:
+        run(tmp, j2)
+    except SystemExit as ex:
+        c = ex.code
+        if c not in [0, -15]:
+            confirm(ex.code)
+    except KeyboardInterrupt:
+        pass
+    except:
+        confirm(0)
 
 
 if __name__ == "__main__":

scripts/speedtest-fs.py

@@ -17,14 +17,15 @@ __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
 
-def get_spd(nbyte, nsec):
+def get_spd(nbyte, nfiles, nsec):
     if not nsec:
-        return "0.000 MB   0.000 sec   0.000 MB/s"
+        return "0.000 MB   0 files   0.000 sec   0.000 MB/s   0.000 f/s"
 
     mb = nbyte / (1024 * 1024.0)
     spd = mb / nsec
+    nspd = nfiles / nsec
 
-    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+    return f"{mb:.3f} MB   {nfiles} files   {nsec:.3f} sec   {spd:.3f} MB/s   {nspd:.3f} f/s"
 
 
 class Inf(object):
@@ -36,6 +37,7 @@ class Inf(object):
         self.mtx_reports = threading.Lock()
 
         self.n_byte = 0
+        self.n_file = 0
         self.n_sec = 0
         self.n_done = 0
         self.t0 = t0
@@ -63,7 +65,8 @@ class Inf(object):
                 continue
 
             msgs = msgs[-64:]
-            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            spd = get_spd(self.n_byte, len(self.reports), self.n_sec)
+            msgs = [f"{spd}   {x}" for x in msgs]
             print("\n".join(msgs))
 
     def report(self, fn, n_byte, n_sec):
@@ -131,8 +134,9 @@ def main():
 
     num_threads = 8
     read_sz = 32 * 1024
+    targs = (q, inf, read_sz)
     for _ in range(num_threads):
-        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr = threading.Thread(target=worker, args=targs)
         thr.daemon = True
         thr.start()
 
@@ -151,14 +155,14 @@ def main():
     log = inf.reports
     log.sort()
     for nbyte, nsec, fn in log[-64:]:
-        print(f"{get_spd(nbyte, nsec)}   {fn}")
+        spd = get_spd(nbyte, len(log), nsec)
+        print(f"{spd}   {fn}")
 
     print()
     print("\n".join(inf.errors))
 
-    print(get_spd(inf.n_byte, t2 - t0))
+    print(get_spd(inf.n_byte, len(log), t2 - t0))
 
 
 if __name__ == "__main__":
     main()
-

tests/run.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import sys
+import runpy
+
+host = sys.argv[1]
+sys.argv = sys.argv[:1] + sys.argv[2:]
+sys.path.insert(0, ".")
+
+
+def rp():
+    runpy.run_module("unittest", run_name="__main__")
+
+
+if host == "vmprof":
+    rp()
+
+elif host == "cprofile":
+    import cProfile
+    import pstats
+
+    log_fn = "cprofile.log"
+    cProfile.run("rp()", log_fn)
+    p = pstats.Stats(log_fn)
+    p.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(64)
+
+
+"""
+python3.9 tests/run.py cprofile -v tests/test_httpcli.py
+
+python3.9 -m pip install --user vmprof
+python3.9 -m vmprof --lines -o vmprof.log tests/run.py vmprof -v tests/test_httpcli.py
+"""

tests/test_httpcli.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import io
+import os
+import time
+import shutil
+import pprint
+import tarfile
+import unittest
+
+from argparse import Namespace
+from copyparty.authsrv import AuthSrv
+from copyparty.httpcli import HttpCli
+
+from tests import util as tu
+
+
+def hdr(query):
+    h = "GET /{} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\n\r\n"
+    return h.format(query).encode("utf-8")
+
+
+class Cfg(Namespace):
+    def __init__(self, a=[], v=[], c=None):
+        super(Cfg, self).__init__(
+            a=a,
+            v=v,
+            c=c,
+            ed=False,
+            no_zip=False,
+            no_scandir=False,
+            no_sendfile=True,
+            nih=True,
+            mtp=[],
+            mte="a",
+            **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        )
+
+
+class TestHttpCli(unittest.TestCase):
+    def test(self):
+        td = os.path.join(tu.get_ramdisk(), "vfs")
+        try:
+            shutil.rmtree(td)
+        except OSError:
+            pass
+
+        os.mkdir(td)
+        os.chdir(td)
+
+        self.dtypes = ["ra", "ro", "rx", "wa", "wo", "wx", "aa", "ao", "ax"]
+        self.can_read = ["ra", "ro", "aa", "ao"]
+        self.can_write = ["wa", "wo", "aa", "ao"]
+        self.fn = "g{:x}g".format(int(time.time() * 3))
+
+        allfiles = []
+        allvols = []
+        for top in self.dtypes:
+            allvols.append(top)
+            allfiles.append("/".join([top, self.fn]))
+            for s1 in self.dtypes:
+                p = "/".join([top, s1])
+                allvols.append(p)
+                allfiles.append(p + "/" + self.fn)
+                allfiles.append(p + "/n/" + self.fn)
+                for s2 in self.dtypes:
+                    p = "/".join([top, s1, "n", s2])
+                    os.makedirs(p)
+                    allvols.append(p)
+                    allfiles.append(p + "/" + self.fn)
+
+        for fp in allfiles:
+            with open(fp, "w") as f:
+                f.write("ok {}\n".format(fp))
+
+        for top in self.dtypes:
+            vcfg = []
+            for vol in allvols:
+                if not vol.startswith(top):
+                    continue
+
+                mode = vol[-2]
+                usr = vol[-1]
+                if usr == "a":
+                    usr = ""
+
+                if "/" not in vol:
+                    vol += "/"
+
+                top, sub = vol.split("/", 1)
+                vcfg.append("{0}/{1}:{1}:{2}{3}".format(top, sub, mode, usr))
+
+            pprint.pprint(vcfg)
+
+            self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
+            self.auth = AuthSrv(self.args, self.log)
+            vfiles = [x for x in allfiles if x.startswith(top)]
+            for fp in vfiles:
+                rok, wok = self.can_rw(fp)
+                furl = fp.split("/", 1)[1]
+                durl = furl.rsplit("/", 1)[0] if "/" in furl else ""
+
+                # file download
+                h, ret = self.curl(furl)
+                res = "ok " + fp in ret
+                print("[{}] {} {} = {}".format(fp, rok, wok, res))
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, furl))
+                    self.fail()
+
+                # file browser: html
+                h, ret = self.curl(durl)
+                res = "'{}'".format(self.fn) in ret
+                print(res)
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, durl))
+                    self.fail()
+
+                # file browser: json
+                url = durl + "?ls"
+                h, ret = self.curl(url)
+                res = '"{}"'.format(self.fn) in ret
+                print(res)
+                if rok != res:
+                    print("\033[33m{}\n# {}\033[0m".format(ret, url))
+                    self.fail()
+
+                # tar
+                url = durl + "?tar"
+                h, b = self.curl(url, True)
+                # with open(os.path.join(td, "tar"), "wb") as f:
+                #    f.write(b)
+                try:
+                    tar = tarfile.open(fileobj=io.BytesIO(b)).getnames()
+                except:
+                    tar = []
+                tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
+                tar = [[x] + self.can_rw(x) for x in tar]
+                tar_ok = [x[0] for x in tar if x[1]]
+                tar_ng = [x[0] for x in tar if not x[1]]
+                self.assertEqual([], tar_ng)
+
+                if durl.split("/")[-1] in self.can_read:
+                    ref = [x for x in vfiles if self.in_dive(top + "/" + durl, x)]
+                    for f in ref:
+                        print("{}: {}".format("ok" if f in tar_ok else "NG", f))
+                    ref.sort()
+                    tar_ok.sort()
+                    self.assertEqual(ref, tar_ok)
+
+                # stash
+                h, ret = self.put(url)
+                res = h.startswith("HTTP/1.1 200 ")
+                self.assertEqual(res, wok)
+
+    def can_rw(self, fp):
+        # lowest non-neutral folder declares permissions
+        expect = fp.split("/")[:-1]
+        for x in reversed(expect):
+            if x != "n":
+                expect = x
+                break
+
+        return [expect in self.can_read, expect in self.can_write]
+
+    def in_dive(self, top, fp):
+        # archiver bails at first inaccessible subvolume
+        top = top.strip("/").split("/")
+        fp = fp.split("/")
+        for f1, f2 in zip(top, fp):
+            if f1 != f2:
+                return False
+
+        for f in fp[len(top) :]:
+            if f == self.fn:
+                return True
+            if f not in self.can_read and f != "n":
+                return False
+
+        return True
+
+    def put(self, url):
+        buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
+        buf = buf.format(url, len(url) + 4).encode("utf-8")
+        conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
+        HttpCli(conn).run()
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def curl(self, url, binary=False):
+        conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
+        HttpCli(conn).run()
+        if binary:
+            h, b = conn.s._reply.split(b"\r\n\r\n", 1)
+            return [h.decode("utf-8"), b]
+
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def log(self, src, msg, c=0):
+        # print(repr(msg))
+        pass

tests/test_vfs.py

@@ -3,22 +3,24 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import time
 import json
 import shutil
 import tempfile
 import unittest
-import subprocess as sp  # nosec
 
 from textwrap import dedent
 from argparse import Namespace
 from copyparty.authsrv import AuthSrv
 from copyparty import util
 
+from tests import util as tu
+
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
 
 
@@ -49,52 +51,11 @@ class TestVFS(unittest.TestCase):
         real = [x[0] for x in real]
         return fsdir, real, virt
 
-    def runcmd(self, *argv):
-        p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
-        stdout, stderr = p.communicate()
-        stdout = stdout.decode("utf-8")
-        stderr = stderr.decode("utf-8")
-        return [p.returncode, stdout, stderr]
-
-    def chkcmd(self, *argv):
-        ok, sout, serr = self.runcmd(*argv)
-        if ok != 0:
-            raise Exception(serr)
-
-        return sout, serr
-
-    def get_ramdisk(self):
-        for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
-            if os.path.exists(vol):
-                return vol
-
-        if os.path.exists("/Volumes"):
-            devname, _ = self.chkcmd("hdiutil", "attach", "-nomount", "ram://8192")
-            devname = devname.strip()
-            print("devname: [{}]".format(devname))
-            for _ in range(10):
-                try:
-                    _, _ = self.chkcmd(
-                        "diskutil", "eraseVolume", "HFS+", "cptd", devname
-                    )
-                    return "/Volumes/cptd"
-                except Exception as ex:
-                    print(repr(ex))
-                    time.sleep(0.25)
-
-            raise Exception("ramdisk creation failed")
-
-        ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
-        try:
-            os.mkdir(ret)
-        finally:
-            return ret
-
     def log(self, src, msg, c=0):
         pass
 
     def test(self):
-        td = os.path.join(self.get_ramdisk(), "vfs")
+        td = os.path.join(tu.get_ramdisk(), "vfs")
         try:
             shutil.rmtree(td)
         except OSError:
@@ -266,7 +227,7 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(list(v1), list(v2))
 
         # config file parser
-        cfg_path = os.path.join(self.get_ramdisk(), "test.cfg")
+        cfg_path = os.path.join(tu.get_ramdisk(), "test.cfg")
         with open(cfg_path, "wb") as f:
             f.write(
                 dedent(

tests/util.py

@@ -0,0 +1,97 @@
+import os
+import time
+import jinja2
+import tempfile
+import subprocess as sp
+
+from copyparty.util import Unrecv
+
+
+J2_ENV = jinja2.Environment(loader=jinja2.BaseLoader)
+J2_FILES = J2_ENV.from_string("{{ files|join('\n') }}")
+
+
+def runcmd(*argv):
+    p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode("utf-8")
+    stderr = stderr.decode("utf-8")
+    return [p.returncode, stdout, stderr]
+
+
+def chkcmd(*argv):
+    ok, sout, serr = runcmd(*argv)
+    if ok != 0:
+        raise Exception(serr)
+
+    return sout, serr
+
+
+def get_ramdisk():
+    for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
+        if os.path.exists(vol):
+            return vol
+
+    if os.path.exists("/Volumes"):
+        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://32768")
+        devname = devname.strip()
+        print("devname: [{}]".format(devname))
+        for _ in range(10):
+            try:
+                _, _ = chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
+                return "/Volumes/cptd"
+            except Exception as ex:
+                print(repr(ex))
+                time.sleep(0.25)
+
+        raise Exception("ramdisk creation failed")
+
+    ret = os.path.join(tempfile.gettempdir(), "copyparty-test")
+    try:
+        os.mkdir(ret)
+    finally:
+        return ret
+
+
+class NullBroker(object):
+    def put(*args):
+        pass
+
+
+class VSock(object):
+    def __init__(self, buf):
+        self._query = buf
+        self._reply = b""
+        self.sendall = self.send
+
+    def recv(self, sz):
+        ret = self._query[:sz]
+        self._query = self._query[sz:]
+        return ret
+
+    def send(self, buf):
+        self._reply += buf
+        return len(buf)
+
+
+class VHttpSrv(object):
+    def __init__(self):
+        self.broker = NullBroker()
+
+        aliases = ["splash", "browser", "browser2", "msg", "md", "mde"]
+        self.j2 = {x: J2_FILES for x in aliases}
+
+
+class VHttpConn(object):
+    def __init__(self, args, auth, log, buf):
+        self.s = VSock(buf)
+        self.sr = Unrecv(self.s)
+        self.addr = ("127.0.0.1", "42069")
+        self.args = args
+        self.auth = auth
+        self.log_func = log
+        self.log_src = "a"
+        self.hsrv = VHttpSrv()
+        self.nbyte = 0
+        self.workload = 0
+        self.t0 = time.time()