v0.10.21

.vscode/launch.py

@@ -12,7 +12,7 @@ sys.path.insert(0, os.getcwd())
 import jstyleson
 from copyparty.__main__ import main as copyparty
 
-with open(".vscode/launch.json", "r") as f:
+with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
 
 oj = jstyleson.loads(tj)

README.md

@@ -9,7 +9,8 @@
 turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
 
 * server runs on anything with `py2.7` or `py3.3+`
-* *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
+* browse/upload with IE4 / netscape4.0 on win3.11 (heh)
+* *resumable* uploads need `firefox 34+` / `chrome 37+` / `safari 7+`
 * code standard: `black`
 
 📷 screenshots: [browser](#the-browser) // [upload](#uploading) // [md-viewer](#markdown-viewer) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [ie4](#browser-support)
@@ -22,6 +23,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [notes](#notes)
     * [status](#status)
 * [bugs](#bugs)
+    * [general bugs](#general-bugs)
     * [not my bugs](#not-my-bugs)
 * [the browser](#the-browser)
     * [tabs](#tabs)
@@ -54,11 +56,11 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 
 download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and you're all set!
 
-running the sfx without arguments (for example doubleclicking it on Windows) will let anyone access the current folder; see `-h` for help if you want accounts and volumes etc
+running the sfx without arguments (for example doubleclicking it on Windows) will give everyone full access to the current folder; see `-h` for help if you want accounts and volumes etc
 
 you may also want these, especially on servers:
 * [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
-* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for legit https)
+* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
 
 
 ## notes
@@ -91,8 +93,8 @@ you may also want these, especially on servers:
   * ☑ tree-view
   * ☑ media player
   * ✖ thumbnails
-  * ✖ SPA (browse while uploading)
-    * currently safe using the file-tree on the left only, not folders in the file list
+  * ☑ SPA (browse while uploading)
+    * if you use the file-tree on the left only, not folders in the file list
 * server indexing
   * ☑ locate files by contents
   * ☑ search by name/path/date/size
@@ -109,6 +111,11 @@ summary: it works! you can use it! (but technically not even close to beta)
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
+
+## general bugs
+
+* all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
 * hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
@@ -175,8 +182,8 @@ you can also zip a selection of files or folders by clicking them in the browser
 ## uploading
 
 two upload methods are available in the html client:
-* 🎈 bup, the basic uploader, supports almost every browser since netscape 4.0
-* 🚀 up2k, the fancy one
+* `🎈 bup`, the basic uploader, supports almost every browser since netscape 4.0
+* `🚀 up2k`, the fancy one
 
 up2k has several advantages:
 * you can drop folders into the browser (files are added recursively)
@@ -190,6 +197,8 @@ see [up2k](#up2k) for details on how it works
 
 ![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/115978061-680b5400-a57d-11eb-9ef6-cbb5f60aeccc.png)
 
+**protip:** you can avoid scaring away users with [docs/minimal-up2k.html](docs/minimal-up2k.html) which makes it look [much simpler](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
 the up2k UI is the epitome of polished inutitive experiences:
 * "parallel uploads" specifies how many chunks to upload at the same time
 * `[🏃]` analysis of other files should continue while one is uploading
@@ -205,20 +214,18 @@ and then theres the tabs below it,
   * plus up to 3 entries each from `[done]` and `[que]` for context
 * `[que]` is all the files that are still queued
 
-protip: you can avoid scaring away users by hiding some of the UI with hacks like [docs/minimal-up2k.html](docs/minimal-up2k.html)
-
 ### file-search
 
 ![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/116008320-36919780-a614-11eb-803f-04162326a700.png)
 
-in the 🚀 up2k tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere already
+in the `[🚀 up2k]` tab, after toggling the `[🔎]` switch green, any files/folders you drop onto the dropzone will be hashed on the client-side. Each hash is sent to the server which checks if that file exists somewhere already
 
 files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
-* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else
+* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else, this is no longer the case but now i've warmed up to the idea too much
 
 adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files
 
-note that since up2k has to read the file twice, 🎈 bup can be up to 2x faster if your internet connection is faster than the read-speed of your HDD
+note that since up2k has to read the file twice, `[🎈 bup]` can be up to 2x faster in extreme cases (if your internet connection is faster than the read-speed of your HDD)
 
 up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well thanks to tls also functioning as an integrity check
 
@@ -298,6 +305,10 @@ copyparty can invoke external programs to collect additional metadata for files
 * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
 * `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
 
+*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` only do non-audio files, or `ad` do all files (d as in dontcare) 
+
+* `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`)
+
 
 ## complete examples
 
@@ -307,7 +318,7 @@ copyparty can invoke external programs to collect additional metadata for files
 
 # browser support
 
-![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/116009043-a1909d80-a617-11eb-9140-037ad6604899.png)
+![copyparty-ie4-fs8](https://user-images.githubusercontent.com/241032/118192791-fb31fe00-b446-11eb-9647-898ea8efc1f7.png)
 
 `ie` = internet-explorer, `ff` = firefox, `c` = chrome, `iOS` = iPhone/iPad, `Andr` = Android
 
@@ -333,14 +344,18 @@ copyparty can invoke external programs to collect additional metadata for files
 * `*2` using a wasm decoder which can sometimes get stuck and consumes a bit more power
 
 quick summary of more eccentric web-browsers trying to view a directory index:
-* safari (14.0.3/macos) is chrome with janky wasm, so playing opus can deadlock the javascript engine
-* safari (14.0.1/iOS) same as macos, except it recovers from the deadlocks if you poke it a bit
-* links (2.21/macports) can browse, login, upload/mkdir/msg
-* lynx (2.8.9/macports) can browse, login, upload/mkdir/msg
-* w3m (0.5.3/macports) can browse, login, upload at 100kB/s, mkdir/msg
-* netsurf (3.10/arch) is basically ie6 with much better css (javascript has almost no effect)
-* ie4 and netscape 4.0 can browse (text is yellow on white), upload with `?b=u`
-* SerenityOS (22d13d8) hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying
+
+| browser | will it blend |
+| ------- | ------------- |
+| **safari** (14.0.3/macos) | is chrome with janky wasm, so playing opus can deadlock the javascript engine |
+| **safari** (14.0.1/iOS)   | same as macos, except it recovers from the deadlocks if you poke it a bit |
+| **links** (2.21/macports) | can browse, login, upload/mkdir/msg |
+| **lynx** (2.8.9/macports) | can browse, login, upload/mkdir/msg |
+| **w3m** (0.5.3/macports)  | can browse, login, upload at 100kB/s, mkdir/msg |
+| **netsurf** (3.10/arch)   | is basically ie6 with much better css (javascript has almost no effect) | 
+| **ie4** and **netscape** 4.0  | can browse (text is yellow on white), upload with `?b=u` |
+| **SerenityOS** (22d13d8)  | hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying |
+
 
 # client examples
 
@@ -371,7 +386,7 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
 quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
 * the up2k client splits a file into an "optimal" number of chunks
   * 1 MiB each, unless that becomes more than 256 chunks
-  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256# or chunksize >= 32M
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
 * client posts the list of hashes, filename, size, last-modified
 * server creates the `wark`, an identifier for this upload
   * `sha512( salt + filesize + chunk_hashes )`
@@ -404,8 +419,8 @@ these are standalone programs and will never be imported / evaluated by copypart
 # sfx
 
 currently there are two self-contained "binaries":
-* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
-* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere, **recommended**
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos, kinda deprecated
 
 launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
 
@@ -473,6 +488,7 @@ roughly sorted by priority
   * terminate client on bad data
 * `os.copy_file_range` for up2k cloning
 * support pillow-simd
+* single sha512 across all up2k chunks? maybe
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||
 

bin/mtag/file-ext.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+
+"""
+example that just prints the file extension
+"""
+
+print(sys.argv[1].split(".")[-1])

copyparty/__main__.py

@@ -237,16 +237,14 @@ def run_argparse(argv, formatter):
     ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
     ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
     ap.add_argument("-q", action="store_true", help="quiet")
-    ap.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
     ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
-    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
     ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
@@ -274,6 +272,13 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
 
+    ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
+    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/", help="dont log URLs matching")
+    
     return ap.parse_args(args=argv[1:])
     # fmt: on
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 10, 18)
+VERSION = (0, 10, 21)
 CODENAME = "zip it"
-BUILD_DT = (2021, 5, 13)
+BUILD_DT = (2021, 5, 16)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -100,6 +100,16 @@ class HttpCli(object):
             self.ip = v.split(",")[0]
             self.log_src = self.conn.set_rproxy(self.ip)
 
+        if self.args.ihead:
+            keys = self.args.ihead
+            if "*" in keys:
+                keys = list(sorted(self.headers.keys()))
+
+            for k in keys:
+                v = self.headers.get(k)
+                if v is not None:
+                    self.log("[H] {}: \033[33m[{}]".format(k, v), 6)
+
         # split req into vpath + uparam
         uparam = {}
         if "?" not in self.req:
@@ -148,6 +158,8 @@ class HttpCli(object):
             uparam["b"] = False
             cookies["b"] = False
 
+        self.do_log = not self.conn.lf_url or not self.conn.lf_url.match(self.req)
+
         try:
             if self.mode in ["GET", "HEAD"]:
                 return self.handle_get() and self.keepalive
@@ -237,14 +249,16 @@ class HttpCli(object):
         r = ["{}={}".format(k, quotep(v)) if v else k for k, v in kv.items()]
         return "?" + "&".join(r)
 
-    def redirect(self, vpath, suf="", msg="aight", flavor="go to", use302=False):
+    def redirect(
+        self, vpath, suf="", msg="aight", flavor="go to", click=True, use302=False
+    ):
         html = self.j2(
             "msg",
             h2='<a href="/{}">{} /{}</a>'.format(
                 quotep(vpath) + suf, flavor, html_escape(vpath, crlf=True) + suf
             ),
             pre=msg,
-            click=True,
+            click=click,
         ).encode("utf-8", "replace")
 
         if use302:
@@ -254,6 +268,7 @@ class HttpCli(object):
             self.reply(html)
 
     def handle_get(self):
+        if self.do_log:
             logmsg = "{:4} {}".format(self.mode, self.req)
 
             if "range" in self.headers:
@@ -305,7 +320,9 @@ class HttpCli(object):
         return self.tx_browser()
 
     def handle_options(self):
+        if self.do_log:
             self.log("OPTIONS " + self.req)
+
         self.send_headers(
             None,
             204,
@@ -780,7 +797,7 @@ class HttpCli(object):
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
 
-                        files.append([sz, sha512_hex])
+                        files.append([sz, sha512_hex, p_file, fname])
                         self.conn.hsrv.broker.put(
                             False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
                         )
@@ -789,12 +806,16 @@ class HttpCli(object):
                 except Pebkac:
                     if fname != os.devnull:
                         fp = os.path.join(fdir, fname)
+                        fp2 = fp
+                        if self.args.dotpart:
+                            fp2 = os.path.join(fdir, "." + fname)
+
                         suffix = ".PARTIAL"
                         try:
-                            os.rename(fsenc(fp), fsenc(fp + suffix))
+                            os.rename(fsenc(fp), fsenc(fp2 + suffix))
                         except:
-                            fp = fp[: -len(suffix)]
-                            os.rename(fsenc(fp), fsenc(fp + suffix))
+                            fp2 = fp2[: -len(suffix) - 1]
+                            os.rename(fsenc(fp), fsenc(fp2 + suffix))
 
                     raise
 
@@ -811,10 +832,13 @@ class HttpCli(object):
             errmsg = "ERROR: " + errmsg
             status = "ERROR"
 
-        msg = "{0} // {1} bytes // {2:.3f} MiB/s\n".format(status, sz_total, spd)
+        msg = "{} // {} bytes // {:.3f} MiB/s\n".format(status, sz_total, spd)
 
-        for sz, sha512 in files:
-            msg += "sha512: {0} // {1} bytes\n".format(sha512[:56], sz)
+        for sz, sha512, ofn, lfn in files:
+            vpath = self.vpath + "/" + lfn
+            msg += 'sha512: {} // {} bytes // <a href="/{}">{}</a>\n'.format(
+                sha512[:56], sz, quotep(vpath), html_escape(ofn, crlf=True)
+            )
             # truncated SHA-512 prevents length extension attacks;
             # using SHA-512/224, optionally SHA-512/256 = :64
 
@@ -822,25 +846,13 @@ class HttpCli(object):
         self.log("{} {}".format(vspd, msg))
 
         if not nullwrite:
-            # TODO this is bad
             log_fn = "up.{:.6f}.txt".format(t0)
             with open(log_fn, "wb") as f:
-                f.write(
-                    (
-                        "\n".join(
-                            unicode(x)
-                            for x in [
-                                ":".join(unicode(x) for x in [self.ip, self.addr[1]]),
-                                msg.rstrip(),
-                            ]
-                        )
-                        + "\n"
-                        + errmsg
-                        + "\n"
-                    ).encode("utf-8")
-                )
+                ft = "{}:{}".format(self.ip, self.addr[1])
+                ft = "{}\n{}\n{}\n".format(ft, msg.rstrip(), errmsg)
+                f.write(ft.encode("utf-8"))
 
-        self.redirect(self.vpath, msg=msg, flavor="return to")
+        self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
         self.parser.drop()
         return True
 
@@ -1106,7 +1118,9 @@ class HttpCli(object):
         logmsg += unicode(status) + logtail
 
         if self.mode == "HEAD" or not do_send:
+            if self.do_log:
                 self.log(logmsg)
+
             return True
 
         ret = True
@@ -1120,7 +1134,9 @@ class HttpCli(object):
             logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
 
         spd = self._spd((upper - lower) - remains)
+        if self.do_log:
             self.log("{},  {}".format(logmsg, spd))
+
         return ret
 
     def tx_zip(self, fmt, uarg, vn, rem, items, dots):
@@ -1229,7 +1245,9 @@ class HttpCli(object):
 
         logmsg += unicode(status)
         if self.mode == "HEAD" or not do_send:
+            if self.do_log:
                 self.log(logmsg)
+
             return True
 
         try:
@@ -1243,7 +1261,9 @@ class HttpCli(object):
             self.log(logmsg + " \033[31md/c\033[0m")
             return False
 
+        if self.do_log:
             self.log(logmsg + " " + unicode(len(html)))
+
         return True
 
     def tx_mounts(self):

copyparty/httpconn.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
 import sys
 import time
@@ -38,6 +39,7 @@ class HttpConn(object):
         self.workload = 0
         self.u2idx = None
         self.log_func = hsrv.log
+        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
         self.set_rproxy()
 
     def set_rproxy(self, ip=None):

copyparty/up2k.py

@@ -511,6 +511,7 @@ class Up2k(object):
 
     def _run_all_mtp(self):
         t0 = time.time()
+        self.mtp_audio = {}
         self.mtp_force = {}
         self.mtp_parsers = {}
         for ptop, flags in self.flags.items():
@@ -527,8 +528,9 @@ class Up2k(object):
 
         entags = self.entags[ptop]
 
-        force = {}
-        timeout = {}
+        audio = {}  # [r]equire [n]ot [d]ontcare
+        force = {}  # bool
+        timeout = {}  # int
         parsers = {}
         for parser in self.flags[ptop]["mtp"]:
             orig = parser
@@ -536,6 +538,8 @@ class Up2k(object):
             if tag not in entags:
                 continue
 
+            audio[tag] = "y"
+
             while True:
                 try:
                     bp = os.path.expanduser(parser)
@@ -549,6 +553,10 @@ class Up2k(object):
                     arg, parser = parser.split(",", 1)
                     arg = arg.lower()
 
+                    if arg.startswith("a"):
+                        audio[tag] = arg[1:]
+                        continue
+
                     if arg == "f":
                         force[tag] = True
                         continue
@@ -563,6 +571,8 @@ class Up2k(object):
                     self.log("invalid argument: " + orig, 1)
                     return
 
+        # todo audio/force => parser attributes
+        self.mtp_audio[ptop] = audio
         self.mtp_force[ptop] = force
         self.mtp_parsers[ptop] = parsers
 
@@ -596,8 +606,8 @@ class Up2k(object):
                     have = cur.execute(q, (w,)).fetchall()
                     have = [x[0] for x in have]
 
-                    if ".dur" not in have and ".dur" in entags:
-                        # skip non-audio
+                    parsers = self._get_parsers(ptop, have)
+                    if not parsers:
                         to_delete[w] = True
                         n_left -= 1
                         continue
@@ -605,10 +615,7 @@ class Up2k(object):
                     if w in in_progress:
                         continue
 
-                    task_parsers = {
-                        k: v for k, v in parsers.items() if k in force or k not in have
-                    }
-                    jobs.append([task_parsers, None, w, abspath])
+                    jobs.append([parsers, None, w, abspath])
                     in_progress[w] = True
 
             done = self._flush_mpool(wcur)
@@ -667,6 +674,31 @@ class Up2k(object):
             wcur.close()
             cur.close()
 
+    def _get_parsers(self, ptop, have):
+        try:
+            all_parsers = self.mtp_parsers[ptop]
+        except:
+            return {}
+
+        audio = self.mtp_audio[ptop]
+        force = self.mtp_force[ptop]
+        entags = self.entags[ptop]
+        parsers = {}
+        for k, v in all_parsers.items():
+            if ".dur" in entags:
+                if ".dur" in have:
+                    # is audio, require non-audio?
+                    if audio[k] == "n":
+                        continue
+                # is not audio, require audio?
+                elif audio[k] == "y":
+                    continue
+
+            parsers[k] = v
+
+        parsers = {k: v for k, v in parsers.items() if k in force or k not in have}
+        return parsers
+
     def _start_mpool(self):
         # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
         # both do crazy runahead so lets reinvent another wheel
@@ -1198,6 +1230,9 @@ class Up2k(object):
         #    raise Exception("aaa")
 
         tnam = job["name"] + ".PARTIAL"
+        if self.args.dotpart:
+            tnam = "." + tnam
+
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f:
             f, job["tnam"] = f["orz"]
@@ -1305,13 +1340,9 @@ class Up2k(object):
             abspath = os.path.join(ptop, rd, fn)
             tags = self.mtag.get(abspath)
             ntags1 = len(tags)
-            if self.mtp_parsers.get(ptop, {}):
-                parser = {
-                    k: v
-                    for k, v in self.mtp_parsers[ptop].items()
-                    if k in self.mtp_force[ptop] or k not in tags
-                }
-                tags.update(self.mtag.get_bin(parser, abspath))
+            parsers = self._get_parsers(ptop, tags)
+            if parsers:
+                tags.update(self.mtag.get_bin(parsers, abspath))
 
             with self.mutex:
                 cur = self.cur[ptop]

copyparty/util.py

@@ -593,8 +593,8 @@ def sanitize_fn(fn, ok="", bad=[]):
             ["?", "？"],
             ["*", "＊"],
         ]
-        for bad, good in [x for x in remap if x[0] not in ok]:
-            fn = fn.replace(bad, good)
+        for a, b in [x for x in remap if x[0] not in ok]:
+            fn = fn.replace(a, b)
 
         bad.extend(["con", "prn", "aux", "nul"])
         for n in range(1, 10):

copyparty/web/browser.css

@@ -418,6 +418,7 @@ a, #files tbody div a:last-child {
 	padding: .3em .6em;
 	border-radius: .3em;
 	border-width: .15em 0;
+	white-space: nowrap;
 }
 .opbox {
 	background: #2d2d2d;

copyparty/web/browser.js

@@ -1306,7 +1306,7 @@ function despin(sel) {
 function apply_perms(perms) {
 	perms = perms || [];
 
-	var o = QSA('#ops>a[data-perm]');
+	var o = QSA('#ops>a[data-perm], #u2footfoot');
 	for (var a = 0; a < o.length; a++) {
 		var display = 'inline';
 		var needed = o[a].getAttribute('data-perm').split(' ');

copyparty/web/browser2.html

@@ -6,6 +6,11 @@
     <title>{{ title }}</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
+    <style>
+        html{font-family:sans-serif}
+        td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
+        a{display:block}
+    </style>
 </head>
 
 <body>

copyparty/web/up2k.js

@@ -18,10 +18,9 @@ function goto_up2k() {
 // usually it's undefined but some chromes throw on invoke
 var up2k = null;
 try {
-    crypto.subtle.digest(
-        'SHA-512', new Uint8Array(1)
-    ).then(
-        function (x) { up2k = up2k_init(true) },
+    var cf = crypto.subtle || crypto.webkitSubtle;
+    cf.digest('SHA-512', new Uint8Array(1)).then(
+        function (x) { up2k = up2k_init(cf) },
         function (x) { up2k = up2k_init(false) }
     );
 }
@@ -401,9 +400,7 @@ function U2pvis(act, btns) {
 }
 
 
-function up2k_init(have_crypto) {
-    //have_crypto = false;
-
+function up2k_init(subtle) {
     // show modal message
     function showmodal(msg) {
         ebi('u2notbtn').innerHTML = msg;
@@ -426,12 +423,12 @@ function up2k_init(have_crypto) {
         is_https = (window.location + '').indexOf('https:') === 0;
 
     if (is_https)
-        // chrome<37 firefox<34 edge<12 ie<11 opera<24 safari<10.1
+        // chrome<37 firefox<34 edge<12 opera<24 safari<7
         shame = 'your browser is impressively ancient';
 
     // upload ui hidden by default, clicking the header shows it
     function init_deps() {
-        if (!have_crypto && !window.asmCrypto) {
+        if (!subtle && !window.asmCrypto) {
             showmodal('<h1>loading sha512.js</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
             import_js('/.cpr/deps/sha512.js', unmodal);
 
@@ -443,8 +440,8 @@ function up2k_init(have_crypto) {
     }
 
     // show uploader if the user only has write-access
-    var perms = (document.body.getAttribute('perms') + '').split(' ');
-    if (!has(perms, 'read'))
+    var perms = document.body.getAttribute('perms');
+    if (perms && !has(perms.split(' '), 'read'))
         goto('up2k');
 
     // shows or clears a message in the basic uploader ui
@@ -986,14 +983,14 @@ function up2k_init(have_crypto) {
                 st.todo.handshake.push(t);
             };
 
-            if (have_crypto)
-                crypto.subtle.digest('SHA-512', buf).then(hash_done);
-            else {
+            if (subtle)
+                subtle.digest('SHA-512', buf).then(hash_done);
+            else setTimeout(function () {
                 var hasher = new asmCrypto.Sha512();
                 hasher.process(new Uint8Array(buf));
                 hasher.finish();
                 hash_done(hasher.result);
-            }
+            }, 1);
         };
 
         t.t1 = Date.now();
@@ -1242,6 +1239,10 @@ function up2k_init(have_crypto) {
     onresize();
 
     function desc_show(e) {
+        var cfg = sread('tooltips');
+        if (cfg !== null && cfg != '1')
+            return;
+
         var msg = this.getAttribute('alt'),
             cdesc = ebi('u2cdesc');
 
@@ -1309,20 +1310,23 @@ function up2k_init(have_crypto) {
     }
 
     function set_fsearch(new_state) {
-        var perms = (document.body.getAttribute('perms') + '').split(' '),
+        var perms = document.body.getAttribute('perms'),
             fixed = false;
 
         if (!ebi('fsearch')) {
             new_state = false;
         }
-        else if (!has(perms, 'write')) {
+        else if (perms) {
+            perms = perms.split(' ');
+            if (!has(perms, 'write')) {
                 new_state = true;
                 fixed = true;
             }
-        else if (!has(perms, 'read')) {
+            if (!has(perms, 'read')) {
                 new_state = false;
                 fixed = true;
             }
+        }
 
         if (new_state !== undefined) {
             fsearch = new_state;

copyparty/web/upload.css

@@ -97,6 +97,7 @@
 #u2cards {
 	padding: 1em 0 .3em 1em;
 	margin: 1.5em auto -2.5em auto;
+	white-space: nowrap;
 	text-align: center;
 	overflow: hidden;
 }

copyparty/web/upload.html

@@ -99,5 +99,5 @@
             </table>
 
             <p id="u2foot"></p>
-            <p id="u2footfoot">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
+            <p id="u2footfoot" data-perm="write">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
     </div>

docs/minimal-up2k.html

@@ -1,12 +1,13 @@
 <!--
-  save this as .epilogue.html inside a write-only folder to declutter the UI
+  save this as .epilogue.html inside a write-only folder to declutter the UI,  makes it look like
+  https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
 -->
 
 <style>
 
     /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
 
-    #ops, #tree, #path,  /* main tabs and navigators (tree/breadcrumbs) */
+    #ops, #tree, #path, #wrap>h2:last-child,  /* main tabs and navigators (tree/breadcrumbs) */
 
     #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
 

scripts/make-sfx.sh

@@ -163,7 +163,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
+rm -f copyparty/web/deps/*.full.* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |

scripts/speedtest-fs.py

@@ -17,14 +17,15 @@ __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
 
-def get_spd(nbyte, nsec):
+def get_spd(nbyte, nfiles, nsec):
     if not nsec:
-        return "0.000 MB   0.000 sec   0.000 MB/s"
+        return "0.000 MB   0 files   0.000 sec   0.000 MB/s   0.000 f/s"
 
     mb = nbyte / (1024 * 1024.0)
     spd = mb / nsec
+    nspd = nfiles / nsec
 
-    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+    return f"{mb:.3f} MB   {nfiles} files   {nsec:.3f} sec   {spd:.3f} MB/s   {nspd:.3f} f/s"
 
 
 class Inf(object):
@@ -36,6 +37,7 @@ class Inf(object):
         self.mtx_reports = threading.Lock()
 
         self.n_byte = 0
+        self.n_file = 0
         self.n_sec = 0
         self.n_done = 0
         self.t0 = t0
@@ -63,7 +65,8 @@ class Inf(object):
                 continue
 
             msgs = msgs[-64:]
-            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            spd = get_spd(self.n_byte, len(self.reports), self.n_sec)
+            msgs = [f"{spd}   {x}" for x in msgs]
             print("\n".join(msgs))
 
     def report(self, fn, n_byte, n_sec):
@@ -131,8 +134,9 @@ def main():
 
     num_threads = 8
     read_sz = 32 * 1024
+    targs = (q, inf, read_sz)
     for _ in range(num_threads):
-        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr = threading.Thread(target=worker, args=targs)
         thr.daemon = True
         thr.start()
 
@@ -151,14 +155,14 @@ def main():
     log = inf.reports
     log.sort()
     for nbyte, nsec, fn in log[-64:]:
-        print(f"{get_spd(nbyte, nsec)}   {fn}")
+        spd = get_spd(nbyte, len(log), nsec)
+        print(f"{spd}   {fn}")
 
     print()
     print("\n".join(inf.errors))
 
-    print(get_spd(inf.n_byte, t2 - t0))
+    print(get_spd(inf.n_byte, len(log), t2 - t0))
 
 
 if __name__ == "__main__":
     main()
-