v0.10.20

README.md

@@ -9,7 +9,8 @@
 turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
 
 * server runs on anything with `py2.7` or `py3.3+`
-* *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
+* browse/upload with IE4 / netscape4.0 on win3.11 (heh)
+* *resumable* uploads need `firefox 34+` / `chrome 37+` / `safari 7+`
 * code standard: `black`
 
 📷 screenshots: [browser](#the-browser) // [upload](#uploading) // [md-viewer](#markdown-viewer) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [ie4](#browser-support)
@@ -190,6 +191,8 @@ see [up2k](#up2k) for details on how it works
 
 ![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/115978061-680b5400-a57d-11eb-9ef6-cbb5f60aeccc.png)
 
+**protip:** you can avoid scaring away users with [docs/minimal-up2k.html](docs/minimal-up2k.html) which makes it look [much simpler](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
 the up2k UI is the epitome of polished inutitive experiences:
 * "parallel uploads" specifies how many chunks to upload at the same time
 * `[🏃]` analysis of other files should continue while one is uploading
@@ -205,8 +208,6 @@ and then theres the tabs below it,
   * plus up to 3 entries each from `[done]` and `[que]` for context
 * `[que]` is all the files that are still queued
 
-protip: you can avoid scaring away users by hiding some of the UI with hacks like [docs/minimal-up2k.html](docs/minimal-up2k.html)
-
 ### file-search
 
 ![copyparty-fsearch-fs8](https://user-images.githubusercontent.com/241032/116008320-36919780-a614-11eb-803f-04162326a700.png)
@@ -298,6 +299,10 @@ copyparty can invoke external programs to collect additional metadata for files
 * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
 * `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
 
+*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` audio files are skipped, or `ad` always do it (d as in dontcare) 
+
+* `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`)
+
 
 ## complete examples
 

bin/mtag/file-ext.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+
+"""
+example that just prints the file extension
+"""
+
+print(sys.argv[1].split(".")[-1])

copyparty/__main__.py

@@ -237,7 +237,6 @@ def run_argparse(argv, formatter):
     ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
     ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
     ap.add_argument("-q", action="store_true", help="quiet")
-    ap.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
@@ -246,8 +245,6 @@ def run_argparse(argv, formatter):
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
     ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
-    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
     ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
@@ -275,6 +272,13 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
 
+    ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
+    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/", help="dont log URLs matching")
+    
     return ap.parse_args(args=argv[1:])
     # fmt: on
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 10, 19)
+VERSION = (0, 10, 20)
 CODENAME = "zip it"
-BUILD_DT = (2021, 5, 14)
+BUILD_DT = (2021, 5, 16)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -100,6 +100,16 @@ class HttpCli(object):
             self.ip = v.split(",")[0]
             self.log_src = self.conn.set_rproxy(self.ip)
 
+        if self.args.ihead:
+            keys = self.args.ihead
+            if "*" in keys:
+                keys = list(sorted(self.headers.keys()))
+
+            for k in keys:
+                v = self.headers.get(k)
+                if v is not None:
+                    self.log("[H] {}: \033[33m[{}]".format(k, v), 6)
+
         # split req into vpath + uparam
         uparam = {}
         if "?" not in self.req:
@@ -148,6 +158,8 @@ class HttpCli(object):
             uparam["b"] = False
             cookies["b"] = False
 
+        self.do_log = not self.conn.lf_url or not self.conn.lf_url.match(self.req)
+
         try:
             if self.mode in ["GET", "HEAD"]:
                 return self.handle_get() and self.keepalive
@@ -237,14 +249,16 @@ class HttpCli(object):
         r = ["{}={}".format(k, quotep(v)) if v else k for k, v in kv.items()]
         return "?" + "&".join(r)
 
-    def redirect(self, vpath, suf="", msg="aight", flavor="go to", use302=False):
+    def redirect(
+        self, vpath, suf="", msg="aight", flavor="go to", click=True, use302=False
+    ):
         html = self.j2(
             "msg",
             h2='<a href="/{}">{} /{}</a>'.format(
                 quotep(vpath) + suf, flavor, html_escape(vpath, crlf=True) + suf
             ),
             pre=msg,
-            click=True,
+            click=click,
         ).encode("utf-8", "replace")
 
         if use302:
@@ -254,17 +268,18 @@ class HttpCli(object):
             self.reply(html)
 
     def handle_get(self):
-        logmsg = "{:4} {}".format(self.mode, self.req)
+        if self.do_log:
+            logmsg = "{:4} {}".format(self.mode, self.req)
 
-        if "range" in self.headers:
+            if "range" in self.headers:
-            try:
+                try:
-                rval = self.headers["range"].split("=", 1)[1]
+                    rval = self.headers["range"].split("=", 1)[1]
-            except:
+                except:
-                rval = self.headers["range"]
+                    rval = self.headers["range"]
 
-            logmsg += " [\033[36m" + rval + "\033[0m]"
+                logmsg += " [\033[36m" + rval + "\033[0m]"
 
-        self.log(logmsg)
+            self.log(logmsg)
 
         # "embedded" resources
         if self.vpath.startswith(".cpr"):
@@ -305,7 +320,9 @@ class HttpCli(object):
         return self.tx_browser()
 
     def handle_options(self):
-        self.log("OPTIONS " + self.req)
+        if self.do_log:
+            self.log("OPTIONS " + self.req)
+
         self.send_headers(
             None,
             204,
@@ -780,7 +797,7 @@ class HttpCli(object):
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
 
-                        files.append([sz, sha512_hex])
+                        files.append([sz, sha512_hex, p_file, fname])
                         self.conn.hsrv.broker.put(
                             False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
                         )
@@ -815,10 +832,13 @@ class HttpCli(object):
             errmsg = "ERROR: " + errmsg
             status = "ERROR"
 
-        msg = "{0} // {1} bytes // {2:.3f} MiB/s\n".format(status, sz_total, spd)
+        msg = "{} // {} bytes // {:.3f} MiB/s\n".format(status, sz_total, spd)
 
-        for sz, sha512 in files:
+        for sz, sha512, ofn, lfn in files:
-            msg += "sha512: {0} // {1} bytes\n".format(sha512[:56], sz)
+            vpath = self.vpath + "/" + lfn
+            msg += 'sha512: {} // {} bytes // <a href="/{}">{}</a>\n'.format(
+                sha512[:56], sz, quotep(vpath), html_escape(ofn, crlf=True)
+            )
             # truncated SHA-512 prevents length extension attacks;
             # using SHA-512/224, optionally SHA-512/256 = :64
 
@@ -826,25 +846,13 @@ class HttpCli(object):
         self.log("{} {}".format(vspd, msg))
 
         if not nullwrite:
-            # TODO this is bad
             log_fn = "up.{:.6f}.txt".format(t0)
             with open(log_fn, "wb") as f:
-                f.write(
+                ft = "{}:{}".format(self.ip, self.addr[1])
-                    (
+                ft = "{}\n{}\n{}\n".format(ft, msg.rstrip(), errmsg)
-                        "\n".join(
+                f.write(ft.encode("utf-8"))
-                            unicode(x)
-                            for x in [
-                                ":".join(unicode(x) for x in [self.ip, self.addr[1]]),
-                                msg.rstrip(),
-                            ]
-                        )
-                        + "\n"
-                        + errmsg
-                        + "\n"
-                    ).encode("utf-8")
-                )
 
-        self.redirect(self.vpath, msg=msg, flavor="return to")
+        self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
         self.parser.drop()
         return True
 
@@ -1110,7 +1118,9 @@ class HttpCli(object):
         logmsg += unicode(status) + logtail
 
         if self.mode == "HEAD" or not do_send:
-            self.log(logmsg)
+            if self.do_log:
+                self.log(logmsg)
+
             return True
 
         ret = True
@@ -1124,7 +1134,9 @@ class HttpCli(object):
             logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
 
         spd = self._spd((upper - lower) - remains)
-        self.log("{},  {}".format(logmsg, spd))
+        if self.do_log:
+            self.log("{},  {}".format(logmsg, spd))
+
         return ret
 
     def tx_zip(self, fmt, uarg, vn, rem, items, dots):
@@ -1233,7 +1245,9 @@ class HttpCli(object):
 
         logmsg += unicode(status)
         if self.mode == "HEAD" or not do_send:
-            self.log(logmsg)
+            if self.do_log:
+                self.log(logmsg)
+
             return True
 
         try:
@@ -1247,7 +1261,9 @@ class HttpCli(object):
             self.log(logmsg + " \033[31md/c\033[0m")
             return False
 
-        self.log(logmsg + " " + unicode(len(html)))
+        if self.do_log:
+            self.log(logmsg + " " + unicode(len(html)))
+
         return True
 
     def tx_mounts(self):

copyparty/httpconn.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import os
 import sys
 import time
@@ -38,6 +39,7 @@ class HttpConn(object):
         self.workload = 0
         self.u2idx = None
         self.log_func = hsrv.log
+        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
         self.set_rproxy()
 
     def set_rproxy(self, ip=None):

copyparty/up2k.py

@@ -511,6 +511,7 @@ class Up2k(object):
 
     def _run_all_mtp(self):
         t0 = time.time()
+        self.mtp_audio = {}
         self.mtp_force = {}
         self.mtp_parsers = {}
         for ptop, flags in self.flags.items():
@@ -527,8 +528,9 @@ class Up2k(object):
 
         entags = self.entags[ptop]
 
-        force = {}
+        audio = {}  # [r]equire [n]ot [d]ontcare
-        timeout = {}
+        force = {}  # bool
+        timeout = {}  # int
         parsers = {}
         for parser in self.flags[ptop]["mtp"]:
             orig = parser
@@ -536,6 +538,8 @@ class Up2k(object):
             if tag not in entags:
                 continue
 
+            audio[tag] = "y"
+
             while True:
                 try:
                     bp = os.path.expanduser(parser)
@@ -549,6 +553,10 @@ class Up2k(object):
                     arg, parser = parser.split(",", 1)
                     arg = arg.lower()
 
+                    if arg.startswith("a"):
+                        audio[tag] = arg[1:]
+                        continue
+
                     if arg == "f":
                         force[tag] = True
                         continue
@@ -563,6 +571,8 @@ class Up2k(object):
                     self.log("invalid argument: " + orig, 1)
                     return
 
+        # todo audio/force => parser attributes
+        self.mtp_audio[ptop] = audio
         self.mtp_force[ptop] = force
         self.mtp_parsers[ptop] = parsers
 
@@ -596,8 +606,8 @@ class Up2k(object):
                     have = cur.execute(q, (w,)).fetchall()
                     have = [x[0] for x in have]
 
-                    if ".dur" not in have and ".dur" in entags:
+                    parsers = self._get_parsers(ptop, have)
-                        # skip non-audio
+                    if not parsers:
                         to_delete[w] = True
                         n_left -= 1
                         continue
@@ -605,10 +615,7 @@ class Up2k(object):
                     if w in in_progress:
                         continue
 
-                    task_parsers = {
+                    jobs.append([parsers, None, w, abspath])
-                        k: v for k, v in parsers.items() if k in force or k not in have
-                    }
-                    jobs.append([task_parsers, None, w, abspath])
                     in_progress[w] = True
 
             done = self._flush_mpool(wcur)
@@ -667,6 +674,26 @@ class Up2k(object):
             wcur.close()
             cur.close()
 
+    def _get_parsers(self, ptop, have):
+        audio = self.mtp_audio[ptop]
+        force = self.mtp_force[ptop]
+        entags = self.entags[ptop]
+        parsers = {}
+        for k, v in self.mtp_parsers[ptop].items():
+            if ".dur" in entags:
+                if ".dur" in have:
+                    # is audio, require non-audio?
+                    if audio[k] == "n":
+                        continue
+                # is not audio, require audio?
+                elif audio[k] == "y":
+                    continue
+
+            parsers[k] = v
+
+        parsers = {k: v for k, v in parsers.items() if k in force or k not in have}
+        return parsers
+
     def _start_mpool(self):
         # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
         # both do crazy runahead so lets reinvent another wheel
@@ -1308,13 +1335,9 @@ class Up2k(object):
             abspath = os.path.join(ptop, rd, fn)
             tags = self.mtag.get(abspath)
             ntags1 = len(tags)
-            if self.mtp_parsers.get(ptop, {}):
+            parsers = self._get_parsers(ptop, tags)
-                parser = {
+            if parsers:
-                    k: v
+                tags.update(self.mtag.get_bin(parsers, abspath))
-                    for k, v in self.mtp_parsers[ptop].items()
-                    if k in self.mtp_force[ptop] or k not in tags
-                }
-                tags.update(self.mtag.get_bin(parser, abspath))
 
             with self.mutex:
                 cur = self.cur[ptop]

copyparty/web/browser.css

@@ -418,6 +418,7 @@ a, #files tbody div a:last-child {
 	padding: .3em .6em;
 	border-radius: .3em;
 	border-width: .15em 0;
+	white-space: nowrap;
 }
 .opbox {
 	background: #2d2d2d;

copyparty/web/browser.js

@@ -1306,7 +1306,7 @@ function despin(sel) {
 function apply_perms(perms) {
 	perms = perms || [];
 
-	var o = QSA('#ops>a[data-perm]');
+	var o = QSA('#ops>a[data-perm], #u2footfoot');
 	for (var a = 0; a < o.length; a++) {
 		var display = 'inline';
 		var needed = o[a].getAttribute('data-perm').split(' ');

copyparty/web/up2k.js

@@ -18,10 +18,9 @@ function goto_up2k() {
 // usually it's undefined but some chromes throw on invoke
 var up2k = null;
 try {
-    crypto.subtle.digest(
+    var cf = crypto.subtle || crypto.webkitSubtle;
-        'SHA-512', new Uint8Array(1)
+    cf.digest('SHA-512', new Uint8Array(1)).then(
-    ).then(
+        function (x) { up2k = up2k_init(cf) },
-        function (x) { up2k = up2k_init(true) },
         function (x) { up2k = up2k_init(false) }
     );
 }
@@ -401,9 +400,7 @@ function U2pvis(act, btns) {
 }
 
 
-function up2k_init(have_crypto) {
+function up2k_init(subtle) {
-    //have_crypto = false;
-
     // show modal message
     function showmodal(msg) {
         ebi('u2notbtn').innerHTML = msg;
@@ -426,12 +423,12 @@ function up2k_init(have_crypto) {
         is_https = (window.location + '').indexOf('https:') === 0;
 
     if (is_https)
-        // chrome<37 firefox<34 edge<12 ie<11 opera<24 safari<10.1
+        // chrome<37 firefox<34 edge<12 opera<24 safari<7
         shame = 'your browser is impressively ancient';
 
     // upload ui hidden by default, clicking the header shows it
     function init_deps() {
-        if (!have_crypto && !window.asmCrypto) {
+        if (!subtle && !window.asmCrypto) {
             showmodal('<h1>loading sha512.js</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
             import_js('/.cpr/deps/sha512.js', unmodal);
 
@@ -443,8 +440,8 @@ function up2k_init(have_crypto) {
     }
 
     // show uploader if the user only has write-access
-    var perms = (document.body.getAttribute('perms') + '').split(' ');
+    var perms = document.body.getAttribute('perms');
-    if (!has(perms, 'read'))
+    if (perms && !has(perms.split(' '), 'read'))
         goto('up2k');
 
     // shows or clears a message in the basic uploader ui
@@ -986,14 +983,14 @@ function up2k_init(have_crypto) {
                 st.todo.handshake.push(t);
             };
 
-            if (have_crypto)
+            if (subtle)
-                crypto.subtle.digest('SHA-512', buf).then(hash_done);
+                subtle.digest('SHA-512', buf).then(hash_done);
-            else {
+            else setTimeout(function () {
                 var hasher = new asmCrypto.Sha512();
                 hasher.process(new Uint8Array(buf));
                 hasher.finish();
                 hash_done(hasher.result);
-            }
+            }, 1);
         };
 
         t.t1 = Date.now();
@@ -1242,6 +1239,10 @@ function up2k_init(have_crypto) {
     onresize();
 
     function desc_show(e) {
+        var cfg = sread('tooltips');
+        if (cfg !== null && cfg != '1')
+            return;
+
         var msg = this.getAttribute('alt'),
             cdesc = ebi('u2cdesc');
 
@@ -1309,19 +1310,22 @@ function up2k_init(have_crypto) {
     }
 
     function set_fsearch(new_state) {
-        var perms = (document.body.getAttribute('perms') + '').split(' '),
+        var perms = document.body.getAttribute('perms'),
             fixed = false;
 
         if (!ebi('fsearch')) {
             new_state = false;
         }
-        else if (!has(perms, 'write')) {
+        else if (perms) {
-            new_state = true;
+            perms = perms.split(' ');
-            fixed = true;
+            if (!has(perms, 'write')) {
-        }
+                new_state = true;
-        else if (!has(perms, 'read')) {
+                fixed = true;
-            new_state = false;
+            }
-            fixed = true;
+            if (!has(perms, 'read')) {
+                new_state = false;
+                fixed = true;
+            }
         }
 
         if (new_state !== undefined) {

copyparty/web/upload.css

@@ -97,6 +97,7 @@
 #u2cards {
 	padding: 1em 0 .3em 1em;
 	margin: 1.5em auto -2.5em auto;
+	white-space: nowrap;
 	text-align: center;
 	overflow: hidden;
 }

copyparty/web/upload.html

@@ -99,5 +99,5 @@
             </table>
 
             <p id="u2foot"></p>
-            <p id="u2footfoot">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
+            <p id="u2footfoot" data-perm="write">( you can use the <a href="#" id="u2nope">basic uploader</a> if you don't need lastmod timestamps, resumable uploads, or progress bars )</p>
     </div>

docs/minimal-up2k.html

@@ -1,12 +1,13 @@
 <!--
-  save this as .epilogue.html inside a write-only folder to declutter the UI
+  save this as .epilogue.html inside a write-only folder to declutter the UI,  makes it look like
+  https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
 -->
 
 <style>
 
     /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
 
-    #ops, #tree, #path,  /* main tabs and navigators (tree/breadcrumbs) */
+    #ops, #tree, #path, #wrap>h2:last-child  /* main tabs and navigators (tree/breadcrumbs) */
 
     #u2cleanup, #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
 

scripts/make-sfx.sh

@@ -163,7 +163,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
+rm -f copyparty/web/deps/*.full.* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |