v0.10.14

README.md

@@ -101,6 +101,11 @@ summary: it works! you can use it! (but technically not even close to beta)
 * hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
+## not my bugs
+
+* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
+  * this is an msys2 bug, the regular windows edition of python is fine
+
 
 # usage
 

copyparty/__init__.py

@@ -16,6 +16,8 @@ if platform.system() == "Windows":
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+
 MACOS = platform.system() == "Darwin"
 
 

copyparty/__main__.py

@@ -247,6 +247,7 @@ def run_argparse(argv, formatter):
     ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
     ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
+    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 10, 12)
+VERSION = (0, 10, 14)
 CODENAME = "zip it"
-BUILD_DT = (2021, 4, 19)
+BUILD_DT = (2021, 4, 21)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -13,7 +13,7 @@ import ctypes
 from datetime import datetime
 import calendar
 
-from .__init__ import E, PY2, WINDOWS
+from .__init__ import E, PY2, WINDOWS, ANYWIN
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
 from .szip import StreamZip
 from .star import StreamTar
@@ -261,12 +261,14 @@ class HttpCli(object):
 
                 self.absolute_urls = True
 
-        # go home if verboten
         self.readable, self.writable = self.conn.auth.vfs.can_access(
             self.vpath, self.uname
         )
         if not self.readable and not self.writable:
-            self.log("inaccessible: [{}]".format(self.vpath))
+            if self.vpath:
+                self.log("inaccessible: [{}]".format(self.vpath))
+                raise Pebkac(404)
+
             self.uparam = {"h": False}
 
         if "h" in self.uparam:
@@ -626,7 +628,7 @@ class HttpCli(object):
             self.loud_reply(x, status=500)
             return False
 
-        if not WINDOWS and num_left == 0:
+        if not ANYWIN and num_left == 0:
             times = (int(time.time()), int(lastmod))
             self.log("no more chunks, setting times {}".format(times))
             try:
@@ -680,7 +682,7 @@ class HttpCli(object):
                 raise Pebkac(500, "mkdir failed, check the logs")
 
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
-        esc_paths = [quotep(vpath), html_escape(vpath)]
+        esc_paths = [quotep(vpath), html_escape(vpath, crlf=True)]
         html = self.j2(
             "msg",
             h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
@@ -1181,17 +1183,16 @@ class HttpCli(object):
         template = self.j2(tpl)
 
         st = os.stat(fsenc(fs_path))
-        # sz_md = st.st_size
         ts_md = st.st_mtime
 
         st = os.stat(fsenc(html_path))
         ts_html = st.st_mtime
 
-        # TODO dont load into memory ;_;
-        #   (trivial fix, count the &'s)
-        with open(fsenc(fs_path), "rb") as f:
-            md = f.read().replace(b"&", b"&amp;")
-            sz_md = len(md)
+        sz_md = 0
+        for buf in yieldfile(fs_path):
+            sz_md += len(buf)
+            for c, v in [[b"&", 4], [b"<", 3], [b">", 3]]:
+                sz_md += (len(buf) - len(buf.replace(c, b""))) * v
 
         file_ts = max(ts_md, ts_html)
         file_lastmod, do_send = self._chk_lastmod(file_ts)
@@ -1199,27 +1200,34 @@ class HttpCli(object):
         self.out_headers["Cache-Control"] = "no-cache"
         status = 200 if do_send else 304
 
+        boundary = "\roll\tide"
         targs = {
             "edit": "edit" in self.uparam,
-            "title": html_escape(self.vpath),
+            "title": html_escape(self.vpath, crlf=True),
             "lastmod": int(ts_md * 1000),
             "md_plug": "true" if self.args.emp else "false",
             "md_chk_rate": self.args.mcr,
-            "md": "",
+            "md": boundary,
         }
-        sz_html = len(template.render(**targs).encode("utf-8"))
-        self.send_headers(sz_html + sz_md, status)
+        html = template.render(**targs).encode("utf-8")
+        html = html.split(boundary.encode("utf-8"))
+        if len(html) != 2:
+            raise Exception("boundary appears in " + html_path)
+
+        self.send_headers(sz_md + len(html[0]) + len(html[1]), status)
 
         logmsg += unicode(status)
         if self.mode == "HEAD" or not do_send:
             self.log(logmsg)
             return True
 
-        # TODO jinja2 can stream this right?
-        targs["md"] = md.decode("utf-8", "replace")
-        html = template.render(**targs).encode("utf-8")
         try:
-            self.s.sendall(html)
+            self.s.sendall(html[0])
+            for buf in yieldfile(fs_path):
+                self.s.sendall(html_bescape(buf))
+
+            self.s.sendall(html[1])
+
         except:
             self.log(logmsg + " \033[31md/c\033[0m")
             return False
@@ -1300,7 +1308,7 @@ class HttpCli(object):
                 else:
                     vpath += "/" + node
 
-                vpnodes.append([quotep(vpath) + "/", html_escape(node)])
+                vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
         vn, rem = self.auth.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
@@ -1394,7 +1402,7 @@ class HttpCli(object):
                     margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
-                    base, html_escape(hist[fn][2], quote=True), hist[fn][0]
+                    base, html_escape(hist[fn][2], quote=True, crlf=True), hist[fn][0]
                 )
             else:
                 margin = "-"
@@ -1536,7 +1544,7 @@ class HttpCli(object):
             have_b_u=(self.writable and self.uparam.get("b") == "u"),
             url_suf=url_suf,
             logues=logues,
-            title=html_escape(self.vpath),
+            title=html_escape(self.vpath, crlf=True),
             srv_info=srv_info,
         )
         self.reply(html.encode("utf-8", "replace"))

copyparty/up2k.py

@@ -16,7 +16,7 @@ import traceback
 import subprocess as sp
 from copy import deepcopy
 
-from .__init__ import WINDOWS
+from .__init__ import WINDOWS, ANYWIN
 from .util import (
     Pebkac,
     Queue,
@@ -79,7 +79,7 @@ class Up2k(object):
             if self.sqlite_ver < (3, 9):
                 self.no_expr_idx = True
 
-        if WINDOWS:
+        if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
             thr = threading.Thread(target=self._lastmodder)
@@ -668,12 +668,6 @@ class Up2k(object):
             cur.close()
 
     def _start_mpool(self):
-        if WINDOWS and False:
-            nah = open(os.devnull, "wb")
-            wmic = "processid={}".format(os.getpid())
-            wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
-            sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
-
         # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
         # both do crazy runahead so lets reinvent another wheel
         nw = os.cpu_count() if hasattr(os, "cpu_count") else 4
@@ -698,12 +692,6 @@ class Up2k(object):
 
         mpool.join()
         done = self._flush_mpool(wcur)
-        if WINDOWS and False:
-            nah = open(os.devnull, "wb")
-            wmic = "processid={}".format(os.getpid())
-            wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
-            sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
-
         return done
 
     def _tag_thr(self, q):
@@ -1110,8 +1098,9 @@ class Up2k(object):
 
             atomic_move(src, dst)
 
-            if WINDOWS:
-                self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))])
+            if ANYWIN:
+                a = [dst, job["size"], (int(time.time()), int(job["lmod"]))]
+                self.lastmod_q.put(a)
 
             # legit api sware 2 me mum
             if self.idx_wark(
@@ -1212,6 +1201,17 @@ class Up2k(object):
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f:
             f, job["tnam"] = f["orz"]
+            if (
+                ANYWIN
+                and self.args.sparse
+                and self.args.sparse * 1024 * 1024 <= job["size"]
+            ):
+                fp = os.path.join(pdir, job["tnam"])
+                try:
+                    sp.check_call(["fsutil", "sparse", "setflag", fp])
+                except:
+                    self.log("could not sparse [{}]".format(fp), 3)
+
             f.seek(job["size"] - 1)
             f.write(b"e")
 
@@ -1223,13 +1223,19 @@ class Up2k(object):
 
             # self.log("lmod: got {}".format(len(ready)))
             time.sleep(5)
-            for path, times in ready:
+            for path, sz, times in ready:
                 self.log("lmod: setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:
                     self.log("lmod: failed to utime ({}, {})".format(path, times))
 
+                if self.args.sparse and self.args.sparse * 1024 * 1024 <= sz:
+                    try:
+                        sp.check_call(["fsutil", "sparse", "setflag", path, "0"])
+                    except:
+                        self.log("could not unsparse [{}]".format(path), 3)
+
     def _snapshot(self):
         persist_interval = 30  # persist unfinished uploads index every 30 sec
         discard_interval = 21600  # drop unfinished uploads after 6 hours inactivity

copyparty/util.py

@@ -16,7 +16,7 @@ import mimetypes
 import contextlib
 import subprocess as sp  # nosec
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, ANYWIN
 from .stolen import surrogateescape
 
 FAKE_MP = False
@@ -580,8 +580,8 @@ def sanitize_fn(fn, ok=""):
     if "/" not in ok:
         fn = fn.replace("\\", "/").split("/")[-1]
 
-    if WINDOWS:
-        for bad, good in [x for x in [
+    if ANYWIN:
+        remap = [
             ["<", "＜"],
             [">", "＞"],
             [":", "："],
@@ -591,7 +591,8 @@ def sanitize_fn(fn, ok=""):
             ["|", "｜"],
             ["?", "？"],
             ["*", "＊"],
-        ] if x[0] not in ok]:
+        ]
+        for bad, good in [x for x in remap if x[0] not in ok]:
             fn = fn.replace(bad, good)
 
         bad = ["con", "prn", "aux", "nul"]
@@ -615,17 +616,24 @@ def exclude_dotfiles(filepaths):
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def html_escape(s, quote=False):
+def html_escape(s, quote=False, crlf=False):
     """html.escape but also newlines"""
-    s = (
-        s.replace("&", "&amp;")
-        .replace("<", "&lt;")
-        .replace(">", "&gt;")
-        .replace("\r", "&#13;")
-        .replace("\n", "&#10;")
-    )
+    s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
     if quote:
         s = s.replace('"', "&quot;").replace("'", "&#x27;")
+    if crlf:
+        s = s.replace("\r", "&#13;").replace("\n", "&#10;")
+
+    return s
+
+
+def html_bescape(s, quote=False, crlf=False):
+    """html.escape but bytestrings"""
+    s = s.replace(b"&", b"&amp;").replace(b"<", b"&lt;").replace(b">", b"&gt;")
+    if quote:
+        s = s.replace(b'"', b"&quot;").replace(b"'", b"&#x27;")
+    if crlf:
+        s = s.replace(b"\r", b"&#13;").replace(b"\n", b"&#10;")
 
     return s
 

copyparty/web/up2k.js

@@ -219,25 +219,28 @@ function U2pvis(act, btns) {
     };
 
     this.hashed = function (fobj) {
-        var fo = this.tab[fobj.n];
-        var nb = fo.bt * (++fo.nh / fo.cb.length);
-        var p = this.perc(nb, 0, fobj.size, fobj.t1);
+        var fo = this.tab[fobj.n],
+            nb = fo.bt * (++fo.nh / fo.cb.length),
+            p = this.perc(nb, 0, fobj.size, fobj.t1);
+
         fo.hp = '{0}%, {1}, {2} MB/s'.format(
             p[0].toFixed(2), p[1], p[2].toFixed(2)
         );
         if (!this.is_act(fo.in))
             return;
 
-        var obj = ebi('f{0}p'.format(fobj.n));
+        var obj = ebi('f{0}p'.format(fobj.n)),
+            o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
+
         obj.innerHTML = fo.hp;
         obj.style.color = '#fff';
-        var o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
         obj.style.background = 'linear-gradient(90deg, #025, #06a ' + o1 + '%, #09d ' + o2 + '%, #333 ' + o3 + '%, #333 99%, #777)';
     };
 
     this.prog = function (fobj, nchunk, cbd) {
-        var fo = this.tab[fobj.n];
-        var delta = cbd - fo.cb[nchunk];
+        var fo = this.tab[fobj.n],
+            delta = cbd - fo.cb[nchunk];
+
         fo.cb[nchunk] = cbd;
         fo.bd += delta;
 
@@ -249,10 +252,11 @@ function U2pvis(act, btns) {
         if (!this.is_act(fo.in))
             return;
 
-        var obj = ebi('f{0}p'.format(fobj.n));
+        var obj = ebi('f{0}p'.format(fobj.n)),
+            o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
+
         obj.innerHTML = fo.hp;
         obj.style.color = '#fff';
-        var o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
         obj.style.background = 'linear-gradient(90deg, #050, #270 ' + o1 + '%, #4b0 ' + o2 + '%, #333 ' + o3 + '%, #333 99%, #777)';
     };
 
@@ -287,15 +291,6 @@ function U2pvis(act, btns) {
         }
     };
 
-    this.bzw_log = function (first, last) {
-        console.log("first %d   head %d   tail %d   last %d", first, this.head, this.tail, last);
-        var trs = document.querySelectorAll('#u2tab>tbody>tr'), msg = [];
-        for (var a = 0; a < trs.length; a++)
-            msg.push(trs[a].getAttribute('id'));
-
-        console.log(msg.join(' '));
-    }
-
     this.bzw = function () {
         var first = document.querySelector('#u2tab>tbody>tr:first-child');
         if (!first)
@@ -304,7 +299,6 @@ function U2pvis(act, btns) {
         var last = document.querySelector('#u2tab>tbody>tr:last-child');
         first = parseInt(first.getAttribute('id').slice(1));
         last = parseInt(last.getAttribute('id').slice(1));
-        //this.bzw_log(first, last);
 
         while (this.head - first > this.wsz) {
             var obj = ebi('f' + (first++));
@@ -315,8 +309,6 @@ function U2pvis(act, btns) {
             if (!obj)
                 this.addrow(last);
         }
-        //this.bzw_log(first, last);
-        //console.log('--');
     };
 
     this.drawcard = function (cat) {
@@ -343,9 +335,9 @@ function U2pvis(act, btns) {
 
     this.changecard = function (card) {
         this.act = card;
-        var html = [];
         this.head = -1;
         this.tail = -1;
+        var html = [];
         for (var a = 0; a < this.tab.length; a++) {
             var rt = this.tab[a].in;
             if (this.is_act(rt)) {
@@ -411,7 +403,6 @@ function U2pvis(act, btns) {
 
 function up2k_init(have_crypto) {
     //have_crypto = false;
-    var need_filereader_cache = undefined;
 
     // show modal message
     function showmodal(msg) {
@@ -429,8 +420,9 @@ function up2k_init(have_crypto) {
         ebi('u2notbtn').innerHTML = '';
     }
 
-    var shame = 'your browser <a href="https://www.chromium.org/blink/webcrypto">disables sha512</a> unless you <a href="' + (window.location + '').replace(':', 's:') + '">use https</a>'
-    var is_https = (window.location + '').indexOf('https:') === 0;
+    var shame = 'your browser <a href="https://www.chromium.org/blink/webcrypto">disables sha512</a> unless you <a href="' + (window.location + '').replace(':', 's:') + '">use https</a>',
+        is_https = (window.location + '').indexOf('https:') === 0;
+
     if (is_https)
         // chrome<37 firefox<34 edge<12 ie<11 opera<24 safari<10.1
         shame = 'your browser is impressively ancient';
@@ -487,13 +479,14 @@ function up2k_init(have_crypto) {
         };
     }
 
-    var parallel_uploads = icfg_get('nthread');
-    var multitask = bcfg_get('multitask', true);
-    var ask_up = bcfg_get('ask_up', true);
-    var flag_en = bcfg_get('flag_en', false);
-    var fsearch = bcfg_get('fsearch', false);
+    var parallel_uploads = icfg_get('nthread'),
+        multitask = bcfg_get('multitask', true),
+        ask_up = bcfg_get('ask_up', true),
+        flag_en = bcfg_get('flag_en', false),
+        fsearch = bcfg_get('fsearch', false),
+        fdom_ctr = 0,
+        min_filebuf = 0;
 
-    var fdom_ctr = 0;
     var st = {
         "files": [],
         "todo": {
@@ -543,8 +536,9 @@ function up2k_init(have_crypto) {
         e.stopPropagation();
         e.preventDefault();
 
-        var files;
-        var is_itemlist = false;
+        var files,
+            is_itemlist = false;
+
         if (e.dataTransfer) {
             if (e.dataTransfer.items) {
                 files = e.dataTransfer.items; // DataTransferItemList
@@ -558,9 +552,10 @@ function up2k_init(have_crypto) {
             return alert('no files selected??');
 
         more_one_file();
-        var bad_files = [];
-        var good_files = [];
-        var dirs = [];
+        var bad_files = [],
+            good_files = [],
+            dirs = [];
+
         for (var a = 0; a < files.length; a++) {
             var fobj = files[a];
             if (is_itemlist) {
@@ -645,12 +640,13 @@ function up2k_init(have_crypto) {
 
     function gotallfiles(good_files, bad_files) {
         if (bad_files.length > 0) {
-            var ntot = bad_files.length + good_files.length;
-            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
+            var ntot = bad_files.length + good_files.length,
+                msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
+
             for (var a = 0, aa = Math.min(20, bad_files.length); a < aa; a++)
                 msg += '-- ' + bad_files[a] + '\n';
 
-            if (good_files.length - bad_files.length <= 1 && /(android)/i.test(navigator.userAgent))
+            if (good_files.length - bad_files.length <= 1 && ANDROID)
                 msg += '\nFirefox-Android has a bug which prevents selecting multiple files. Try selecting one file at a time. For more info, see firefox bug 1456557';
 
             alert(msg);
@@ -664,9 +660,10 @@ function up2k_init(have_crypto) {
             return;
 
         for (var a = 0; a < good_files.length; a++) {
-            var fobj = good_files[a][0];
-            var now = new Date().getTime();
-            var lmod = fobj.lastModified || now;
+            var fobj = good_files[a][0],
+                now = new Date().getTime(),
+                lmod = fobj.lastModified || now;
+
             var entry = {
                 "n": parseInt(st.files.length.toString()),
                 "t0": now,
@@ -702,7 +699,7 @@ function up2k_init(have_crypto) {
 
     function more_one_file() {
         fdom_ctr++;
-        var elm = document.createElement('div')
+        var elm = document.createElement('div');
         elm.innerHTML = '<input id="file{0}" type="file" name="file{0}[]" multiple="multiple" />'.format(fdom_ctr);
         ebi('u2form').appendChild(elm);
         ebi('file' + fdom_ctr).addEventListener('change', gotfile, false);
@@ -749,26 +746,23 @@ function up2k_init(have_crypto) {
     }
 
     var tasker = (function () {
-        var mutex = false;
-        var was_busy = false;
+        var tto = null,
+            running = false,
+            was_busy = false;
+
+        function defer() {
+            running = false;
+            clearTimeout(tto);
+            tto = setTimeout(taskerd, 100);
+        }
 
         function taskerd() {
-            if (mutex)
+            if (running)
                 return;
 
-            mutex = true;
+            clearTimeout(tto);
+            running = true;
             while (true) {
-                if (false) {
-                    ebi('srv_info').innerHTML =
-                        new Date().getTime() + ", " +
-                        st.todo.hash.length + ", " +
-                        st.todo.handshake.length + ", " +
-                        st.todo.upload.length + ", " +
-                        st.busy.hash.length + ", " +
-                        st.busy.handshake.length + ", " +
-                        st.busy.upload.length;
-                }
-
                 var is_busy = 0 !=
                     st.todo.hash.length +
                     st.todo.handshake.length +
@@ -780,21 +774,16 @@ function up2k_init(have_crypto) {
                 if (was_busy != is_busy) {
                     was_busy = is_busy;
 
-                    if (is_busy)
-                        window.addEventListener("beforeunload", warn_uploader_busy);
-                    else
-                        window.removeEventListener("beforeunload", warn_uploader_busy);
+                    window[(is_busy ? "add" : "remove") +
+                        "EventListener"]("beforeunload", warn_uploader_busy);
                 }
 
                 if (flag) {
                     if (is_busy) {
                         var now = new Date().getTime();
                         flag.take(now);
-                        if (!flag.ours) {
-                            setTimeout(taskerd, 100);
-                            mutex = false;
-                            return;
-                        }
+                        if (!flag.ours)
+                            return defer();
                     }
                     else if (flag.ours) {
                         flag.give();
@@ -836,11 +825,8 @@ function up2k_init(have_crypto) {
                     mou_ikkai = true;
                 }
 
-                if (!mou_ikkai) {
-                    setTimeout(taskerd, 100);
-                    mutex = false;
-                    return;
-                }
+                if (!mou_ikkai)
+                    return defer();
             }
         }
         taskerd();
@@ -854,47 +840,47 @@ function up2k_init(have_crypto) {
 
     // https://gist.github.com/jonleighton/958841
     function buf2b64(arrayBuffer) {
-        var base64 = '';
-        var encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
-        var bytes = new Uint8Array(arrayBuffer);
-        var byteLength = bytes.byteLength;
-        var byteRemainder = byteLength % 3;
-        var mainLength = byteLength - byteRemainder;
-        var a, b, c, d;
-        var chunk;
+        var base64 = '',
+            cset = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_',
+            src = new Uint8Array(arrayBuffer),
+            nbytes = src.byteLength,
+            byteRem = nbytes % 3,
+            mainLen = nbytes - byteRem,
+            a, b, c, d, chunk;
 
-        for (var i = 0; i < mainLength; i = i + 3) {
-            chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
+        for (var i = 0; i < mainLen; i = i + 3) {
+            chunk = (src[i] << 16) | (src[i + 1] << 8) | src[i + 2];
             // create 8*3=24bit segment then split into 6bit segments
-            a = (chunk & 16515072) >> 18; // 16515072 = (2^6 - 1) << 18
-            b = (chunk & 258048) >> 12; // 258048   = (2^6 - 1) << 12
-            c = (chunk & 4032) >> 6; // 4032     = (2^6 - 1) << 6
-            d = chunk & 63;               // 63       = 2^6 - 1
+            a = (chunk & 16515072) >> 18; // (2^6 - 1) << 18
+            b = (chunk & 258048) >> 12; // (2^6 - 1) << 12
+            c = (chunk & 4032) >> 6; // (2^6 - 1) << 6
+            d = chunk & 63; // 2^6 - 1
 
             // Convert the raw binary segments to the appropriate ASCII encoding
-            base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d];
+            base64 += cset[a] + cset[b] + cset[c] + cset[d];
         }
 
-        if (byteRemainder == 1) {
-            chunk = bytes[mainLength];
-            a = (chunk & 252) >> 2; // 252 = (2^6 - 1) << 2
-            b = (chunk & 3) << 4; // 3   = 2^2 - 1  (zero 4 LSB)
-            base64 += encodings[a] + encodings[b];//+ '==';
+        if (byteRem == 1) {
+            chunk = src[mainLen];
+            a = (chunk & 252) >> 2; // (2^6 - 1) << 2
+            b = (chunk & 3) << 4; // 2^2 - 1  (zero 4 LSB)
+            base64 += cset[a] + cset[b];//+ '==';
         }
-        else if (byteRemainder == 2) {
-            chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1];
-            a = (chunk & 64512) >> 10; // 64512 = (2^6 - 1) << 10
-            b = (chunk & 1008) >> 4; // 1008  = (2^6 - 1) << 4
-            c = (chunk & 15) << 2; // 15    = 2^4 - 1  (zero 2 LSB)
-            base64 += encodings[a] + encodings[b] + encodings[c];//+ '=';
+        else if (byteRem == 2) {
+            chunk = (src[mainLen] << 8) | src[mainLen + 1];
+            a = (chunk & 64512) >> 10; // (2^6 - 1) << 10
+            b = (chunk & 1008) >> 4; // (2^6 - 1) << 4
+            c = (chunk & 15) << 2; // 2^4 - 1  (zero 2 LSB)
+            base64 += cset[a] + cset[b] + cset[c];//+ '=';
         }
 
         return base64;
     }
 
     function get_chunksize(filesize) {
-        var chunksize = 1024 * 1024;
-        var stepsize = 512 * 1024;
+        var chunksize = 1024 * 1024,
+            stepsize = 512 * 1024;
+
         while (true) {
             for (var mul = 1; mul <= 2; mul++) {
                 var nchunks = Math.ceil(filesize / chunksize);
@@ -907,156 +893,99 @@ function up2k_init(have_crypto) {
         }
     }
 
-    function test_filereader_speed(segm_err) {
-        var f = st.todo.hash[0].fobj,
-            sz = Math.min(2, f.size),
-            reader = new FileReader(),
-            t0, ctr = 0;
-
-        var segm_next = function () {
-            var t = new Date().getTime(),
-                td = t - t0;
-
-            if (++ctr > 2) {
-                need_filereader_cache = td > 50;
-                st.busy.hash.pop();
-                return;
-            }
-            t0 = t;
-            reader.onload = segm_next;
-            reader.onerror = segm_err;
-            reader.readAsArrayBuffer(
-                bobslice.call(f, 0, sz));
-        };
-
-        segm_next();
-    }
-
-    function ensure_rendered(func) {
-        var hidden = false;
-        var keys = ['hidden', 'msHidden', 'webkitHidden'];
-        for (var a = 0; a < keys.length; a++)
-            if (typeof document[keys[a]] !== "undefined")
-                hidden = document[keys[a]];
-
-        if (hidden)
-            return func();
-
-        window.requestAnimationFrame(func);
-    }
-
     function exec_hash() {
-        if (need_filereader_cache === undefined) {
-            st.busy.hash.push(1);
-            return test_filereader_speed(segm_err);
-        }
-
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
         st.bytes.hashed += t.size;
         t.bytes_uploaded = 0;
-        t.t1 = new Date().getTime();
 
-        var nchunk = 0;
-        var chunksize = get_chunksize(t.size);
-        var nchunks = Math.ceil(t.size / chunksize);
-
-        // android-chrome has 180ms latency on FileReader calls,
-        // detect this and do 32MB at a time
-        var cache_buf = undefined,
-            cache_ofs = 0,
-            subchunks = 2;
-
-        while (subchunks * chunksize <= 32 * 1024 * 1024)
-            subchunks++;
-
-        subchunks--;
-        if (!need_filereader_cache)
-            subchunks = 1;
+        var bpend = 0,
+            nchunk = 0,
+            chunksize = get_chunksize(t.size),
+            nchunks = Math.ceil(t.size / chunksize),
+            hashtab = {};
 
         pvis.setab(t.n, nchunks);
         pvis.move(t.n, 'bz');
 
-        var reader = new FileReader();
-
         var segm_next = function () {
-            if (cache_buf) {
-                return hash_calc();
-            }
-            reader.onload = segm_load;
-            reader.onerror = segm_err;
+            if (nchunk >= nchunks || (bpend > chunksize && bpend >= min_filebuf))
+                return false;
+
+            var reader = new FileReader(),
+                nch = nchunk++,
+                car = nch * chunksize,
+                cdr = car + chunksize,
+                t0 = new Date().getTime();
 
-            var car = nchunk * chunksize;
-            var cdr = car + chunksize * subchunks;
             if (cdr >= t.size)
                 cdr = t.size;
 
+            bpend += cdr - car;
+
+            reader.onload = function (e) {
+                if (!min_filebuf && nch == 1) {
+                    min_filebuf = 1;
+                    var td = (new Date().getTime()) - t0;
+                    if (td > 50) {
+                        ebi('u2foot').innerHTML += "<p>excessive filereader latency (" + td + " ms), increasing readahead</p>";
+                        min_filebuf = 32 * 1024 * 1024;
+                    }
+                }
+                hash_calc(nch, e.target.result);
+            };
+            reader.onerror = function () {
+                alert('y o u   b r o k e    i t\nerror: ' + reader.error);
+            };
             reader.readAsArrayBuffer(
                 bobslice.call(t.fobj, car, cdr));
+
+            return true;
         };
 
-        var segm_load = function (e) {
-            cache_buf = e.target.result;
-            cache_ofs = 0;
-            hash_calc();
-        };
+        var hash_calc = function (nch, buf) {
+            while (segm_next());
 
-        var hash_calc = function () {
-            var buf = cache_buf;
-            if (chunksize >= buf.byteLength)
-                cache_buf = undefined;
-            else {
-                var ofs = cache_ofs;
-                var ofs2 = ofs + Math.min(chunksize, cache_buf.byteLength - cache_ofs);
-                cache_ofs = ofs2;
-                buf = new Uint8Array(cache_buf).subarray(ofs, ofs2);
-                if (ofs2 >= cache_buf.byteLength)
-                    cache_buf = undefined;
-            }
+            var hash_done = function (hashbuf) {
+                var hslice = new Uint8Array(hashbuf).subarray(0, 32),
+                    b64str = buf2b64(hslice).replace(/=$/, '');
 
-            var func = function () {
-                if (have_crypto)
-                    crypto.subtle.digest('SHA-512', buf).then(hash_done);
-                else {
-                    var hasher = new asmCrypto.Sha512();
-                    hasher.process(new Uint8Array(buf));
-                    hasher.finish();
-                    hash_done(hasher.result);
+                hashtab[nch] = b64str;
+                t.hash.push(nch);
+                pvis.hashed(t);
+
+                bpend -= buf.byteLength;
+                if (t.hash.length < nchunks) {
+                    return segm_next();
                 }
+                t.hash = [];
+                for (var a = 0; a < nchunks; a++) {
+                    t.hash.push(hashtab[a]);
+                }
+
+                t.t2 = new Date().getTime();
+                if (t.n == 0 && window.location.hash == '#dbg') {
+                    var spd = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
+                    alert('{0} ms, {1} MB/s\n'.format(t.t2 - t.t1, spd.toFixed(3)) + t.hash.join('\n'));
+                }
+
+                pvis.seth(t.n, 2, 'hashing done');
+                pvis.seth(t.n, 1, '📦 wait');
+                st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
+                st.todo.handshake.push(t);
             };
 
-            if (cache_buf)
-                ensure_rendered(func);
-            else
-                func();
-        };
-
-        var hash_done = function (hashbuf) {
-            var hslice = new Uint8Array(hashbuf).subarray(0, 32);
-            var b64str = buf2b64(hslice).replace(/=$/, '');
-            t.hash.push(b64str);
-
-            pvis.hashed(t);
-            if (++nchunk < nchunks) {
-                return segm_next();
+            if (have_crypto)
+                crypto.subtle.digest('SHA-512', buf).then(hash_done);
+            else {
+                var hasher = new asmCrypto.Sha512();
+                hasher.process(new Uint8Array(buf));
+                hasher.finish();
+                hash_done(hasher.result);
             }
-
-            t.t2 = new Date().getTime();
-            if (t.n == 0 && window.location.hash == '#dbg') {
-                var spd = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
-                alert('{0} ms, {1} MB/s\n'.format(t.t2 - t.t1, spd.toFixed(3)) + t.hash.join('\n'));
-            }
-
-            pvis.seth(t.n, 2, 'hashing done');
-            pvis.seth(t.n, 1, '📦 wait');
-            st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
-            st.todo.handshake.push(t);
-        };
-
-        var segm_err = function () {
-            alert('y o u   b r o k e    i t\nerror: ' + reader.error);
         };
 
+        t.t1 = new Date().getTime();
         segm_next();
     }
 
@@ -1075,8 +1004,9 @@ function up2k_init(have_crypto) {
                 var response = JSON.parse(xhr.responseText);
 
                 if (!response.name) {
-                    var msg = '';
-                    var smsg = '';
+                    var msg = '',
+                        smsg = '';
+
                     if (!response || !response.hits || !response.hits.length) {
                         msg = 'not found on server';
                         smsg = '404';
@@ -1109,10 +1039,11 @@ function up2k_init(have_crypto) {
                     pvis.seth(t.n, 0, linksplit(esc(t.purl + t.name)).join(' '));
                 }
 
-                var chunksize = get_chunksize(t.size);
-                var cdr_idx = Math.ceil(t.size / chunksize) - 1;
-                var cdr_sz = (t.size % chunksize) || chunksize;
-                var cbd = [];
+                var chunksize = get_chunksize(t.size),
+                    cdr_idx = Math.ceil(t.size / chunksize) - 1,
+                    cdr_sz = (t.size % chunksize) || chunksize,
+                    cbd = [];
+
                 for (var a = 0; a <= cdr_idx; a++) {
                     cbd.push(a == cdr_idx ? cdr_sz : chunksize);
                 }
@@ -1133,8 +1064,9 @@ function up2k_init(have_crypto) {
                 pvis.setat(t.n, cbd);
                 pvis.prog(t, 0, cbd[0]);
 
-                var done = true;
-                var msg = '&#x1f3b7;&#x1f41b;';
+                var done = true,
+                    msg = '&#x1f3b7;&#x1f41b;';
+
                 if (t.postlist.length > 0) {
                     for (var a = 0; a < t.postlist.length; a++)
                         st.todo.upload.push({
@@ -1151,10 +1083,12 @@ function up2k_init(have_crypto) {
                 if (done) {
                     t.done = true;
                     st.bytes.uploaded += t.size - t.bytes_uploaded;
-                    var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
-                    var spd2 = (t.size / ((t.t4 - t.t3) / 1000.)) / (1024 * 1024.);
+                    var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.),
+                        spd2 = (t.size / ((t.t4 - t.t3) / 1000.)) / (1024 * 1024.);
+
                     pvis.seth(t.n, 2, 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2)));
+
                     pvis.move(t.n, 'ok');
                 }
                 else t.t4 = undefined;
@@ -1221,16 +1155,18 @@ function up2k_init(have_crypto) {
         var upt = st.todo.upload.shift();
         st.busy.upload.push(upt);
 
-        var npart = upt.npart;
-        var t = st.files[upt.nfile];
+        var npart = upt.npart,
+            t = st.files[upt.nfile];
+
         if (!t.t3)
             t.t3 = new Date().getTime();
 
         pvis.seth(t.n, 1, "🚀 send");
 
-        var chunksize = get_chunksize(t.size);
-        var car = npart * chunksize;
-        var cdr = car + chunksize;
+        var chunksize = get_chunksize(t.size),
+            car = npart * chunksize,
+            cdr = car + chunksize;
+
         if (cdr >= t.size)
             cdr = t.size;
 
@@ -1295,10 +1231,10 @@ function up2k_init(have_crypto) {
     onresize();
 
     function desc_show(e) {
-        var msg = this.getAttribute('alt');
-        msg = msg.replace(/\$N/g, "<br />");
-        var cdesc = ebi('u2cdesc');
-        cdesc.innerHTML = msg;
+        var msg = this.getAttribute('alt'),
+            cdesc = ebi('u2cdesc');
+
+        cdesc.innerHTML = msg.replace(/\$N/g, "<br />");
         cdesc.setAttribute('class', 'show');
     }
     function desc_hide(e) {
@@ -1362,8 +1298,8 @@ function up2k_init(have_crypto) {
     }
 
     function set_fsearch(new_state) {
-        var perms = document.body.getAttribute('perms');
-        var read_only = false;
+        var perms = document.body.getAttribute('perms'),
+            read_only = false;
 
         if (!ebi('fsearch')) {
             new_state = false;
@@ -1384,11 +1320,11 @@ function up2k_init(have_crypto) {
         catch (ex) { }
 
         try {
-            var fun = fsearch ? 'add' : 'remove';
-            ebi('op_up2k').classList[fun]('srch');
+            var fun = fsearch ? 'add' : 'remove',
+                ico = fsearch ? '🔎' : '🚀',
+                desc = fsearch ? 'Search' : 'Upload';
 
-            var ico = fsearch ? '🔎' : '🚀';
-            var desc = fsearch ? 'Search' : 'Upload';
+            ebi('op_up2k').classList[fun]('srch');
             ebi('u2bm').innerHTML = ico + ' <sup>' + desc + '</sup>';
         }
         catch (ex) { }

copyparty/web/upload.css

@@ -90,8 +90,10 @@
 	background: #222;
 }
 #u2cards {
-	margin: 2.5em auto -2.5em auto;
+	padding: 1em 0 .3em 0;
+	margin: 1.5em auto -2.5em auto;
 	text-align: center;
+	overflow: hidden;
 }
 #u2cards.w {
 	width: 45em;
@@ -110,10 +112,15 @@
 	border-radius: 0 .4em 0 0;
 }
 #u2cards a.act {
-	border-width: 1px 1px 0 1px;
+	padding-bottom: .5em;
+	border-width: 1px 1px .1em 1px;
 	border-radius: .3em .3em 0 0;
 	margin-left: -1px;
-	background: transparent;
+	background: linear-gradient(to bottom, #464, #333 80%);
+	box-shadow: 0 -.17em .67em #280;
+	border-color: #7c5 #583 #333 #583;
+	position: relative;
+	color: #fd7;
 }
 #u2cards span {
 	color: #fff;
@@ -134,12 +141,13 @@
 	outline: none;
 }
 #u2conf .txtbox {
-	width: 4em;
+	width: 3em;
 	color: #fff;
 	background: #444;
 	border: 1px solid #777;
 	font-size: 1.2em;
 	padding: .15em 0;
+	height: 1.05em;
 }
 #u2conf .txtbox.err {
 	background: #922;
@@ -151,13 +159,12 @@
 	border-radius: .1em;
 	font-size: 1.5em;
 	padding: .1em 0;
-	margin: 0 -.25em;
+	margin: 0 -1px;
 	width: 1.5em;
 	height: 1em;
 	display: inline-block;
 	position: relative;
-	line-height: 1em;
-	bottom: -.08em;
+	bottom: -0.08em;
 }
 #u2conf input+a {
 	background: #d80;
@@ -208,12 +215,13 @@
 	text-align: center;
 	overflow: hidden;
 	margin: 0 -2em;
-	height: 0;
 	padding: 0 1em;
+	height: 0;
 	opacity: .1;
     transition: all 0.14s ease-in-out;
-	border-radius: .4em;
 	box-shadow: 0 .2em .5em #222;
+	border-radius: .4em;
+	z-index: 1;
 }
 #u2cdesc.show {
 	padding: 1em;
@@ -256,7 +264,10 @@ html.light #u2cards a {
 	background: linear-gradient(to bottom, #eee, #fff);
 }
 html.light #u2cards a.act {
+	color: #037;
 	background: inherit;
+	box-shadow: 0 -.17em .67em #0ad;
+	border-color: #09c #05a #eee #05a;
 }
 html.light #u2conf .txtbox {
 	background: #fff;

copyparty/web/upload.html

@@ -59,9 +59,9 @@
                 </tr>
                 <tr>
                     <td>
-                        <a href="#" id="nthread_sub">&ndash;</a>
-                        <input class="txtbox" id="nthread" value="2" />
-                        <a href="#" id="nthread_add">+</a>
+                        <a href="#" id="nthread_sub">&ndash;</a><input
+                            class="txtbox" id="nthread" value="2"/><a
+                            href="#" id="nthread_add">+</a>
                     </td>
                 </tr>
             </table>

copyparty/web/util.js

@@ -6,7 +6,8 @@ if (!window['console'])
     };
 
 
-var clickev = window.Touch ? 'touchstart' : 'click';
+var clickev = window.Touch ? 'touchstart' : 'click',
+    ANDROID = /(android)/i.test(navigator.userAgent);
 
 
 // error handler for mobile devices

scripts/sfx.py

@@ -26,7 +26,7 @@ CKSUM = None
 STAMP = None
 
 PY2 = sys.version_info[0] == 2
-WINDOWS = sys.platform == "win32"
+WINDOWS = sys.platform in ["win32", "msys"]
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
 cpp = None