v0.6.0

* show per-connection and per-transfer speeds
* support multiple cookies in parser
* set SameSite=Lax
* restore macos support in sfx.sh
* md-editor: add mojibake/unicode hunter
* md-editor: add table formatter
* md-editor: make bold bolder
* md-editor: more hotkeys
* md-editor: fix saving in fancy
* md-editor: fix eof-scrolling in chrome
* md-editor: fix text erasure with newline
* md-editor: fix backspace behavior in gutter

.eslintrc.json

@@ -0,0 +1,12 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 12
+    },
+    "rules": {
+    }
+}

.vscode/launch.json

@@ -10,6 +10,8 @@
             "cwd": "${workspaceFolder}",
             "args": [
                 //"-nw",
+                "-ed",
+                "-emp",
                 "-a",
                 "ed:wark",
                 "-v",

README.md

@@ -38,7 +38,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * [x] accounts
 * [x] markdown viewer
 * [x] markdown editor
-* [x] FUSE client
+* [x] FUSE client (read-only)
 
 summary: it works! you can use it! (but technically not even close to beta)
 
@@ -49,7 +49,9 @@ summary: it works! you can use it! (but technically not even close to beta)
   * `await fetch('https://127.0.0.1:3923/', {method:"PUT", body: JSON.stringify(foo)});`
   * `var xhr = new XMLHttpRequest(); xhr.open('POST', 'https://127.0.0.1:3923/msgs?raw'); xhr.send('foo');`
 
-* FUSE: mount a copyparty server as a local filesystem (see [./bin/](bin/))
+* FUSE: mount a copyparty server as a local filesystem
+  * cross-platform python client available in [./bin/](bin/)
+  * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
 
 
 # dependencies
@@ -85,16 +87,18 @@ the features you can opt to drop are
 
 for the `re`pack to work, first run one of the sfx'es once to unpack it
 
+**note:** you can also just download and run [scripts/copyparty-repack.sh](scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a `no-ogv no-cm` repack; works on linux/macos (and windows with msys2 or WSL)
+
 
 # install on android
 
 install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
 ```sh
-apt update && apt -y full-upgrade && termux-setup-storage && apt -y install curl && cd && curl -L https://github.com/9001/copyparty/raw/master/scripts/copyparty-android.sh > copyparty-android.sh && chmod 755 copyparty-android.sh && ./copyparty-android.sh -h
+apt update && apt -y full-upgrade && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install -U copyparty
 echo $?
 ```
 
-after the initial setup (and restarting bash), you can launch copyparty at any time by running "copyparty" in Termux
+after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
 
 
 # dev env setup

bin/README.md

@@ -10,6 +10,8 @@ filecache is default-on for windows and macos;
 
 note that copyparty should run with `-ed` to enable dotfiles (hidden otherwise)
 
+also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x performance
+
 
 ## to run this on windows:
 * install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)

bin/copyparty-fuse.py

@@ -19,6 +19,9 @@ dependencies:
   + on Linux: sudo apk add fuse
   + on Macos: https://osxfuse.github.io/
   + on Windows: https://github.com/billziss-gh/winfsp/releases/latest
+
+get server cert:
+  awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
 """
 
 
@@ -29,21 +32,21 @@ import time
 import stat
 import errno
 import struct
+import codecs
 import builtins
 import platform
+import argparse
 import threading
 import traceback
 import http.client  # py2: httplib
 import urllib.parse
 from datetime import datetime
 from urllib.parse import quote_from_bytes as quote
-
+from urllib.parse import unquote_to_bytes as unquote
-
-DEBUG = False  # ctrl-f this to configure logging
-
 
 WINDOWS = sys.platform == "win32"
 MACOS = platform.system() == "Darwin"
+info = log = dbg = None
 
 
 try:
@@ -104,6 +107,47 @@ def null_log(msg):
     pass
 
 
+def hexler(binary):
+    return binary.replace("\r", "\\r").replace("\n", "\\n")
+    return " ".join(["{}\033[36m{:02x}\033[0m".format(b, ord(b)) for b in binary])
+    return " ".join(map(lambda b: format(ord(b), "02x"), binary))
+
+
+def register_wtf8():
+    def wtf8_enc(text):
+        return str(text).encode("utf-8", "surrogateescape"), len(text)
+
+    def wtf8_dec(binary):
+        return bytes(binary).decode("utf-8", "surrogateescape"), len(binary)
+
+    def wtf8_search(encoding_name):
+        return codecs.CodecInfo(wtf8_enc, wtf8_dec, name="wtf-8")
+
+    codecs.register(wtf8_search)
+
+
+bad_good = {}
+good_bad = {}
+
+
+def enwin(txt):
+    return "".join([bad_good.get(x, x) for x in txt])
+
+    for bad, good in bad_good.items():
+        txt = txt.replace(bad, good)
+
+    return txt
+
+
+def dewin(txt):
+    return "".join([good_bad.get(x, x) for x in txt])
+
+    for bad, good in bad_good.items():
+        txt = txt.replace(good, bad)
+
+    return txt
+
+
 class RecentLog(object):
     def __init__(self):
         self.mtx = threading.Lock()
@@ -138,22 +182,6 @@ class RecentLog(object):
             print("".join(q), end="")
 
 
-if DEBUG:
-    # debug=on,
-    #   windows terminals are slow (cmd.exe, mintty)
-    #   otoh fancy_log beats RecentLog on linux
-    logger = RecentLog().put if WINDOWS else fancy_log
-
-    info = logger
-    log = logger
-    dbg = logger
-else:
-    # debug=off, speed is dontcare
-    info = fancy_log
-    log = null_log
-    dbg = null_log
-
-
 # [windows/cmd/cpy3]  python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
 # [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
 # [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
@@ -183,6 +211,8 @@ def html_dec(txt):
         txt.replace("&lt;", "<")
         .replace("&gt;", ">")
         .replace("&quot;", '"')
+        .replace("&#13;", "\r")
+        .replace("&#10;", "\n")
         .replace("&amp;", "&")
     )
 
@@ -195,10 +225,11 @@ class CacheNode(object):
 
 
 class Gateway(object):
-    def __init__(self, base_url):
+    def __init__(self, ar):
-        self.base_url = base_url
+        self.base_url = ar.base_url
+        self.password = ar.a
 
-        ui = urllib.parse.urlparse(base_url)
+        ui = urllib.parse.urlparse(self.base_url)
         self.web_root = ui.path.strip("/")
         try:
             self.web_host, self.web_port = ui.netloc.split(":")
@@ -208,15 +239,25 @@ class Gateway(object):
             if ui.scheme == "http":
                 self.web_port = 80
             elif ui.scheme == "https":
-                raise Exception("todo")
+                self.web_port = 443
             else:
                 raise Exception("bad url?")
 
+        self.ssl_context = None
+        self.use_tls = ui.scheme.lower() == "https"
+        if self.use_tls:
+            import ssl
+
+            if ar.td:
+                self.ssl_context = ssl._create_unverified_context()
+            elif ar.te:
+                self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS)
+                self.ssl_context.load_verify_locations(ar.te)
+
         self.conns = {}
 
     def quotep(self, path):
-        # TODO: mojibake support
+        path = path.encode("wtf-8")
-        path = path.encode("utf-8", "ignore")
         return quote(path, safe="/")
 
     def getconn(self, tid=None):
@@ -226,7 +267,15 @@ class Gateway(object):
         except:
             info("new conn [{}] [{}]".format(self.web_host, self.web_port))
 
-            conn = http.client.HTTPConnection(self.web_host, self.web_port, timeout=260)
+            args = {}
+            if not self.use_tls:
+                C = http.client.HTTPConnection
+            else:
+                C = http.client.HTTPSConnection
+                if self.ssl_context:
+                    args = {"context": self.ssl_context}
+
+            conn = C(self.web_host, self.web_port, timeout=260, **args)
 
             self.conns[tid] = conn
             return conn
@@ -239,41 +288,67 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, **kwargs):
+    def sendreq(self, *args, headers={}, **kwargs):
         tid = get_tid()
+        if self.password:
+            headers["Cookie"] = "=".join(["cppwd", self.password])
+
         try:
             c = self.getconn(tid)
-            c.request(*list(args), **kwargs)
+            c.request(*list(args), headers=headers, **kwargs)
             return c.getresponse()
         except:
+            dbg("bad conn")
+
         self.closeconn(tid)
+        try:
             c = self.getconn(tid)
-            c.request(*list(args), **kwargs)
+            c.request(*list(args), headers=headers, **kwargs)
             return c.getresponse()
+        except:
+            info("http connection failed:\n" + traceback.format_exc())
+            if self.use_tls and not self.ssl_context:
+                import ssl
+
+                cert = ssl.get_server_certificate((self.web_host, self.web_port))
+                info("server certificate probably not trusted:\n" + cert)
+
+            raise
 
     def listdir(self, path):
+        if bad_good:
+            path = dewin(path)
+
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
         r = self.sendreq("GET", web_path)
         if r.status != 200:
             self.closeconn()
-            raise Exception(
+            log(
                 "http error {} reading dir {} in {}".format(
                     r.status, web_path, rice_tid()
                 )
             )
+            raise FuseOSError(errno.ENOENT)
+
+        if not r.getheader("Content-Type", "").startswith("text/html"):
+            log("listdir on file: {}".format(path))
+            raise FuseOSError(errno.ENOENT)
 
         try:
             return self.parse_html(r)
         except:
-            traceback.print_exc()
+            info(repr(path) + "\n" + traceback.format_exc())
             raise
 
     def download_file_range(self, path, ofs1, ofs2):
+        if bad_good:
+            path = dewin(path)
+
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
         hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
         info(
             "DL {:4.0f}K\033[36m{:>9}-{:<9}\033[0m{}".format(
-                (ofs2 - ofs1) / 1024.0, ofs1, ofs2 - 1, path
+                (ofs2 - ofs1) / 1024.0, ofs1, ofs2 - 1, hexler(path)
             )
         )
 
@@ -292,7 +367,7 @@ class Gateway(object):
         ret = []
         remainder = b""
         ptn = re.compile(
-            r"^<tr><td>(-|DIR)</td><td><a [^>]+>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$"
+            r'^<tr><td>(-|DIR)</td><td><a[^>]* href="([^"]+)"[^>]*>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$'
         )
 
         while True:
@@ -314,8 +389,13 @@ class Gateway(object):
                     # print(line)
                     continue
 
-                ftype, fname, fsize, fdate = m.groups()
+                ftype, furl, fname, fsize, fdate = m.groups()
-                fname = html_dec(fname)
+                fname = furl.rstrip("/").split("/")[-1]
+                fname = unquote(fname)
+                fname = fname.decode("wtf-8")
+                if bad_good:
+                    fname = enwin(fname)
+
                 sz = 1
                 ts = 60 * 60 * 24 * 2
                 try:
@@ -358,11 +438,11 @@ class Gateway(object):
 
 
 class CPPF(Operations):
-    def __init__(self, base_url, dircache, filecache):
+    def __init__(self, ar):
-        self.gw = Gateway(base_url)
+        self.gw = Gateway(ar)
         self.junk_fh_ctr = 3
-        self.n_dircache = dircache
+        self.n_dircache = ar.cd
-        self.n_filecache = filecache
+        self.n_filecache = ar.cf
 
         self.dircache = []
         self.dircache_mtx = threading.Lock()
@@ -379,7 +459,11 @@ class CPPF(Operations):
                 cache_path, cache1 = cn.tag
                 cache2 = cache1 + len(cn.data)
                 msg += "\n{:<2} {:>7} {:>10}:{:<9} {}".format(
-                    n, len(cn.data), cache1, cache2, cache_path
+                    n,
+                    len(cn.data),
+                    cache1,
+                    cache2,
+                    cache_path.replace("\r", "\\r").replace("\n", "\\n"),
                 )
         return msg
 
@@ -610,7 +694,7 @@ class CPPF(Operations):
 
     def _readdir(self, path, fh=None):
         path = path.strip("/")
-        log("readdir [{}] [{}]".format(path, fh))
+        log("readdir [{}] [{}]".format(hexler(path), fh))
 
         ret = self.gw.listdir(path)
         if not self.n_dircache:
@@ -637,7 +721,11 @@ class CPPF(Operations):
         path = path.strip("/")
         ofs2 = offset + length
         file_sz = self.getattr(path)["st_size"]
-        log("read {} |{}| {}:{} max {}".format(path, length, offset, ofs2, file_sz))
+        log(
+            "read {} |{}| {}:{} max {}".format(
+                hexler(path), length, offset, ofs2, file_sz
+            )
+        )
         if ofs2 > file_sz:
             ofs2 = file_sz
             log("truncate to |{}| :{}".format(ofs2 - offset, ofs2))
@@ -676,7 +764,9 @@ class CPPF(Operations):
         return ret
 
     def getattr(self, path, fh=None):
-        log("getattr [{}]".format(path))
+        log("getattr [{}]".format(hexler(path)))
+        if WINDOWS:
+            path = enwin(path)  # windows occasionally decodes f0xx to xx
 
         path = path.strip("/")
         try:
@@ -699,11 +789,20 @@ class CPPF(Operations):
             dents = self._readdir(dirpath)
 
         for cache_name, cache_stat, _ in dents:
+            # if "qw" in cache_name and "qw" in fname:
+            #     info(
+            #         "cmp\n  [{}]\n  [{}]\n\n{}\n".format(
+            #             hexler(cache_name),
+            #             hexler(fname),
+            #             "\n".join(traceback.format_stack()[:-1]),
+            #         )
+            #     )
+
             if cache_name == fname:
                 # dbg("=" + repr(cache_stat))
                 return cache_stat
 
-        info("=ENOENT ({})".format(path))
+        info("=ENOENT ({})".format(hexler(path)))
         raise FuseOSError(errno.ENOENT)
 
     access = None
@@ -773,24 +872,24 @@ class CPPF(Operations):
                 raise FuseOSError(errno.ENOENT)
 
         def open(self, path, flags):
-            dbg("open [{}] [{}]".format(path, flags))
+            dbg("open [{}] [{}]".format(hexler(path), flags))
             return self._open(path)
 
         def opendir(self, path):
-            dbg("opendir [{}]".format(path))
+            dbg("opendir [{}]".format(hexler(path)))
             return self._open(path)
 
         def flush(self, path, fh):
-            dbg("flush [{}] [{}]".format(path, fh))
+            dbg("flush [{}] [{}]".format(hexler(path), fh))
 
         def release(self, ino, fi):
-            dbg("release [{}] [{}]".format(ino, fi))
+            dbg("release [{}] [{}]".format(hexler(ino), fi))
 
         def releasedir(self, ino, fi):
-            dbg("releasedir [{}] [{}]".format(ino, fi))
+            dbg("releasedir [{}] [{}]".format(hexler(ino), fi))
 
         def access(self, path, mode):
-            dbg("access [{}] [{}]".format(path, mode))
+            dbg("access [{}] [{}]".format(hexler(path), mode))
             try:
                 x = self.getattr(path)
                 if x["st_mode"] <= 0:
@@ -799,42 +898,84 @@ class CPPF(Operations):
                 raise FuseOSError(errno.ENOENT)
 
 
+class TheArgparseFormatter(
+    argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter
+):
+    pass
+
+
 def main():
+    global info, log, dbg
+
     # filecache helps for reads that are ~64k or smaller;
     #   linux generally does 128k so the cache is a slowdown,
     #   windows likes to use 4k and 64k so cache is required,
     #   value is numChunks (1~3M each) to keep in the cache
-    nf = 24 if WINDOWS or MACOS else 0
+    nf = 24
 
     # dircache is always a boost,
     #   only want to disable it for tests etc,
     #   value is numSec until an entry goes stale
     nd = 1
 
-    try:
-        local, remote = sys.argv[1:3]
-        filecache = nf if len(sys.argv) <= 3 else int(sys.argv[3])
-        dircache = nd if len(sys.argv) <= 4 else float(sys.argv[4])
-    except:
     where = "local directory"
     if WINDOWS:
         where += " or DRIVE:"
 
-        print("need arg 1: " + where)
+    ex_pre = "\n  " + os.path.basename(__file__) + "  "
-        print("need arg 2: root url")
+    examples = ["http://192.168.1.69:3923/music/  ./music"]
-        print("optional 3: num files in filecache ({})".format(nf))
-        print("optional 4: num seconds / dircache ({})".format(nd))
-        print()
-        print("example:")
-        print("  copyparty-fuse.py ./music http://192.168.1.69:3923/music/")
     if WINDOWS:
-            print("  copyparty-fuse.py M: http://192.168.1.69:3923/music/")
+        examples.append("http://192.168.1.69:3923/music/  M:")
 
-        return
+    ap = argparse.ArgumentParser(
+        formatter_class=TheArgparseFormatter,
+        epilog="example:" + ex_pre + ex_pre.join(examples),
+    )
+    ap.add_argument(
+        "-cd", metavar="NUM_SECONDS", type=float, default=nd, help="directory cache"
+    )
+    ap.add_argument(
+        "-cf", metavar="NUM_BLOCKS", type=int, default=nf, help="file cache"
+    )
+    ap.add_argument("-a", metavar="PASSWORD", help="password")
+    ap.add_argument("-d", action="store_true", help="enable debug")
+    ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
+    ap.add_argument("-td", action="store_true", help="disable certificate check")
+    ap.add_argument("base_url", type=str, help="remote copyparty URL to mount")
+    ap.add_argument("local_path", type=str, help=where + " to mount it on")
+    ar = ap.parse_args()
+
+    if ar.d:
+        # windows terminals are slow (cmd.exe, mintty)
+        # otoh fancy_log beats RecentLog on linux
+        logger = RecentLog().put if WINDOWS else fancy_log
+
+        info = logger
+        log = logger
+        dbg = logger
+    else:
+        # debug=off, speed is dontcare
+        info = fancy_log
+        log = null_log
+        dbg = null_log
 
     if WINDOWS:
         os.system("")
 
+        for ch in '<>:"\\|?*':
+            # microsoft maps illegal characters to f0xx
+            # (e000 to f8ff is basic-plane private-use)
+            bad_good[ch] = chr(ord(ch) + 0xF000)
+
+        for n in range(0, 0x100):
+            # map surrogateescape to another private-use area
+            bad_good[chr(n + 0xDC00)] = chr(n + 0xF100)
+
+        for k, v in bad_good.items():
+            good_bad[v] = k
+
+    register_wtf8()
+
     try:
         with open("/etc/fuse.conf", "rb") as f:
             allow_other = b"\nuser_allow_other" in f.read()
@@ -845,7 +986,7 @@ def main():
     if not MACOS:
         args["nonempty"] = True
 
-    FUSE(CPPF(remote, dircache, filecache), local, **args)
+    FUSE(CPPF(ar), ar.local_path, encoding="wtf-8", **args)
 
 
 if __name__ == "__main__":

contrib/README.md

@@ -0,0 +1,19 @@
+### [`copyparty.bat`](copyparty.bat)
+* launches copyparty with no arguments (anon read+write within same folder)
+* intended for windows machines with no python.exe in PATH
+* works on windows, linux and macos
+* assumes `copyparty-sfx.py` was renamed to `copyparty.py` in the same folder as `copyparty.bat`
+
+### [`index.html`](index.html)
+* drop-in redirect from an httpd to copyparty
+* assumes the webserver and copyparty is running on the same server/IP
+* modify `10.13.1.1` as necessary if you wish to support browsers without javascript
+
+# OS integration
+init-scripts to start copyparty as a service
+* [`systemd/copyparty.service`](systemd/copyparty.service)
+* [`openrc/copyparty`](openrc/copyparty)
+
+# Reverse-proxy
+copyparty has basic support for running behind another webserver
+* [`nginx/copyparty.conf`](nginx/copyparty.conf)

contrib/copyparty.bat

@@ -0,0 +1,33 @@
+exec python "$(dirname "$0")"/copyparty.py
+
+@rem on linux, the above will execute and the script will terminate
+@rem on windows, the rest of this script will run
+
+@echo off
+cls
+
+set py=
+for /f %%i in ('where python 2^>nul') do (
+    set "py=%%i"
+    goto c1
+)
+:c1
+
+if [%py%] == [] (
+    for /f %%i in ('where /r "%localappdata%\programs\python" python 2^>nul') do (
+        set "py=%%i"
+        goto c2
+    )
+)
+:c2
+
+if [%py%] == [] set "py=c:\python27\python.exe"
+
+if not exist "%py%" (
+    echo could not find python
+    echo(
+    pause
+    exit /b
+)
+
+start cmd /c %py% "%~dp0\copyparty.py"

contrib/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>⇆🎉 redirect</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<style>
+
+html, body {
+	font-family: sans-serif;
+}
+body {
+	padding: 1em 2em;
+	font-size: 1.5em;
+}
+a {
+	font-size: 1.2em;
+	padding: .1em;
+}
+
+</style>
+</head>
+<body>
+	<span id="desc">you probably want</span> <a id="redir" href="//10.13.1.1:3923/">copyparty</a>
+	<script>
+
+var a = document.getElementById('redir'),
+	proto = window.location.protocol.indexOf('https') === 0 ? 'https' : 'http',
+	loc = window.location.hostname || '127.0.0.1',
+	port = a.getAttribute('href').split(':').pop().split('/')[0],
+	url = proto + '://' + loc + ':' + port + '/';
+
+a.setAttribute('href', url);
+document.getElementById('desc').innerHTML = 'redirecting to';
+
+setTimeout(function() {
+	window.location.href = url;
+}, 500);
+
+</script>
+</body>
+</html>

contrib/nginx/copyparty.conf

@@ -0,0 +1,26 @@
+upstream cpp {
+	server 127.0.0.1:3923;
+	keepalive 120;
+}
+server {
+	listen 443 ssl;
+	listen [::]:443 ssl;
+
+	server_name fs.example.com;
+	
+	location / {
+		proxy_pass http://cpp;
+		proxy_redirect off;
+		# disable buffering (next 4 lines)
+		proxy_http_version 1.1;
+		client_max_body_size 0;
+		proxy_buffering off;
+		proxy_request_buffering off;
+
+		proxy_set_header   Host              $host;
+		proxy_set_header   X-Real-IP         $remote_addr;
+		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+		proxy_set_header   X-Forwarded-Proto $scheme;
+		proxy_set_header   Connection        "Keep-Alive";
+	}
+}

contrib/openrc/copyparty

@@ -0,0 +1,18 @@
+#!/sbin/openrc-run
+
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   cp -pv copyparty /etc/init.d && rc-update add copyparty
+#
+# you may want to:
+#   change '/usr/bin/python' to another interpreter
+#   change '/mnt::a' to another location or permission-set
+
+name="$SVCNAME"
+command_background=true
+pidfile="/var/run/$SVCNAME.pid"
+
+command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
+command_args="-q -v /mnt::a"

contrib/systemd/copyparty.service

@@ -0,0 +1,19 @@
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
+#
+# you may want to:
+#   change '/usr/bin/python' to another interpreter
+#   change '/mnt::a' to another location or permission-set
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+ExecStart=/usr/bin/python /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
+
+[Install]
+WantedBy=multi-user.target

copyparty/__main__.py

@@ -123,19 +123,17 @@ def main():
             """
         ),
     )
-    ap.add_argument(
+    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-        "-c", metavar="PATH", type=str, action="append", help="add config file"
-    )
     ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind")
     ap.add_argument("-p", metavar="PORT", type=int, default=3923, help="port to bind")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=16, help="max num clients")
+    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
-    ap.add_argument(
+    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-        "-j", metavar="CORES", type=int, default=1, help="max num cpu cores"
-    )
     ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
     ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
     ap.add_argument("-q", action="store_true", help="quiet")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 5, 2)
+VERSION = (0, 6, 0)
-CODENAME = "fuse jelly"
+CODENAME = "CHRISTMAAAAAS"
-BUILD_DT = (2020, 8, 18)
+BUILD_DT = (2020, 12, 1)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -16,9 +16,6 @@ from .util import *  # noqa  # pylint: disable=unused-wildcard-import
 
 if not PY2:
     unicode = str
-    from html import escape as html_escape
-else:
-    from cgi import escape as html_escape  # pylint: disable=no-name-in-module
 
 
 class HttpCli(object):
@@ -27,6 +24,7 @@ class HttpCli(object):
     """
 
     def __init__(self, conn):
+        self.t0 = time.time()
         self.conn = conn
         self.s = conn.s
         self.sr = conn.sr
@@ -85,11 +83,15 @@ class HttpCli(object):
         v = self.headers.get("connection", "").lower()
         self.keepalive = not v.startswith("close")
 
+        v = self.headers.get("x-forwarded-for", None)
+        if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
+            self.log_src = self.conn.set_rproxy(v.split(",")[0])
+
         self.uname = "*"
         if "cookie" in self.headers:
             cookies = self.headers["cookie"].split(";")
             for k, v in [x.split("=", 1) for x in cookies]:
-                if k != "cppwd":
+                if k.strip() != "cppwd":
                     continue
 
                 v = unescape_cookie(v)
@@ -125,6 +127,11 @@ class HttpCli(object):
         self.uparam = uparam
         self.vpath = unquotep(vpath)
 
+        ua = self.headers.get("user-agent", "")
+        if ua.startswith("rclone/"):
+            uparam["raw"] = True
+            uparam["dots"] = True
+
         try:
             if self.mode in ["GET", "HEAD"]:
                 return self.handle_get() and self.keepalive
@@ -141,7 +148,7 @@ class HttpCli(object):
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
                 self.keepalive = self._check_nonfatal(ex)
-                self.loud_reply(str(ex), status=ex.code)
+                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
@@ -180,6 +187,7 @@ class HttpCli(object):
         self.send_headers(len(body), status, mime, headers)
 
         try:
+            if self.mode != "HEAD":
                 self.s.sendall(body)
         except:
             raise Pebkac(400, "client d/c while replying body")
@@ -188,7 +196,7 @@ class HttpCli(object):
 
     def loud_reply(self, body, *args, **kwargs):
         self.log(body.rstrip())
-        self.reply(b"<pre>" + body.encode("utf-8"), *list(args), **kwargs)
+        self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
     def handle_get(self):
         logmsg = "{:4} {}".format(self.mode, self.req)
@@ -304,10 +312,19 @@ class HttpCli(object):
         with open(path, "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
-        self.log("wrote {}/{} bytes to {}".format(post_sz, remains, path))
+        spd = self._spd(post_sz)
+        self.log("{} wrote {}/{} bytes to {}".format(spd, post_sz, remains, path))
         self.reply("{}\n{}\n".format(post_sz, sha_b64).encode("utf-8"))
         return True
 
+    def _spd(self, nbytes, add=True):
+        if add:
+            self.conn.nbyte += nbytes
+
+        spd1 = get_spd(nbytes, self.t0)
+        spd2 = get_spd(self.conn.nbyte, self.conn.t0)
+        return spd1 + " " + spd2
+
     def handle_post_multipart(self):
         self.parser = MultipartParser(self.log, self.sr, self.headers)
         self.parser.parse()
@@ -447,7 +464,9 @@ class HttpCli(object):
             except:
                 self.log("failed to utime ({}, {})".format(path, times))
 
-        self.loud_reply("thank")
+        spd = self._spd(post_sz)
+        self.log("{} thank".format(spd))
+        self.reply(b"thank")
         return True
 
     def handle_login(self):
@@ -460,7 +479,7 @@ class HttpCli(object):
             msg = "naw dude"
             pwd = "x"  # nosec
 
-        h = {"Set-Cookie": "cppwd={}; Path=/".format(pwd)}
+        h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
         html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
         self.reply(html.encode("utf-8"), headers=h)
         return True
@@ -493,7 +512,7 @@ class HttpCli(object):
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         html = self.conn.tpl_msg.render(
             h2='<a href="/{}">go to /{}</a>'.format(
-                quotep(vpath), html_escape(vpath, quote=False)
+                quotep(vpath), html_escape(vpath)
             ),
             pre="aight",
             click=True,
@@ -527,7 +546,7 @@ class HttpCli(object):
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         html = self.conn.tpl_msg.render(
             h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
-                quotep(vpath), html_escape(vpath, quote=False)
+                quotep(vpath), html_escape(vpath)
             ),
             pre="aight",
             click=True,
@@ -572,6 +591,7 @@ class HttpCli(object):
                             raise Pebkac(400, "empty files in post")
 
                         files.append([sz, sha512_hex])
+                        self.conn.nbyte += sz
 
                 except Pebkac:
                     if fn != os.devnull:
@@ -599,7 +619,9 @@ class HttpCli(object):
             # truncated SHA-512 prevents length extension attacks;
             # using SHA-512/224, optionally SHA-512/256 = :64
 
-        self.log(msg)
+        vspd = self._spd(sz_total, False)
+        self.log("{} {}".format(vspd, msg))
+
         if not nullwrite:
             # TODO this is bad
             log_fn = "up.{:.6f}.txt".format(t0)
@@ -621,7 +643,7 @@ class HttpCli(object):
 
         html = self.conn.tpl_msg.render(
             h2='<a href="/{}">return to /{}</a>'.format(
-                quotep(self.vpath), html_escape(self.vpath, quote=False)
+                quotep(self.vpath), html_escape(self.vpath)
             ),
             pre=msg,
         )
@@ -734,9 +756,12 @@ class HttpCli(object):
                 cli_dt = time.strptime(cli_lastmod, "%a, %d %b %Y %H:%M:%S GMT")
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
-            except:
+            except Exception as ex:
-                self.log("bad lastmod format: {}".format(cli_lastmod))
+                self.log(
-                self.log("   expected format: {}".format(file_lastmod))
+                    "lastmod {}\nremote: [{}]\n local: [{}]".format(
+                        repr(ex), cli_lastmod, file_lastmod
+                    )
+                )
                 return file_lastmod, file_lastmod != cli_lastmod
 
         return file_lastmod, True
@@ -882,6 +907,7 @@ class HttpCli(object):
             self.log(logmsg)
             return True
 
+        ret = True
         with open_func(*open_args) as f:
             remains = upper - lower
             f.seek(lower)
@@ -894,17 +920,17 @@ class HttpCli(object):
                 if remains < len(buf):
                     buf = buf[:remains]
 
-                remains -= len(buf)
-
                 try:
                     self.s.sendall(buf)
+                    remains -= len(buf)
                 except:
                     logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
-                    self.log(logmsg)
+                    ret = False
-                    return False
+                    break
 
-        self.log(logmsg)
+        spd = self._spd((upper - lower) - remains)
-        return True
+        self.log("{},  {}".format(logmsg, spd))
+        return ret
 
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
@@ -938,8 +964,10 @@ class HttpCli(object):
 
         targs = {
             "edit": "edit" in self.uparam,
-            "title": html_escape(self.vpath, quote=False),
+            "title": html_escape(self.vpath),
             "lastmod": int(ts_md * 1000),
+            "md_plug": "true" if self.args.emp else "false",
+            "md_chk_rate": self.args.mcr,
             "md": "",
         }
         sz_html = len(template.render(**targs).encode("utf-8"))
@@ -979,7 +1007,7 @@ class HttpCli(object):
                 else:
                     vpath += "/" + node
 
-                vpnodes.append([quotep(vpath) + "/", html_escape(node, quote=False)])
+                vpnodes.append([quotep(vpath) + "/", html_escape(node)])
 
         vn, rem = self.auth.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
@@ -1054,7 +1082,12 @@ class HttpCli(object):
             dt = datetime.utcfromtimestamp(inf.st_mtime)
             dt = dt.strftime("%Y-%m-%d %H:%M:%S")
 
-            item = [margin, quotep(href), html_escape(fn, quote=False), sz, dt]
+            try:
+                ext = "---" if is_dir else fn.rsplit(".", 1)[1]
+            except:
+                ext = "%"
+
+            item = [margin, quotep(href), html_escape(fn), sz, ext, dt]
             if is_dir:
                 dirs.append(item)
             else:
@@ -1119,7 +1152,7 @@ class HttpCli(object):
             ts=ts,
             prologue=logues[0],
             epilogue=logues[1],
-            title=html_escape(self.vpath, quote=False),
+            title=html_escape(self.vpath),
             srv_info="</span> /// <span>".join(srv_info),
         )
         self.reply(html.encode("utf-8", "replace"))

copyparty/httpconn.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import os
 import sys
 import ssl
+import time
 import socket
 
 try:
@@ -41,9 +42,11 @@ class HttpConn(object):
         self.auth = hsrv.auth
         self.cert_path = hsrv.cert_path
 
+        self.t0 = time.time()
+        self.nbyte = 0
         self.workload = 0
         self.log_func = hsrv.log
-        self.log_src = "{} \033[36m{}".format(addr[0], addr[1]).ljust(26)
+        self.set_rproxy()
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
@@ -53,6 +56,18 @@ class HttpConn(object):
         self.tpl_md = env.get_template("md.html")
         self.tpl_mde = env.get_template("mde.html")
 
+    def set_rproxy(self, ip=None):
+        if ip is None:
+            color = 36
+            ip = self.addr[0]
+            self.rproxy = None
+        else:
+            color = 34
+            self.rproxy = ip
+
+        self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
+        return self.log_src
+
     def respath(self, res_name):
         return os.path.join(E.mod, "web", res_name)
 

copyparty/util.py

@@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals
 
 import re
 import sys
+import time
 import base64
 import struct
 import hashlib
@@ -335,18 +336,28 @@ def read_header(sr):
 
 
 def humansize(sz, terse=False):
-    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB']:
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
         if sz < 1024:
             break
 
-        sz /= 1024.
+        sz /= 1024.0
 
-    ret = ' '.join([str(sz)[:4].rstrip('.'), unit])
+    ret = " ".join([str(sz)[:4].rstrip("."), unit])
 
     if not terse:
         return ret
 
-    return ret.replace('iB', '').replace(' ', '')
+    return ret.replace("iB", "").replace(" ", "")
+
+
+def get_spd(nbyte, t0, t=None):
+    if t is None:
+        t = time.time()
+
+    bps = nbyte / ((t - t0) + 0.001)
+    s1 = humansize(nbyte).replace(" ", "\033[33m").replace("iB", "")
+    s2 = humansize(bps).replace(" ", "\033[35m").replace("iB", "")
+    return "{} \033[0m{}/s\033[0m".format(s1, s2)
 
 
 def undot(path):
@@ -398,6 +409,21 @@ def exclude_dotfiles(filepaths):
             yield fpath
 
 
+def html_escape(s, quote=False):
+    """html.escape but also newlines"""
+    s = (
+        s.replace("&", "&amp;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace("\r", "&#13;")
+        .replace("\n", "&#10;")
+    )
+    if quote:
+        s = s.replace('"', "&quot;").replace("'", "&#x27;")
+
+    return s
+
+
 def quotep(txt):
     """url quoter which deals with bytes correctly"""
     btxt = w8enc(txt)
@@ -412,8 +438,8 @@ def quotep(txt):
 def unquotep(txt):
     """url unquoter which deals with bytes correctly"""
     btxt = w8enc(txt)
-    unq1 = btxt.replace(b"+", b" ")
+    # btxt = btxt.replace(b"+", b" ")
-    unq2 = unquote(unq1)
+    unq2 = unquote(btxt)
     return w8dec(unq2)
 
 

copyparty/web/Makefile

@@ -0,0 +1,12 @@
+# run me to zopfli all the static files
+# which should help on really slow connections
+# but then why are you using copyparty in the first place
+
+pk: $(addsuffix .gz, $(wildcard *.js *.css))
+un: $(addsuffix .un, $(wildcard *.gz))
+
+%.gz: %
+	pigz -11 -J 34 -I 5730 $<
+
+%.un: %
+	pigz -d $<

copyparty/web/browser.html

@@ -34,13 +34,14 @@
                 <th></th>
                 <th>File Name</th>
                 <th sort="int">File Size</th>
+                <th>T</th>
                 <th>Date</th>
             </tr>
         </thead>
         <tbody>
 
 {%- for f in files %}
-<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td></tr>
+<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td><td>{{ f[5] }}</td></tr>
 {%- endfor %}
 
         </tbody>
@@ -67,6 +68,8 @@
         </div>
     </div>
     
+    <script src="/.cpr/util.js{{ ts }}"></script>
+
     {%- if can_read %}
     <script src="/.cpr/browser.js{{ ts }}"></script>
     {%- endif %}

copyparty/web/browser.js

@@ -1,75 +1,9 @@
 "use strict";
 
-// error handler for mobile devices
+window.onerror = vis_exh;
-function hcroak(msg) {
-	document.body.innerHTML = msg;
-	window.onerror = undefined;
-	throw 'fatal_err';
-}
-function croak(msg) {
-	document.body.textContent = msg;
-	window.onerror = undefined;
-	throw msg;
-}
-function esc(txt) {
-	return txt.replace(/[&"<>]/g, function (c) {
-		return {
-			'&': '&amp;',
-			'"': '&quot;',
-			'<': '&lt;',
-			'>': '&gt;'
-		}[c];
-	});
-}
-window.onerror = function (msg, url, lineNo, columnNo, error) {
-	window.onerror = undefined;
-	var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
-		esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
-
-	if (error) {
-		var find = ['desc', 'stack', 'trace'];
-		for (var a = 0; a < find.length; a++)
-			if (String(error[find[a]]) !== 'undefined')
-				html.push('<h2>' + find[a] + '</h2>' +
-					esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
-	}
-	document.body.style.fontSize = '0.8em';
-	document.body.style.padding = '0 1em 1em 1em';
-	hcroak(html.join('\n'));
-};
-
-
-// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
-if (!String.prototype.endsWith) {
-	String.prototype.endsWith = function (search, this_len) {
-		if (this_len === undefined || this_len > this.length) {
-			this_len = this.length;
-		}
-		return this.substring(this_len - search.length, this_len) === search;
-	};
-}
-
-
-// https://stackoverflow.com/a/950146
-function import_js(url, cb) {
-	var head = document.head || document.getElementsByTagName('head')[0];
-	var script = document.createElement('script');
-	script.type = 'text/javascript';
-	script.src = url;
-
-	script.onreadystatechange = cb;
-	script.onload = cb;
-
-	head.appendChild(script);
-}
-
-
-function o(id) {
-	return document.getElementById(id);
-}
 
 function dbg(msg) {
-	o('path').innerHTML = msg;
+	ebi('path').innerHTML = msg;
 }
 
 function ev(e) {
@@ -78,40 +12,7 @@ function ev(e) {
 	return e;
 }
 
-
+makeSortable(ebi('files'));
-function sortTable(table, col) {
-	var tb = table.tBodies[0], // use `<tbody>` to ignore `<thead>` and `<tfoot>` rows
-		th = table.tHead.rows[0].cells,
-		tr = Array.prototype.slice.call(tb.rows, 0),
-		i, reverse = th[col].className == 'sort1' ? -1 : 1;
-	for (var a = 0, thl = th.length; a < thl; a++)
-		th[a].className = '';
-	th[col].className = 'sort' + reverse;
-	var stype = th[col].getAttribute('sort');
-	tr = tr.sort(function (a, b) {
-		var v1 = a.cells[col].textContent.trim();
-		var v2 = b.cells[col].textContent.trim();
-		if (stype == 'int') {
-			v1 = parseInt(v1.replace(/,/g, ''));
-			v2 = parseInt(v2.replace(/,/g, ''));
-			return reverse * (v1 - v2);
-		}
-		return reverse * (v1.localeCompare(v2));
-	});
-	for (i = 0; i < tr.length; ++i) tb.appendChild(tr[i]);
-}
-function makeSortable(table) {
-	var th = table.tHead, i;
-	th && (th = th.rows[0]) && (th = th.cells);
-	if (th) i = th.length;
-	else return; // if no `<thead>` then do nothing
-	while (--i >= 0) (function (i) {
-		th[i].onclick = function () {
-			sortTable(table, i);
-		};
-	}(i));
-}
-makeSortable(o('files'));
 
 
 // extract songs + add play column
@@ -124,9 +25,9 @@ var mp = (function () {
 		'tracks': tracks,
 		'cover_url': ''
 	};
-	var re_audio = new RegExp('\.(opus|ogg|m4a|aac|mp3|wav|flac)$', 'i');
+	var re_audio = /\.(opus|ogg|m4a|aac|mp3|wav|flac)$/i;
 
-	var trs = document.getElementById('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr');
+	var trs = ebi('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr');
 	for (var a = 0, aa = trs.length; a < aa; a++) {
 		var tds = trs[a].getElementsByTagName('td');
 		var link = tds[1].getElementsByTagName('a')[0];
@@ -142,7 +43,7 @@ var mp = (function () {
 	}
 
 	for (var a = 0, aa = tracks.length; a < aa; a++)
-		o('trk' + a).onclick = ev_play;
+		ebi('trk' + a).onclick = ev_play;
 
 	ret.vol = localStorage.getItem('vol');
 	if (ret.vol !== null)
@@ -169,8 +70,8 @@ var mp = (function () {
 // toggle player widget
 var widget = (function () {
 	var ret = {};
-	var widget = document.getElementById('widget');
+	var widget = ebi('widget');
-	var wtoggle = document.getElementById('wtoggle');
+	var wtoggle = ebi('wtoggle');
 	var touchmode = false;
 	var side_open = false;
 	var was_paused = true;
@@ -199,7 +100,7 @@ var widget = (function () {
 	ret.paused = function (paused) {
 		if (was_paused != paused) {
 			was_paused = paused;
-			o('bplay').innerHTML = paused ? '▶' : '⏸';
+			ebi('bplay').innerHTML = paused ? '▶' : '⏸';
 		}
 	};
 	var click_handler = function (e) {
@@ -223,8 +124,8 @@ var widget = (function () {
 // buffer/position bar
 var pbar = (function () {
 	var r = {};
-	r.bcan = o('barbuf');
+	r.bcan = ebi('barbuf');
-	r.pcan = o('barpos');
+	r.pcan = ebi('barpos');
 	r.bctx = r.bcan.getContext('2d');
 	r.pctx = r.pcan.getContext('2d');
 
@@ -289,7 +190,7 @@ var pbar = (function () {
 // volume bar
 var vbar = (function () {
 	var r = {};
-	r.can = o('pvol');
+	r.can = ebi('pvol');
 	r.ctx = r.can.getContext('2d');
 
 	var bctx = r.ctx;
@@ -386,7 +287,7 @@ var vbar = (function () {
 		else
 			play(0);
 	};
-	o('bplay').onclick = function (e) {
+	ebi('bplay').onclick = function (e) {
 		ev(e);
 		if (mp.au) {
 			if (mp.au.paused)
@@ -397,15 +298,15 @@ var vbar = (function () {
 		else
 			play(0);
 	};
-	o('bprev').onclick = function (e) {
+	ebi('bprev').onclick = function (e) {
 		ev(e);
 		bskip(-1);
 	};
-	o('bnext').onclick = function (e) {
+	ebi('bnext').onclick = function (e) {
 		ev(e);
 		bskip(1);
 	};
-	o('barpos').onclick = function (e) {
+	ebi('barpos').onclick = function (e) {
 		if (!mp.au) {
 			//dbg((new Date()).getTime());
 			return play(0);
@@ -471,7 +372,7 @@ function ev_play(e) {
 
 
 function setclass(id, clas) {
-	o(id).setAttribute('class', clas);
+	ebi(id).setAttribute('class', clas);
 }
 
 
@@ -567,7 +468,6 @@ function play(tid, call_depth) {
 function evau_error(e) {
 	var err = '';
 	var eplaya = (e && e.target) || (window.event && window.event.srcElement);
-	var url = eplaya.src;
 
 	switch (eplaya.error.code) {
 		case eplaya.error.MEDIA_ERR_ABORTED:
@@ -608,26 +508,27 @@ function show_modal(html) {
 
 // hide fullscreen message
 function unblocked() {
-	var dom = o('blocked');
+	var dom = ebi('blocked');
 	if (dom)
 		dom.parentNode.removeChild(dom);
 }
 
 
 // show ui to manually start playback of a linked song
-function autoplay_blocked(tid) {
+function autoplay_blocked() {
 	show_modal(
-		'<div id="blk_play"><a id="blk_go"></a></div>' +
+		'<div id="blk_play"><a href="#" id="blk_go"></a></div>' +
-		'<div id="blk_abrt"><a id="blk_na">Cancel<br />(show file list)</a></div>');
+		'<div id="blk_abrt"><a href="#" id="blk_na">Cancel<br />(show file list)</a></div>');
 
-	var go = o('blk_go');
+	var go = ebi('blk_go');
-	var na = o('blk_na');
+	var na = ebi('blk_na');
 
 	var fn = mp.tracks[mp.au.tid].split(/\//).pop();
 	fn = decodeURIComponent(fn.replace(/\+/g, ' '));
 
 	go.textContent = 'Play "' + fn + '"';
-	go.onclick = function () {
+	go.onclick = function (e) {
+		if (e) e.preventDefault();
 		unblocked();
 		mp.au.play();
 	};

copyparty/web/md.css

@@ -109,8 +109,12 @@ h2 a, h4 a, h6 a {
 #mp ol>li {
 	margin: .7em 0;
 }
+strong {
+	color: #000;
+}
 p>em,
-li>em {
+li>em,
+td>em {
 	color: #c50;
 	padding: .1em;
 	border-bottom: .1em solid #bbb;
@@ -289,6 +293,32 @@ blink {
 		text-decoration: underline;
 		border: none;
 	}
+	#mh a:hover {
+		color: #000;
+		background: #ddd;
+	}
+	#toolsbox {
+		overflow: hidden;
+		display: inline-block;
+		background: #eee;
+		height: 1.5em;
+		padding: 0 .2em;
+		margin: 0 .2em;
+		position: absolute;
+	}
+	#toolsbox.open {
+		height: auto;
+		overflow: visible;
+		background: #eee;
+		box-shadow: 0 .2em .2em #ccc;
+		padding-bottom: .2em;
+	}
+	#toolsbox a {
+		display: block;
+	}
+	#toolsbox a+a {
+		text-decoration: none;
+	}
 
 
 
@@ -332,8 +362,12 @@ blink {
 	html.dark #m>ol {
 		border-color: #555;
 	}
+	html.dark strong {
+		color: #fff;
+	}
 	html.dark p>em,
-	html.dark li>em {
+	html.dark li>em,
+	html.dark td>em {
 		color: #f94;
 		border-color: #666;
 	}
@@ -371,6 +405,17 @@ blink {
 		color: #ccc;
 		background: none;
 	}
+	html.dark #mh a:hover {
+		background: #333;
+		color: #fff;
+	}
+	html.dark #toolsbox {
+		background: #222;
+	}
+	html.dark #toolsbox.open {
+		box-shadow: 0 .2em .2em #069;
+		border-radius: 0 0 .4em .4em;
+	}
 }
 
 @media screen and (min-width: 66em) {
@@ -541,7 +586,8 @@ blink {
 		color: #240;
 	}
 	html.dark p>em,
-	html.dark li>em {
+	html.dark li>em,
+	html.dark td>em {
 		color: #940;
 	}
 }

copyparty/web/md.html

@@ -17,7 +17,14 @@
 			<a id="save" href="?edit">save</a>
 			<a id="sbs" href="#">sbs</a>
 			<a id="nsbs" href="#">editor</a>
+			<div id="toolsbox">
+				<a id="tools" href="#">tools</a>
+				<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
+				<a id="iter_uni" href="#">non-ascii: iterate (ctrl-u)</a>
+				<a id="mark_uni" href="#">non-ascii: markup</a>
+				<a id="cfg_uni" href="#">non-ascii: whitelist</a>
 				<a id="help" href="#">help</a>
+			</div>
 		{%- else %}
 			<a href="?edit">edit (basic)</a>
 			<a href="?edit2">edit (fancy)</a>
@@ -46,6 +53,9 @@ write markdown (most html is 🙆 too)
 
 ## hotkey list
 * `Ctrl-S` to save
+* `Ctrl-E` to toggle mode
+* `Ctrl-K` to prettyprint a table
+* `Ctrl-U` to iterate non-ascii chars
 * `Ctrl-H` / `Ctrl-Shift-H` to create a header
 * `TAB` / `Shift-TAB` to indent/dedent a selection
 
@@ -113,8 +123,12 @@ write markdown (most html is 🙆 too)
 	
 	<script>
 
-var link_md_as_html = false;  // TODO (does nothing)
 var last_modified = {{ lastmod }};
+var md_opt = {
+	link_md_as_html: false,
+	allow_plugins: {{ md_plug }},
+	modpoll_freq: {{ md_chk_rate }}
+};
 
 (function () {
     var btn = document.getElementById("lightswitch");
@@ -131,14 +145,8 @@ var last_modified = {{ lastmod }};
 		toggle();
 })();
 
-if (!String.startsWith) {
-	String.prototype.startsWith = function(s, i) {
-		i = i>0 ? i|0 : 0;
-		return this.substring(i, i + s.length) === s;
-	};
-}
-
 	</script>
+    <script src="/.cpr/util.js"></script>
 	<script src="/.cpr/deps/marked.full.js"></script>
 	<script src="/.cpr/md.js"></script>
 	{%- if edit %}

copyparty/web/md.js

@@ -1,10 +1,12 @@
-var dom_toc = document.getElementById('toc');
+"use strict";
-var dom_wrap = document.getElementById('mw');
+
-var dom_hbar = document.getElementById('mh');
+var dom_toc = ebi('toc');
-var dom_nav = document.getElementById('mn');
+var dom_wrap = ebi('mw');
-var dom_pre = document.getElementById('mp');
+var dom_hbar = ebi('mh');
-var dom_src = document.getElementById('mt');
+var dom_nav = ebi('mn');
-var dom_navtgl = document.getElementById('navtoggle');
+var dom_pre = ebi('mp');
+var dom_src = ebi('mt');
+var dom_navtgl = ebi('navtoggle');
 
 
 // chrome 49 needs this
@@ -18,6 +20,10 @@ var dbg = function () { };
 // dbg = console.log
 
 
+// plugins
+var md_plug = {};
+
+
 function hesc(txt) {
     return txt.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 }
@@ -30,7 +36,7 @@ function cls(dom, name, add) {
 }
 
 
-function static(obj) {
+function statify(obj) {
     return JSON.parse(JSON.stringify(obj));
 }
 
@@ -154,13 +160,110 @@ function copydom(src, dst, lv) {
 }
 
 
+function md_plug_err(ex, js) {
+    var errbox = ebi('md_errbox');
+    if (errbox)
+        errbox.parentNode.removeChild(errbox);
+
+    if (!ex)
+        return;
+
+    var msg = (ex + '').split('\n')[0];
+    var ln = ex.lineNumber;
+    var o = null;
+    if (ln) {
+        msg = "Line " + ln + ", " + msg;
+        var lns = js.split('\n');
+        if (ln < lns.length) {
+            o = document.createElement('span');
+            o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
+            o.textContent = lns[ln - 1];
+        }
+    }
+    errbox = document.createElement('div');
+    errbox.setAttribute('id', 'md_errbox');
+    errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
+    errbox.textContent = msg;
+    errbox.onclick = function () {
+        alert('' + ex.stack);
+    };
+    if (o) {
+        errbox.appendChild(o);
+        errbox.style.padding = '.25em .5em';
+    }
+    dom_nav.appendChild(errbox);
+
+    try {
+        console.trace();
+    }
+    catch (ex2) { }
+}
+
+
+function load_plug(md_text, plug_type) {
+    if (!md_opt.allow_plugins)
+        return md_text;
+
+    var find = '\n```copyparty_' + plug_type + '\n';
+    var ofs = md_text.indexOf(find);
+    if (ofs === -1)
+        return md_text;
+
+    var ofs2 = md_text.indexOf('\n```', ofs + 1);
+    if (ofs2 == -1)
+        return md_text;
+
+    var js = md_text.slice(ofs + find.length, ofs2 + 1);
+    var md = md_text.slice(0, ofs + 1) + md_text.slice(ofs2 + 4);
+
+    var old_plug = md_plug[plug_type];
+    if (!old_plug || old_plug[1] != js) {
+        js = 'const x = { ' + js + ' }; x;';
+        try {
+            var x = eval(js);
+        }
+        catch (ex) {
+            md_plug[plug_type] = null;
+            md_plug_err(ex, js);
+            return md;
+        }
+        if (x['ctor']) {
+            x['ctor']();
+            delete x['ctor'];
+        }
+        md_plug[plug_type] = [x, js];
+    }
+
+    return md;
+}
+
+
 function convert_markdown(md_text, dest_dom) {
-    marked.setOptions({
+    md_text = md_text.replace(/\r/g, '');
+
+    md_plug_err(null);
+    md_text = load_plug(md_text, 'pre');
+    md_text = load_plug(md_text, 'post');
+
+    var marked_opts = {
         //headerPrefix: 'h-',
         breaks: true,
         gfm: true
-    });
+    };
-    var md_html = marked(md_text);
+
+    var ext = md_plug['pre'];
+    if (ext)
+        Object.assign(marked_opts, ext[0]);
+
+    try {
+        var md_html = marked(md_text, marked_opts);
+    }
+    catch (ex) {
+        if (ext)
+            md_plug_err(ex, ext[1]);
+
+        throw ex;
+    }
     var md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
 
     var nodes = md_dom.getElementsByTagName('a');
@@ -196,7 +299,7 @@ function convert_markdown(md_text, dest_dom) {
     }
 
     // separate <code> for each line in <pre>
-    var nodes = md_dom.getElementsByTagName('pre');
+    nodes = md_dom.getElementsByTagName('pre');
     for (var a = nodes.length - 1; a >= 0; a--) {
         var el = nodes[a];
 
@@ -209,7 +312,7 @@ function convert_markdown(md_text, dest_dom) {
             continue;
 
         var nline = parseInt(el.getAttribute('data-ln')) + 1;
-        var lines = el.innerHTML.replace(/\r?\n<\/code>$/i, '</code>').split(/\r?\n/g);
+        var lines = el.innerHTML.replace(/\n<\/code>$/i, '</code>').split(/\n/g);
         for (var b = 0; b < lines.length - 1; b++)
             lines[b] += '</code>\n<code data-ln="' + (nline + b) + '">';
 
@@ -242,12 +345,29 @@ function convert_markdown(md_text, dest_dom) {
         el.innerHTML = '<a href="#' + id + '">' + el.innerHTML + '</a>';
     }
 
+    ext = md_plug['post'];
+    if (ext && ext[0].render)
+        try {
+            ext[0].render(md_dom);
+        }
+        catch (ex) {
+            md_plug_err(ex, ext[1]);
+        }
+
     copydom(md_dom, dest_dom, 0);
+
+    if (ext && ext[0].render2)
+        try {
+            ext[0].render2(dest_dom);
+        }
+        catch (ex) {
+            md_plug_err(ex, ext[1]);
+        }
 }
 
 
 function init_toc() {
-    var loader = document.getElementById('ml');
+    var loader = ebi('ml');
     loader.parentNode.removeChild(loader);
 
     var anchors = [];  // list of toc entries, complex objects
@@ -281,7 +401,12 @@ function init_toc() {
 
             elm.childNodes[0].setAttribute('ctr', ctr.slice(0, lv).join('.'));
 
-            html.push('<li>' + elm.innerHTML + '</li>');
+            var elm2 = elm.cloneNode(true);
+            elm2.childNodes[0].textContent = elm.textContent;
+            while (elm2.childNodes.length > 1)
+                elm2.removeChild(elm2.childNodes[1]);
+
+            html.push('<li>' + elm2.innerHTML + '</li>');
 
             if (anchor != null)
                 anchors.push(anchor);

copyparty/web/md2.css

@@ -77,32 +77,52 @@ html.dark #mt {
     background: #f97;
     border-radius: .15em;
 }
+html.dark #save.force-save {
+    color: #fca;
+    background: #720;
+}
 #save.disabled {
     opacity: .4;
 }
+#helpbox,
+#toast {
+    background: #f7f7f7;
+    border-radius: .4em;
+    z-index: 9001;
+}
 #helpbox {
     display: none;
     position: fixed;
-    background: #f7f7f7;
-    box-shadow: 0 .5em 2em #777;
-    border-radius: .4em;
     padding: 2em;
     top: 4em;
     overflow-y: auto;
+    box-shadow: 0 .5em 2em #777;
     height: calc(100% - 12em);
     left: calc(50% - 15em);
     right: 0;
     width: 30em;
-    z-index: 9001;
 }
 #helpclose {
     display: block;
 }
 html.dark #helpbox {
-    background: #222;
     box-shadow: 0 .5em 2em #444;
+}
+html.dark #helpbox,
+html.dark #toast {
+    background: #222;
     border: 1px solid #079;
     border-width: 1px 0;
 }
+#toast {
+    font-weight: bold;
+    text-align: center;
+    padding: .6em 0;
+    position: fixed;
+    z-index: 9001;
+    top: 30%;
+    transition: opacity 0.2s ease-in-out;
+    opacity: 1;
+}
 
 # mt {opacity: .5;top:1px}

copyparty/web/md2.js

@@ -1,16 +1,25 @@
+"use strict";
+
+
 // server state
 var server_md = dom_src.value;
 
 
+// the non-ascii whitelist
+var esc_uni_whitelist = '\\n\\t\\x20-\\x7eÆØÅæøå';
+var js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+
+
 // dom nodes
-var dom_swrap = document.getElementById('mtw');
+var dom_swrap = ebi('mtw');
-var dom_sbs = document.getElementById('sbs');
+var dom_sbs = ebi('sbs');
-var dom_nsbs = document.getElementById('nsbs');
+var dom_nsbs = ebi('nsbs');
+var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
     var d = document.createElement('div');
     d.setAttribute('id', 'mtr');
     dom_swrap.appendChild(d);
-    d = document.getElementById('mtr');
+    d = ebi('mtr');
     // hide behind the textarea (offsetTop is not computed if display:none)
     dom_src.style.zIndex = '4';
     d.style.zIndex = '3';
@@ -99,7 +108,7 @@ var draw_md = (function () {
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
 
-        cls(document.getElementById('save'), 'disabled', src == server_md);
+        cls(ebi('save'), 'disabled', src == server_md);
 
         var t1 = new Date().getTime();
         delay = t1 - t0 > 100 ? 25 : 1;
@@ -135,7 +144,7 @@ redraw = (function () {
         onresize();
     }
     function modetoggle() {
-        mode = dom_nsbs.innerHTML;
+        var mode = dom_nsbs.innerHTML;
         dom_nsbs.innerHTML = mode == 'editor' ? 'preview' : 'editor';
         mode += ' single';
         dom_wrap.setAttribute('class', mode);
@@ -164,14 +173,14 @@ redraw = (function () {
             dst.scrollTop = 0;
             return;
         }
-        if (y + 8 + src.clientHeight > src.scrollHeight) {
+        if (y + 48 + src.clientHeight > src.scrollHeight) {
             dst.scrollTop = dst.scrollHeight - dst.clientHeight;
             return;
         }
         y += src.clientHeight / 2;
         var sy1 = -1, sy2 = -1, dy1 = -1, dy2 = -1;
         for (var a = 1; a < nlines + 1; a++) {
-            if (srcmap[a] === null || dstmap[a] === null)
+            if (srcmap[a] == null || dstmap[a] == null)
                 continue;
 
             if (srcmap[a] > y) {
@@ -214,14 +223,108 @@ redraw = (function () {
 })();
 
 
+// modification checker
+function Modpoll() {
+    this.skip_one = true;
+    this.disabled = false;
+
+    this.periodic = function () {
+        var that = this;
+        setTimeout(function () {
+            that.periodic();
+        }, 1000 * md_opt.modpoll_freq);
+
+        var skip = null;
+
+        if (ebi('toast'))
+            skip = 'toast';
+
+        else if (this.skip_one)
+            skip = 'saved';
+
+        else if (this.disabled)
+            skip = 'disabled';
+
+        if (skip) {
+            console.log('modpoll skip, ' + skip);
+            this.skip_one = false;
+            return;
+        }
+
+        console.log('modpoll...');
+        var url = (document.location + '').split('?')[0] + '?raw&_=' + new Date().getTime();
+        var xhr = new XMLHttpRequest();
+        xhr.modpoll = this;
+        xhr.open('GET', url, true);
+        xhr.responseType = 'text';
+        xhr.onreadystatechange = this.cb;
+        xhr.send();
+    }
+
+    this.cb = function () {
+        if (this.modpoll.disabled || this.modpoll.skip_one) {
+            console.log('modpoll abort');
+            return;
+        }
+
+        if (this.readyState != XMLHttpRequest.DONE)
+            return;
+
+        if (this.status !== 200) {
+            console.log('modpoll err ' + this.status + ": " + this.responseText);
+            return;
+        }
+
+        if (!this.responseText)
+            return;
+
+        var server_ref = server_md.replace(/\r/g, '');
+        var server_now = this.responseText.replace(/\r/g, '');
+
+        if (server_ref != server_now) {
+            console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
+            this.modpoll.disabled = true;
+            var msg = [
+                "The document has changed on the server.<br />" +
+                "The changes will NOT be loaded into your editor automatically.",
+
+                "Press F5 or CTRL-R to refresh the page,<br />" +
+                "replacing your document with the server copy.",
+
+                "You can click this message to ignore and contnue."
+            ];
+            return toast(false, "box-shadow:0 1em 2em rgba(64,64,64,0.8);font-weight:normal",
+                36, "<p>" + msg.join('</p>\n<p>') + '</p>');
+        }
+
+        console.log('modpoll eq');
+    }
+
+    if (md_opt.modpoll_freq > 0)
+        this.periodic();
+
+    return this;
+}
+var modpoll = new Modpoll();
+
+
+window.onbeforeunload = function (e) {
+    if ((ebi("save").getAttribute('class') + '').indexOf('disabled') >= 0)
+        return; //nice (todo)
+
+    e.preventDefault(); //ff
+    e.returnValue = ''; //chrome
+};
+
+
 // save handler
 function save(e) {
     if (e) e.preventDefault();
-    var save_btn = document.getElementById("save"),
+    var save_btn = ebi("save"),
         save_cls = save_btn.getAttribute('class') + '';
 
     if (save_cls.indexOf('disabled') >= 0) {
-        toast('font-size:2em;color:#fc6;width:9em;', 'no changes');
+        toast(true, ";font-size:2em;color:#c90", 9, "no changes");
         return;
     }
 
@@ -245,6 +348,8 @@ function save(e) {
     xhr.onreadystatechange = save_cb;
     xhr.btn = save_btn;
     xhr.txt = txt;
+
+    modpoll.skip_one = true;  // skip one iteration while we save
     xhr.send(fd);
 }
 
@@ -338,23 +443,44 @@ function savechk_cb() {
     last_modified = this.lastmod;
     server_md = this.txt;
     draw_md();
-    toast('font-size:6em;font-family:serif;color:#cf6;width:4em;',
+    toast(true, ";font-size:6em;font-family:serif;color:#9b4", 4,
-        'OK✔️<span style="font-size:.2em;color:#999">' + this.ntry + '</span>');
+        'OK✔️<span style="font-size:.2em;color:#999;position:absolute">' + this.ntry + '</span>');
+
+    modpoll.disabled = false;
 }
 
-function toast(style, msg) {
+function toast(autoclose, style, width, msg) {
-    var ok = document.createElement('div');
+    var ok = ebi("toast");
-    style += 'font-weight:bold;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1';
+    if (ok)
+        ok.parentNode.removeChild(ok);
+
+    style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
+    ok = document.createElement('div');
+    ok.setAttribute('id', 'toast');
     ok.setAttribute('style', style);
     ok.innerHTML = msg;
-    var parent = document.getElementById('m');
+    var parent = ebi('m');
     document.documentElement.appendChild(ok);
+
+    var hide = function (delay) {
+        delay = delay || 0;
+
         setTimeout(function () {
             ok.style.opacity = 0;
-    }, 500);
+        }, delay);
+
         setTimeout(function () {
+            if (ok.parentNode)
                 ok.parentNode.removeChild(ok);
-    }, 750);
+        }, delay + 250);
+    }
+
+    ok.onclick = function () {
+        hide(0);
+    };
+
+    if (autoclose)
+        hide(500);
 }
 
 
@@ -427,6 +553,9 @@ function setsel(s) {
     dom_src.value = [s.pre, s.sel, s.post].join('');
     dom_src.setSelectionRange(s.car, s.cdr, dom_src.selectionDirection);
     dom_src.oninput();
+    // support chrome:
+    dom_src.blur();
+    dom_src.focus();
 }
 
 
@@ -500,7 +629,8 @@ function md_newline() {
     var s = linebounds(true),
         ln = s.md.substring(s.n1, s.n2),
         m1 = /^( *)([0-9]+)(\. +)/.exec(ln),
-        m2 = /^[ \t>+-]*(\* )?/.exec(ln);
+        m2 = /^[ \t>+-]*(\* )?/.exec(ln),
+        drop = dom_src.selectionEnd - dom_src.selectionStart;
 
     var pre = m2[0];
     if (m1 !== null)
@@ -512,7 +642,7 @@ function md_newline() {
 
     s.pre = s.md.substring(0, s.car) + '\n' + pre;
     s.sel = '';
-    s.post = s.md.substring(s.car);
+    s.post = s.md.substring(s.car + drop);
     s.car = s.cdr = s.pre.length;
     setsel(s);
     return false;
@@ -522,11 +652,17 @@ function md_newline() {
 // backspace
 function md_backspace() {
     var s = linebounds(true),
-        ln = s.md.substring(s.n1, s.n2),
+        o0 = dom_src.selectionStart,
-        m = /^[ \t>+-]*(\* )?([0-9]+\. +)?/.exec(ln);
+        left = s.md.slice(s.n1, o0),
+        m = /^[ \t>+-]*(\* )?([0-9]+\. +)?/.exec(left);
 
+    // if car is in whitespace area, do nothing
+    if (/^\s*$/.test(left))
+        return true;
+
+    // same if line is all-whitespace or non-markup
     var v = m[0].replace(/[^ ]/g, " ");
-    if (v === m[0] || v.length !== ln.length)
+    if (v === m[0] || v.length !== left.length)
         return true;
 
     s.pre = s.md.substring(0, s.n1) + v;
@@ -540,8 +676,8 @@ function md_backspace() {
 
 // paragraph jump
 function md_p_jump(down) {
-    var ofs = dom_src.selectionStart;
+    var txt = dom_src.value,
-    var txt = dom_src.value;
+        ofs = dom_src.selectionStart;
 
     if (down) {
         while (txt[ofs] == '\n' && --ofs > 0);
@@ -562,6 +698,224 @@ function md_p_jump(down) {
 }
 
 
+function reLastIndexOf(txt, ptn, end) {
+    var ofs = (typeof end !== 'undefined') ? end : txt.length;
+    end = ofs;
+    while (ofs >= 0) {
+        var sub = txt.slice(ofs, end);
+        if (ptn.test(sub))
+            return ofs;
+
+        ofs--;
+    }
+    return -1;
+}
+
+
+// table formatter
+function fmt_table(e) {
+    if (e) e.preventDefault();
+    //dom_tbox.setAttribute('class', '');
+
+    var txt = dom_src.value,
+        ofs = dom_src.selectionStart,
+        //o0 = txt.lastIndexOf('\n\n', ofs),
+        //o1 = txt.indexOf('\n\n', ofs);
+        o0 = reLastIndexOf(txt, /\n\s*\n/m, ofs),
+        o1 = txt.slice(ofs).search(/\n\s*\n|\n\s*$/m);
+    // note \s contains \n but its fine
+
+    if (o0 < 0)
+        o0 = 0;
+    else {
+        // seek past the hit
+        var m = /\n\s*\n/m.exec(txt.slice(o0));
+        o0 += m[0].length;
+    }
+
+    o1 = o1 < 0 ? txt.length : o1 + ofs;
+
+    var err = 'cannot format table due to ',
+        tab = txt.slice(o0, o1).split(/\s*\n/),
+        re_ind = /^\s*/,
+        ind = tab[1].match(re_ind)[0],
+        r0_ind = tab[0].slice(0, ind.length),
+        lpipe = tab[1].indexOf('|') < tab[1].indexOf('-'),
+        rpipe = tab[1].lastIndexOf('|') > tab[1].lastIndexOf('-'),
+        re_lpipe = lpipe ? /^\s*\|\s*/ : /^\s*/,
+        re_rpipe = rpipe ? /\s*\|\s*$/ : /\s*$/,
+        ncols;
+
+    // the second row defines the table,
+    // need to process that first
+    var tmp = tab[0];
+    tab[0] = tab[1];
+    tab[1] = tmp;
+
+    for (var a = 0; a < tab.length; a++) {
+        var row_name = (a == 1) ? 'header' : 'row#' + (a + 1);
+
+        var ind2 = tab[a].match(re_ind)[0];
+        if (ind != ind2 && a != 1)  // the table can be a list entry or something, ignore [0]
+            return alert(err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
+
+        var t = tab[a].slice(ind.length);
+        t = t.replace(re_lpipe, "");
+        t = t.replace(re_rpipe, "");
+        tab[a] = t.split(/\s*\|\s*/g);
+
+        if (a == 0)
+            ncols = tab[a].length;
+        else if (ncols < tab[a].length)
+            return alert(err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
+
+        // if row has less columns than row2, fill them in
+        while (tab[a].length < ncols)
+            tab[a].push('');
+    }
+
+    // aight now swap em back
+    tmp = tab[0];
+    tab[0] = tab[1];
+    tab[1] = tmp;
+
+    var re_align = /^ *(:?)-+(:?) *$/;
+    var align = [];
+    for (var col = 0; col < tab[1].length; col++) {
+        var m = tab[1][col].match(re_align);
+        if (!m)
+            return alert(err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
+
+        if (m[2]) {
+            if (m[1])
+                align.push('c');
+            else
+                align.push('r');
+        }
+        else
+            align.push('l');
+    }
+
+    var pad = [];
+    var tmax = 0;
+    for (var col = 0; col < ncols; col++) {
+        var max = 0;
+        for (var row = 0; row < tab.length; row++)
+            if (row != 1)
+                max = Math.max(max, tab[row][col].length);
+
+        var s = '';
+        for (var n = 0; n < max; n++)
+            s += ' ';
+
+        pad.push(s);
+        tmax = Math.max(max, tmax);
+    }
+
+    var dashes = '';
+    for (var a = 0; a < tmax; a++)
+        dashes += '-';
+
+    var ret = [];
+    for (var row = 0; row < tab.length; row++) {
+        var ln = [];
+        for (var col = 0; col < tab[row].length; col++) {
+            var p = pad[col];
+            var s = tab[row][col];
+
+            if (align[col] == 'l') {
+                s = (s + p).slice(0, p.length);
+            }
+            else if (align[col] == 'r') {
+                s = (p + s).slice(-p.length);
+            }
+            else {
+                var pt = p.length - s.length;
+                var pl = p.slice(0, Math.floor(pt / 2));
+                var pr = p.slice(0, pt - pl.length);
+                s = pl + s + pr;
+            }
+
+            if (row == 1) {
+                if (align[col] == 'l')
+                    s = dashes.slice(0, p.length);
+                else if (align[col] == 'r')
+                    s = dashes.slice(0, p.length - 1) + ':';
+                else
+                    s = ':' + dashes.slice(0, p.length - 2) + ':';
+            }
+            ln.push(s);
+        }
+        ret.push(ind + '| ' + ln.join(' | ') + ' |');
+    }
+
+    // restore any markup in the row0 gutter
+    ret[0] = r0_ind + ret[0].slice(ind.length);
+
+    ret = {
+        "pre": txt.slice(0, o0),
+        "sel": ret.join('\n'),
+        "post": txt.slice(o1),
+        "car": o0,
+        "cdr": o0
+    };
+    setsel(ret);
+}
+
+
+// show unicode
+function mark_uni(e) {
+    if (e) e.preventDefault();
+    dom_tbox.setAttribute('class', '');
+
+    var txt = dom_src.value,
+        ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g'),
+        mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
+
+    if (txt == mod) {
+        alert('no results;  no modifications were made');
+        return;
+    }
+    dom_src.value = mod;
+}
+
+
+// iterate unicode
+function iter_uni(e) {
+    if (e) e.preventDefault();
+
+    var txt = dom_src.value,
+        ofs = dom_src.selectionDirection == "forward" ? dom_src.selectionEnd : dom_src.selectionStart,
+        re = new RegExp('([^' + js_uni_whitelist + ']+)'),
+        m = re.exec(txt.slice(ofs));
+
+    if (!m) {
+        alert('no more hits from cursor onwards');
+        return;
+    }
+    ofs += m.index;
+
+    dom_src.setSelectionRange(ofs, ofs + m[0].length, "forward");
+    dom_src.oninput();
+    // support chrome:
+    dom_src.blur();
+    dom_src.focus();
+}
+
+
+// configure whitelist
+function cfg_uni(e) {
+    if (e) e.preventDefault();
+
+    var reply = prompt("unicode whitelist", esc_uni_whitelist);
+    if (reply === null)
+        return;
+
+    esc_uni_whitelist = reply;
+    js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+}
+
+
 // hotkeys / toolbar
 (function () {
     function keydown(ev) {
@@ -574,7 +928,7 @@ function md_p_jump(down) {
             return false;
         }
         if (ev.code == "Escape" || kc == 27) {
-            var d = document.getElementById('helpclose');
+            var d = ebi('helpclose');
             if (d)
                 d.click();
         }
@@ -609,6 +963,19 @@ function md_p_jump(down) {
             if (!ctrl && !ev.shiftKey && kc == 8) {
                 return md_backspace();
             }
+            if (ctrl && (ev.code == "KeyK")) {
+                fmt_table();
+                return false;
+            }
+            if (ctrl && (ev.code == "KeyU")) {
+                iter_uni();
+                return false;
+            }
+            if (ctrl && (ev.code == "KeyE")) {
+                dom_nsbs.click();
+                //fmt_table();
+                return false;
+            }
             var up = ev.code == "ArrowUp" || kc == 38;
             var dn = ev.code == "ArrowDown" || kc == 40;
             if (ctrl && (up || dn)) {
@@ -618,13 +985,22 @@ function md_p_jump(down) {
         }
     }
     document.onkeydown = keydown;
-    document.getElementById('save').onclick = save;
+    ebi('save').onclick = save;
 })();
 
 
-document.getElementById('help').onclick = function (e) {
+ebi('tools').onclick = function (e) {
     if (e) e.preventDefault();
-    var dom = document.getElementById('helpbox');
+    var is_open = dom_tbox.getAttribute('class') != 'open';
+    dom_tbox.setAttribute('class', is_open ? 'open' : '');
+};
+
+
+ebi('help').onclick = function (e) {
+    if (e) e.preventDefault();
+    dom_tbox.setAttribute('class', '');
+
+    var dom = ebi('helpbox');
     var dtxt = dom.getElementsByTagName('textarea');
     if (dtxt.length > 0) {
         convert_markdown(dtxt[0].value, dom);
@@ -632,12 +1008,18 @@ document.getElementById('help').onclick = function (e) {
     }
 
     dom.style.display = 'block';
-    document.getElementById('helpclose').onclick = function () {
+    ebi('helpclose').onclick = function () {
         dom.style.display = 'none';
     };
 };
 
 
+ebi('fmt_table').onclick = fmt_table;
+ebi('mark_uni').onclick = mark_uni;
+ebi('iter_uni').onclick = iter_uni;
+ebi('cfg_uni').onclick = cfg_uni;
+
+
 // blame steen
 action_stack = (function () {
     var hist = {
@@ -743,13 +1125,12 @@ action_stack = (function () {
         ref = newtxt;
         dbg('undos(%d) redos(%d)', hist.un.length, hist.re.length);
         if (hist.un.length > 0)
-            dbg(static(hist.un.slice(-1)[0]));
+            dbg(statify(hist.un.slice(-1)[0]));
         if (hist.re.length > 0)
-            dbg(static(hist.re.slice(-1)[0]));
+            dbg(statify(hist.re.slice(-1)[0]));
     }
 
     return {
-        push: push,
         undo: undo,
         redo: redo,
         push: schedule_push,
@@ -759,7 +1140,7 @@ action_stack = (function () {
 })();
 
 /*
-document.getElementById('help').onclick = function () {
+ebi('help').onclick = function () {
     var c1 = getComputedStyle(dom_src).cssText.split(';');
     var c2 = getComputedStyle(dom_ref).cssText.split(';');
     var max = Math.min(c1.length, c2.length);

copyparty/web/mde.css

@@ -160,8 +160,12 @@ h2 {
 .mdo ol>li {
 	margin: .7em 0;
 }
+strong {
+	color: #000;
+}
 p>em,
-li>em {
+li>em,
+td>em {
 	color: #c50;
 	padding: .1em;
 	border-bottom: .1em solid #bbb;
@@ -253,8 +257,12 @@ html.dark .mdo>ul,
 html.dark .mdo>ol {
     border-color: #555;
 }
+html.dark strong {
+    color: #fff;
+}
 html.dark p>em,
-html.dark li>em {
+html.dark li>em,
+html.dark td>em {
     color: #f94;
     border-color: #666;
 }

copyparty/web/mde.html

@@ -22,8 +22,12 @@
 	</div>
 	<script>
 
-var link_md_as_html = false;  // TODO (does nothing)
 var last_modified = {{ lastmod }};
+var md_opt = {
+	link_md_as_html: false,
+	allow_plugins: {{ md_plug }},
+	modpoll_freq: {{ md_chk_rate }}
+};
 
 var lightswitch = (function () {
 	var fun = function () {
@@ -39,6 +43,7 @@ var lightswitch = (function () {
 })();
 
 	</script>
+    <script src="/.cpr/util.js"></script>
 	<script src="/.cpr/deps/easymde.js"></script>
 	<script src="/.cpr/mde.js"></script>
 </body></html>

copyparty/web/mde.js

@@ -1,7 +1,9 @@
-var dom_wrap = document.getElementById('mw');
+"use strict";
-var dom_nav = document.getElementById('mn');
+
-var dom_doc = document.getElementById('m');
+var dom_wrap = ebi('mw');
-var dom_md = document.getElementById('mt');
+var dom_nav = ebi('mn');
+var dom_doc = ebi('m');
+var dom_md = ebi('mt');
 
 (function () {
     var n = document.location + '';
@@ -63,7 +65,7 @@ var mde = (function () {
     mde.codemirror.on("change", function () {
         md_changed(mde);
     });
-    var loader = document.getElementById('ml');
+    var loader = ebi('ml');
     loader.parentNode.removeChild(loader);
     return mde;
 })();
@@ -121,7 +123,7 @@ function save(mde) {
     fd.append("lastmod", (force ? -1 : last_modified));
     fd.append("body", txt);
 
-    var url = (document.location + '').split('?')[0] + '?raw';
+    var url = (document.location + '').split('?')[0];
     var xhr = new XMLHttpRequest();
     xhr.open('POST', url, true);
     xhr.responseType = 'text';
@@ -213,7 +215,7 @@ function save_chk() {
     var ok = document.createElement('div');
     ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
     ok.innerHTML = 'OK✔️';
-    var parent = document.getElementById('m');
+    var parent = ebi('m');
     document.documentElement.appendChild(ok);
     setTimeout(function () {
         ok.style.opacity = 0;

copyparty/web/up2k.js

@@ -1,61 +1,6 @@
 "use strict";
 
-// error handler for mobile devices
+window.onerror = vis_exh;
-function hcroak(msg) {
-    document.body.innerHTML = msg;
-    window.onerror = undefined;
-    throw 'fatal_err';
-}
-function croak(msg) {
-    document.body.textContent = msg;
-    window.onerror = undefined;
-    throw msg;
-}
-function esc(txt) {
-    return txt.replace(/[&"<>]/g, function (c) {
-        return {
-            '&': '&amp;',
-            '"': '&quot;',
-            '<': '&lt;',
-            '>': '&gt;'
-        }[c];
-    });
-}
-window.onerror = function (msg, url, lineNo, columnNo, error) {
-    window.onerror = undefined;
-    var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
-        esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
-
-    if (error) {
-        var find = ['desc', 'stack', 'trace'];
-        for (var a = 0; a < find.length; a++)
-            if (String(error[find[a]]) !== 'undefined')
-                html.push('<h2>' + find[a] + '</h2>' +
-                    esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
-    }
-    document.body.style.fontSize = '0.8em';
-    document.body.style.padding = '0 1em 1em 1em';
-    hcroak(html.join('\n'));
-};
-
-
-// https://stackoverflow.com/a/950146
-function import_js(url, cb) {
-    var head = document.head || document.getElementsByTagName('head')[0];
-    var script = document.createElement('script');
-    script.type = 'text/javascript';
-    script.src = url;
-
-    script.onreadystatechange = cb;
-    script.onload = cb;
-
-    head.appendChild(script);
-}
-
-
-function o(id) {
-    return document.getElementById(id);
-}
 
 
 (function () {
@@ -88,12 +33,12 @@ function goto(dest) {
     for (var a = obj.length - 1; a >= 0; a--)
         obj[a].classList.remove('act');
 
-    var obj = document.querySelectorAll('#ops>a');
+    obj = document.querySelectorAll('#ops>a');
     for (var a = obj.length - 1; a >= 0; a--)
         obj[a].classList.remove('act');
 
     if (dest) {
-        document.getElementById('op_' + dest).classList.add('act');
+        ebi('op_' + dest).classList.add('act');
         document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act');
 
         var fn = window['goto_' + dest];
@@ -121,7 +66,7 @@ function goto_up2k() {
         if (op !== null && op !== '.')
             goto(op);
     }
-    document.getElementById('ops').style.display = 'block';
+    ebi('ops').style.display = 'block';
 })();
 
 
@@ -150,21 +95,21 @@ function up2k_init(have_crypto) {
 
     // show modal message
     function showmodal(msg) {
-        o('u2notbtn').innerHTML = msg;
+        ebi('u2notbtn').innerHTML = msg;
-        o('u2btn').style.display = 'none';
+        ebi('u2btn').style.display = 'none';
-        o('u2notbtn').style.display = 'block';
+        ebi('u2notbtn').style.display = 'block';
-        o('u2conf').style.opacity = '0.5';
+        ebi('u2conf').style.opacity = '0.5';
     }
 
     // hide modal message
     function unmodal() {
-        o('u2notbtn').style.display = 'none';
+        ebi('u2notbtn').style.display = 'none';
-        o('u2btn').style.display = 'block';
+        ebi('u2btn').style.display = 'block';
-        o('u2conf').style.opacity = '1';
+        ebi('u2conf').style.opacity = '1';
-        o('u2notbtn').innerHTML = '';
+        ebi('u2notbtn').innerHTML = '';
     }
 
-    var post_url = o('op_bup').getElementsByTagName('form')[0].getAttribute('action');
+    var post_url = ebi('op_bup').getElementsByTagName('form')[0].getAttribute('action');
     if (post_url && post_url.charAt(post_url.length - 1) !== '/')
         post_url += '/';
 
@@ -181,25 +126,25 @@ function up2k_init(have_crypto) {
             import_js('/.cpr/deps/sha512.js', unmodal);
 
             if (is_https)
-                o('u2foot').innerHTML = shame + ' so <em>this</em> uploader will do like 500kB/s at best';
+                ebi('u2foot').innerHTML = shame + ' so <em>this</em> uploader will do like 500kB/s at best';
             else
-                o('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance';
+                ebi('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance';
+        }
     }
-    };
 
     // show uploader if the user only has write-access
-    if (!o('files'))
+    if (!ebi('files'))
         goto('up2k');
 
     // shows or clears an error message in the basic uploader ui
     function setmsg(msg) {
         if (msg !== undefined) {
-            o('u2err').setAttribute('class', 'err');
+            ebi('u2err').setAttribute('class', 'err');
-            o('u2err').innerHTML = msg;
+            ebi('u2err').innerHTML = msg;
         }
         else {
-            o('u2err').setAttribute('class', '');
+            ebi('u2err').setAttribute('class', '');
-            o('u2err').innerHTML = '';
+            ebi('u2err').innerHTML = '';
         }
     }
 
@@ -210,7 +155,7 @@ function up2k_init(have_crypto) {
     }
 
     // handle user intent to use the basic uploader instead
-    o('u2nope').onclick = function (e) {
+    ebi('u2nope').onclick = function (e) {
         e.preventDefault();
         setmsg('');
         goto('bup');
@@ -229,9 +174,9 @@ function up2k_init(have_crypto) {
     function cfg_get(name) {
         var val = localStorage.getItem(name);
         if (val === null)
-            return parseInt(o(name).value);
+            return parseInt(ebi(name).value);
 
-        o(name).value = val;
+        ebi(name).value = val;
         return val;
     }
 
@@ -242,7 +187,7 @@ function up2k_init(have_crypto) {
         else
             val = (val == '1');
 
-        o(name).checked = val;
+        ebi(name).checked = val;
         return val;
     }
 
@@ -250,7 +195,7 @@ function up2k_init(have_crypto) {
         localStorage.setItem(
             name, val ? '1' : '0');
 
-        o(name).checked = val;
+        ebi(name).checked = val;
         return val;
     }
 
@@ -284,9 +229,9 @@ function up2k_init(have_crypto) {
         return un2k("this is the basic uploader; up2k needs at least<br />chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1");
 
     function nav() {
-        o('file' + fdom_ctr).click();
+        ebi('file' + fdom_ctr).click();
     }
-    o('u2btn').addEventListener('click', nav, false);
+    ebi('u2btn').addEventListener('click', nav, false);
 
     function ondrag(ev) {
         ev.stopPropagation();
@@ -294,8 +239,8 @@ function up2k_init(have_crypto) {
         ev.dataTransfer.dropEffect = 'copy';
         ev.dataTransfer.effectAllowed = 'copy';
     }
-    o('u2btn').addEventListener('dragover', ondrag, false);
+    ebi('u2btn').addEventListener('dragover', ondrag, false);
-    o('u2btn').addEventListener('dragenter', ondrag, false);
+    ebi('u2btn').addEventListener('dragenter', ondrag, false);
 
     function gotfile(ev) {
         ev.stopPropagation();
@@ -357,7 +302,7 @@ function up2k_init(have_crypto) {
             var tr = document.createElement('tr');
             tr.innerHTML = '<td id="f{0}n"></td><td id="f{0}t">hashing</td><td id="f{0}p" class="prog"></td>'.format(st.files.length);
             tr.getElementsByTagName('td')[0].textContent = entry.name;
-            o('u2tab').appendChild(tr);
+            ebi('u2tab').appendChild(tr);
 
             st.files.push(entry);
             st.todo.hash.push(entry);
@@ -374,14 +319,14 @@ function up2k_init(have_crypto) {
             alert(msg);
         }
     }
-    o('u2btn').addEventListener('drop', gotfile, false);
+    ebi('u2btn').addEventListener('drop', gotfile, false);
 
     function more_one_file() {
         fdom_ctr++;
         var elm = document.createElement('div')
         elm.innerHTML = '<input id="file{0}" type="file" name="file{0}[]" multiple="multiple" />'.format(fdom_ctr);
-        o('u2form').appendChild(elm);
+        ebi('u2form').appendChild(elm);
-        o('file' + fdom_ctr).addEventListener('change', gotfile, false);
+        ebi('file' + fdom_ctr).addEventListener('change', gotfile, false);
     }
     more_one_file();
 
@@ -451,17 +396,6 @@ function up2k_init(have_crypto) {
     ///   hashing
     //
 
-    // https://gist.github.com/jonleighton/958841
-    function buf2b64_maybe_fucky(buffer) {
-        var ret = '';
-        var view = new DataView(buffer);
-        for (var i = 0; i < view.byteLength; i++) {
-            ret += String.fromCharCode(view.getUint8(i));
-        }
-        return window.btoa(ret).replace(
-            /\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
-    }
-
     // https://gist.github.com/jonleighton/958841
     function buf2b64(arrayBuffer) {
         var base64 = '';
@@ -502,20 +436,6 @@ function up2k_init(have_crypto) {
         return base64;
     }
 
-    // https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
-    function buf2hex(buffer) {
-        var hexCodes = [];
-        var view = new DataView(buffer);
-        for (var i = 0; i < view.byteLength; i += 4) {
-            var value = view.getUint32(i) // 4 bytes per iter
-            var stringValue = value.toString(16) // doesn't pad
-            var padding = '00000000'
-            var paddedValue = (padding + stringValue).slice(-padding.length)
-            hexCodes.push(paddedValue);
-        }
-        return hexCodes.join("");
-    }
-
     function get_chunksize(filesize) {
         var chunksize = 1024 * 1024;
         var stepsize = 512 * 1024;
@@ -602,7 +522,7 @@ function up2k_init(have_crypto) {
             pb_html += '<div id="f{0}p{1}" style="width:{2}%"><div></div></div>'.format(
                 t.n, a, pb_perc);
 
-        o('f{0}p'.format(t.n)).innerHTML = pb_html;
+        ebi('f{0}p'.format(t.n)).innerHTML = pb_html;
 
         var reader = new FileReader();
 
@@ -677,7 +597,7 @@ function up2k_init(have_crypto) {
                 alert('{0} ms, {1} MB/s\n'.format(t.t2 - t.t1, spd.toFixed(3)) + t.hash.join('\n'));
             }
 
-            o('f{0}t'.format(t.n)).innerHTML = 'connecting';
+            ebi('f{0}t'.format(t.n)).innerHTML = 'connecting';
             st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
             st.todo.handshake.push(t);
         };
@@ -706,7 +626,7 @@ function up2k_init(have_crypto) {
                 if (response.name !== t.name) {
                     // file exists; server renamed us
                     t.name = response.name;
-                    o('f{0}n'.format(t.n)).textContent = t.name;
+                    ebi('f{0}n'.format(t.n)).textContent = t.name;
                 }
 
                 t.postlist = [];
@@ -736,13 +656,13 @@ function up2k_init(have_crypto) {
                     msg = 'uploading';
                     done = false;
                 }
-                o('f{0}t'.format(t.n)).innerHTML = msg;
+                ebi('f{0}t'.format(t.n)).innerHTML = msg;
                 st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
 
                 if (done) {
                     var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
                     var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
-                    o('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
+                    ebi('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2));
                 }
                 tasker();
@@ -803,7 +723,7 @@ function up2k_init(have_crypto) {
                     t.postlist.splice(t.postlist.indexOf(npart), 1);
                     if (t.postlist.length == 0) {
                         t.t3 = new Date().getTime();
-                        o('f{0}t'.format(t.n)).innerHTML = 'verifying';
+                        ebi('f{0}t'.format(t.n)).innerHTML = 'verifying';
                         st.todo.handshake.push(t);
                     }
                     tasker();
@@ -834,7 +754,7 @@ function up2k_init(have_crypto) {
     //
 
     function prog(nfile, nchunk, color, percent) {
-        var n1 = o('f{0}p{1}'.format(nfile, nchunk));
+        var n1 = ebi('f{0}p{1}'.format(nfile, nchunk));
         var n2 = n1.getElementsByTagName('div')[0];
         if (percent === undefined) {
             n1.style.background = color;
@@ -857,7 +777,7 @@ function up2k_init(have_crypto) {
             dir.preventDefault();
         } catch (ex) { }
 
-        var obj = o('nthread');
+        var obj = ebi('nthread');
         if (dir.target) {
             obj.style.background = '#922';
             var v = Math.floor(parseInt(obj.value));
@@ -892,19 +812,19 @@ function up2k_init(have_crypto) {
         this.click();
     }
 
-    o('nthread_add').onclick = function (ev) {
+    ebi('nthread_add').onclick = function (ev) {
         ev.preventDefault();
         bumpthread(1);
     };
-    o('nthread_sub').onclick = function (ev) {
+    ebi('nthread_sub').onclick = function (ev) {
         ev.preventDefault();
         bumpthread(-1);
     };
 
-    o('nthread').addEventListener('input', bumpthread, false);
+    ebi('nthread').addEventListener('input', bumpthread, false);
-    o('multitask').addEventListener('click', tgl_multitask, false);
+    ebi('multitask').addEventListener('click', tgl_multitask, false);
 
-    var nodes = o('u2conf').getElementsByTagName('a');
+    var nodes = ebi('u2conf').getElementsByTagName('a');
     for (var a = nodes.length - 1; a >= 0; a--)
         nodes[a].addEventListener('touchend', nop, false);
 

copyparty/web/util.js

@@ -0,0 +1,109 @@
+"use strict";
+
+// error handler for mobile devices
+function hcroak(msg) {
+    document.body.innerHTML = msg;
+    window.onerror = undefined;
+    throw 'fatal_err';
+}
+function croak(msg) {
+    document.body.textContent = msg;
+    window.onerror = undefined;
+    throw msg;
+}
+function esc(txt) {
+    return txt.replace(/[&"<>]/g, function (c) {
+        return {
+            '&': '&amp;',
+            '"': '&quot;',
+            '<': '&lt;',
+            '>': '&gt;'
+        }[c];
+    });
+}
+function vis_exh(msg, url, lineNo, columnNo, error) {
+    window.onerror = undefined;
+    var html = ['<h1>you hit a bug!</h1><p>please screenshot this error and send me a copy arigathanks gozaimuch (ed/irc.rizon.net or ed#2644)</p><p>',
+        esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
+
+    if (error) {
+        var find = ['desc', 'stack', 'trace'];
+        for (var a = 0; a < find.length; a++)
+            if (String(error[find[a]]) !== 'undefined')
+                html.push('<h2>' + find[a] + '</h2>' +
+                    esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
+    }
+    document.body.style.fontSize = '0.8em';
+    document.body.style.padding = '0 1em 1em 1em';
+    hcroak(html.join('\n'));
+}
+
+
+function ebi(id) {
+    return document.getElementById(id);
+}
+
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
+if (!String.prototype.endsWith) {
+    String.prototype.endsWith = function (search, this_len) {
+        if (this_len === undefined || this_len > this.length) {
+            this_len = this.length;
+        }
+        return this.substring(this_len - search.length, this_len) === search;
+    };
+}
+if (!String.startsWith) {
+    String.prototype.startsWith = function (s, i) {
+        i = i > 0 ? i | 0 : 0;
+        return this.substring(i, i + s.length) === s;
+    };
+}
+
+
+// https://stackoverflow.com/a/950146
+function import_js(url, cb) {
+    var head = document.head || document.getElementsByTagName('head')[0];
+    var script = document.createElement('script');
+    script.type = 'text/javascript';
+    script.src = url;
+
+    script.onreadystatechange = cb;
+    script.onload = cb;
+
+    head.appendChild(script);
+}
+
+
+function sortTable(table, col) {
+    var tb = table.tBodies[0], // use `<tbody>` to ignore `<thead>` and `<tfoot>` rows
+        th = table.tHead.rows[0].cells,
+        tr = Array.prototype.slice.call(tb.rows, 0),
+        i, reverse = th[col].className == 'sort1' ? -1 : 1;
+    for (var a = 0, thl = th.length; a < thl; a++)
+        th[a].className = '';
+    th[col].className = 'sort' + reverse;
+    var stype = th[col].getAttribute('sort');
+    tr = tr.sort(function (a, b) {
+        var v1 = a.cells[col].textContent.trim();
+        var v2 = b.cells[col].textContent.trim();
+        if (stype == 'int') {
+            v1 = parseInt(v1.replace(/,/g, ''));
+            v2 = parseInt(v2.replace(/,/g, ''));
+            return reverse * (v1 - v2);
+        }
+        return reverse * (v1.localeCompare(v2));
+    });
+    for (i = 0; i < tr.length; ++i) tb.appendChild(tr[i]);
+}
+function makeSortable(table) {
+    var th = table.tHead, i;
+    th && (th = th.rows[0]) && (th = th.cells);
+    if (th) i = th.length;
+    else return; // if no `<thead>` then do nothing
+    while (--i >= 0) (function (i) {
+        th[i].onclick = function () {
+            sortTable(table, i);
+        };
+    }(i));
+}

docs/rclone.md

@@ -0,0 +1,62 @@
+# using rclone to mount a remote copyparty server as a local filesystem
+
+speed estimates with server and client on the same win10 machine:
+* `1070 MiB/s` with rclone as both server and client
+* `570 MiB/s` with rclone-client and `copyparty -ed -j16` as server
+* `220 MiB/s` with rclone-client and `copyparty -ed` as server
+* `100 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+
+when server is on another machine (1gbit LAN),
+* `75 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+* `92 MiB/s` with rclone-client and `copyparty -ed` as server
+* `103 MiB/s` (connection max) with `copyparty -ed -j16` and all the others
+
+
+# creating the config file
+
+if you want to use password auth, add `headers = Cookie,cppwd=fgsfds` below
+
+
+### on windows clients:
+```
+(
+echo [cpp]
+echo type = http
+echo url = http://127.0.0.1:3923/
+) > %userprofile%\.config\rclone\rclone.conf
+```
+
+also install the windows dependencies: [winfsp](https://github.com/billziss-gh/winfsp/releases/latest)
+
+
+### on unix clients:
+```
+cat > ~/.config/rclone/rclone.conf <<'EOF'
+[cpp]
+type = http
+url = http://127.0.0.1:3923/
+EOF
+```
+
+
+# mounting the copyparty server locally
+```
+rclone.exe mount --vfs-cache-max-age 5s --attr-timeout 5s --dir-cache-time 5s cpp: Z:
+```
+
+
+# use rclone as server too, replacing copyparty
+
+feels out of place but is too good not to mention
+
+```
+rclone.exe serve http --read-only .
+```
+
+* `webdav` gives write-access but `http` is twice as fast
+* `ftp` is buggy, avoid
+
+
+# bugs
+
+* rclone-client throws an exception if you try to read an empty file (should return zero bytes)

scripts/copyparty-repack.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+set -e
+
+# -- download latest copyparty (source.tgz and sfx),
+# -- build minimal sfx versions,
+# -- create a .tar.gz bundle
+#
+# convenient for deploying updates to inconvenient locations
+#  (and those are usually linux so bash is good inaff)
+#   (but that said this even has macos support)
+#
+# bundle will look like:
+# -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty
+# -rw-r--r--  0 ed ed  491318 Nov 19 00:40 copyparty-extras/copyparty-0.5.4.tar.gz
+# -rwxr-xr-x  0 ed ed   30254 Nov 17 23:58 copyparty-extras/copyparty-fuse.py
+# -rwxr-xr-x  0 ed ed  481403 Nov 19 00:40 copyparty-extras/sfx-full/copyparty-sfx.sh
+# -rwxr-xr-x  0 ed ed  506043 Nov 19 00:40 copyparty-extras/sfx-full/copyparty-sfx.py
+# -rwxr-xr-x  0 ed ed  167699 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.sh
+# -rwxr-xr-x  0 ed ed  183808 Nov 19 00:43 copyparty-extras/sfx-lite/copyparty-sfx.py
+
+
+command -v gtar && tar() { gtar "$@"; }
+command -v gsed && sed() { gsed "$@"; }
+td="$(mktemp -d)"
+od="$(pwd)"
+cd "$td"
+pwd
+
+
+# debug: if cache exists, use that instead of bothering github
+cache="$od/.copyparty-repack.cache"
+[ -e "$cache" ] &&
+	tar -xvf "$cache" ||
+{
+	# get download links from github
+	curl https://api.github.com/repos/9001/copyparty/releases/latest |
+	(
+		# prefer jq if available
+		jq -r '.assets[]|select(.name|test("-sfx|tar.gz")).browser_download_url' ||
+
+		# fallback to awk (sorry)
+		awk -F\" '/"browser_download_url".*(\.tar\.gz|-sfx\.)/ {print$4}'
+	) |
+	tee /dev/stderr |
+	tr -d '\r' | tr '\n' '\0' | xargs -0 curl -L --remote-name-all
+
+	# debug: create cache
+	#tar -czvf "$cache" *
+}
+
+
+# move src into copyparty-extras/,
+# move sfx into copyparty-extras/sfx-full/
+mkdir -p copyparty-extras/sfx-{full,lite}
+mv copyparty-sfx.* copyparty-extras/sfx-full/
+mv copyparty-*.tar.gz copyparty-extras/
+
+
+# unpack the source code
+( cd copyparty-extras/
+tar -xvf *.tar.gz
+)
+
+
+# fix permissions
+chmod 755 \
+  copyparty-extras/sfx-full/* \
+  copyparty-extras/copyparty-*/{scripts,bin}/*
+
+
+# extract and repack the sfx with less features enabled
+( cd copyparty-extras/sfx-full/
+./copyparty-sfx.py -h
+cd ../copyparty-*/
+./scripts/make-sfx.sh re no-ogv no-cm
+)
+
+
+# put new sfx into copyparty-extras/sfx-lite/,
+# fuse client into copyparty-extras/,
+# copy lite-sfx.py to ./copyparty,
+# delete extracted source code
+( cd copyparty-extras/
+mv copyparty-*/dist/* sfx-lite/
+mv copyparty-*/bin/copyparty-fuse.py .
+cp -pv sfx-lite/copyparty-sfx.py ../copyparty
+rm -rf copyparty-{0..9}*.*.*{0..9}
+)
+
+
+# and include the repacker itself too
+cp -pv "$od/$0" copyparty-extras/ 
+
+
+# create the bundle
+fn=copyparty-$(date +%Y-%m%d-%H%M%S).tgz
+tar -czvf "$od/$fn" *
+cd "$od"
+rm -rf "$td"
+
+
+echo
+echo "done, here's your bundle:"
+ls -al "$fn"

scripts/make-sfx.sh

@@ -94,8 +94,39 @@ cd sfx
 	rm -f ../tar
 }
 
-ver="$(awk '/^VERSION *= \(/ {
+ver=
-	gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < ../copyparty/__version__.py)"
+git describe --tags >/dev/null 2>/dev/null && {
+	git_ver="$(git describe --tags)";  # v0.5.5-2-gb164aa0
+	ver="$(printf '%s\n' "$git_ver" | sed -r 's/^v//; s/-g?/./g')";
+	t_ver=
+
+	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+$' && {
+		# short format (exact version number)
+		t_ver="$(printf '%s\n' "$ver" | sed -r 's/\./, /g')";
+	}
+
+	printf '%s\n' "$git_ver" | grep -qE '^v[0-9\.]+-[0-9]+-g[0-9a-f]+$' && {
+		# long format (unreleased commit)
+		t_ver="$(printf '%s\n' "$ver" | sed -r 's/\./, /g; s/(.*) (.*)/\1 "\2"/')"
+	}
+
+	[ -z "$t_ver" ] && {
+		printf 'unexpected git version format: [%s]\n' "$git_ver"
+		exit 1
+	}
+
+	dt="$(git log -1 --format=%cd --date=format:'%Y, %m, %d')"
+	printf 'git %3s: \033[36m%s\033[0m\n' ver "$ver" dt "$dt"
+	sed -ri '
+		s/^(VERSION =)(.*)/#\1\2\n\1 ('"$t_ver"')/;
+		s/^(S_VERSION =)(.*)/#\1\2\n\1 "'"$ver"'"/;
+		s/^(BUILD_DT =)(.*)/#\1\2\n\1 ('"$dt"')/;
+	' copyparty/__version__.py
+}
+
+[ -z "$ver" ] && 
+	ver="$(awk '/^VERSION *= \(/ {
+		gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < copyparty/__version__.py)"
 
 ts=$(date -u +%s)
 hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)

scripts/sfx.py

@@ -2,7 +2,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re, os, sys, stat, time, shutil, tarfile, hashlib, platform, tempfile
+import re, os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
 import subprocess as sp
 
 """
@@ -29,6 +29,7 @@ STAMP = None
 PY2 = sys.version_info[0] == 2
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
+cpp = None
 
 
 def eprint(*args, **kwargs):
@@ -191,6 +192,16 @@ def makesfx(tar_src, ver, ts):
 # skip 0
 
 
+def u8(gen):
+    try:
+        for s in gen:
+            yield s.decode("utf-8", "ignore")
+    except:
+        yield s
+        for s in gen:
+            yield s
+
+
 def get_py_win(ret):
     tops = []
     p = str(os.getenv("LocalAppdata"))
@@ -216,11 +227,11 @@ def get_py_win(ret):
     # $WIRESHARK_SLOGAN
     for top in tops:
         try:
-            for name1 in sorted(os.listdir(top), reverse=True):
+            for name1 in u8(sorted(os.listdir(top), reverse=True)):
                 if name1.lower().startswith("python"):
                     path1 = os.path.join(top, name1)
                     try:
-                        for name2 in os.listdir(path1):
+                        for name2 in u8(os.listdir(path1)):
                             if name2.lower() == "python.exe":
                                 path2 = os.path.join(path1, name2)
                                 ret[path2.lower()] = path2
@@ -237,7 +248,7 @@ def get_py_nix(ret):
             next
 
         try:
-            for fn in os.listdir(bindir):
+            for fn in u8(os.listdir(bindir)):
                 if ptn.match(fn):
                     fn = os.path.join(bindir, fn)
                     ret[fn.lower()] = fn
@@ -295,17 +306,19 @@ def hashfile(fn):
 def unpack():
     """unpacks the tar yielded by `data`"""
     name = "pe-copyparty"
+    tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
     final = os.path.join(top, name)
     mine = os.path.join(top, withpid)
     tar = os.path.join(mine, "tar")
-    tag_mine = os.path.join(mine, "v" + str(STAMP))
-    tag_final = os.path.join(final, "v" + str(STAMP))
 
-    if os.path.exists(tag_final):
+    try:
+        if tag in os.listdir(final):
             msg("found early")
             return final
+    except:
+        pass
 
     nwrite = 0
     os.mkdir(mine)
@@ -328,12 +341,15 @@ def unpack():
 
     os.remove(tar)
 
-    with open(tag_mine, "wb") as f:
+    with open(os.path.join(mine, tag), "wb") as f:
         f.write(b"h\n")
 
-    if os.path.exists(tag_final):
+    try:
+        if tag in os.listdir(final):
             msg("found late")
             return final
+    except:
+        pass
 
     try:
         if os.path.islink(final):
@@ -352,7 +368,7 @@ def unpack():
             msg("reloc fail,", mine)
             return mine
 
-    for fn in os.listdir(top):
+    for fn in u8(os.listdir(top)):
         if fn.startswith(name) and fn not in [name, withpid]:
             try:
                 old = os.path.join(top, fn)
@@ -418,10 +434,15 @@ def get_payload():
 def confirm():
     msg()
     msg("*** hit enter to exit ***")
+    try:
         raw_input() if PY2 else input()
+    except:
+        pass
 
 
 def run(tmp, py):
+    global cpp
+
     msg("OK")
     msg("will use:", py)
     msg("bound to:", tmp)
@@ -437,8 +458,11 @@ def run(tmp, py):
         pass
 
     fp_py = os.path.join(tmp, "py")
+    try:
         with open(fp_py, "wb") as f:
             f.write(py.encode("utf-8") + b"\n")
+    except:
+        pass
 
     # avoid loading ./copyparty.py
     cmd = [
@@ -450,16 +474,21 @@ def run(tmp, py):
     ] + list(sys.argv[1:])
 
     msg("\n", cmd, "\n")
-    p = sp.Popen(str(x) for x in cmd)
+    cpp = sp.Popen(str(x) for x in cmd)
     try:
-        p.wait()
+        cpp.wait()
     except:
-        p.wait()
+        cpp.wait()
 
-    if p.returncode != 0:
+    if cpp.returncode != 0:
         confirm()
 
-    sys.exit(p.returncode)
+    sys.exit(cpp.returncode)
+
+
+def bye(sig, frame):
+    if cpp is not None:
+        cpp.terminate()
 
 
 def main():
@@ -494,6 +523,8 @@ def main():
 
     # skip 0
 
+    signal.signal(signal.SIGTERM, bye)
+
     tmp = unpack()
     fp_py = os.path.join(tmp, "py")
     if os.path.exists(fp_py):

scripts/sfx.sh

@@ -32,8 +32,12 @@ dir="$(
 
 # detect available pythons
 (IFS=:; for d in $PATH; do
-	printf '%s\n' "$d"/python* "$d"/pypy* | tac;
+	printf '%s\n' "$d"/python* "$d"/pypy*;
-done) | grep -E '(python|pypy)[0-9\.-]*$' > $dir/pys || true
+done) |
+(sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) |
+(sort -nr || cat) |
+(sed -E 's/([^ ]*) (.*)/\2\1/' || cat) |
+grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
 
 # see if we made a choice before
 [ -z "$pybin" ] && pybin="$(cat $dir/py 2>/dev/null || true)"

scripts/speedtest-fs.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import stat
+import time
+import signal
+import traceback
+import threading
+from queue import Queue
+
+
+"""speedtest-fs: filesystem performance estimate"""
+__author__ = "ed <copyparty@ocv.me>"
+__copyright__ = 2020
+__license__ = "MIT"
+__url__ = "https://github.com/9001/copyparty/"
+
+
+def get_spd(nbyte, nsec):
+    if not nsec:
+        return "0.000 MB   0.000 sec   0.000 MB/s"
+
+    mb = nbyte / (1024 * 1024.0)
+    spd = mb / nsec
+
+    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+
+
+class Inf(object):
+    def __init__(self, t0):
+        self.msgs = []
+        self.errors = []
+        self.reports = []
+        self.mtx_msgs = threading.Lock()
+        self.mtx_reports = threading.Lock()
+
+        self.n_byte = 0
+        self.n_sec = 0
+        self.n_done = 0
+        self.t0 = t0
+
+        thr = threading.Thread(target=self.print_msgs)
+        thr.daemon = True
+        thr.start()
+
+    def msg(self, fn, n_read):
+        with self.mtx_msgs:
+            self.msgs.append(f"{fn} {n_read}")
+
+    def err(self, fn):
+        with self.mtx_reports:
+            self.errors.append(f"{fn}\n{traceback.format_exc()}")
+
+    def print_msgs(self):
+        while True:
+            time.sleep(0.02)
+            with self.mtx_msgs:
+                msgs = self.msgs
+                self.msgs = []
+
+            if not msgs:
+                continue
+
+            msgs = msgs[-64:]
+            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            print("\n".join(msgs))
+
+    def report(self, fn, n_byte, n_sec):
+        with self.mtx_reports:
+            self.reports.append([n_byte, n_sec, fn])
+            self.n_byte += n_byte
+            self.n_sec += n_sec
+
+    def done(self):
+        with self.mtx_reports:
+            self.n_done += 1
+
+
+def get_files(dir_path):
+    for fn in os.listdir(dir_path):
+        fn = os.path.join(dir_path, fn)
+        st = os.stat(fn).st_mode
+
+        if stat.S_ISDIR(st):
+            yield from get_files(fn)
+
+        if stat.S_ISREG(st):
+            yield fn
+
+
+def worker(q, inf, read_sz):
+    while True:
+        fn = q.get()
+        if not fn:
+            break
+
+        n_read = 0
+        try:
+            t0 = time.time()
+            with open(fn, "rb") as f:
+                while True:
+                    buf = f.read(read_sz)
+                    if not buf:
+                        break
+
+                    n_read += len(buf)
+                    inf.msg(fn, n_read)
+
+            inf.report(fn, n_read, time.time() - t0)
+        except:
+            inf.err(fn)
+
+    inf.done()
+
+
+def sighandler(signo, frame):
+    os._exit(0)
+
+
+def main():
+    signal.signal(signal.SIGINT, sighandler)
+
+    root = "."
+    if len(sys.argv) > 1:
+        root = sys.argv[1]
+
+    t0 = time.time()
+    q = Queue(256)
+    inf = Inf(t0)
+
+    num_threads = 8
+    read_sz = 32 * 1024
+    for _ in range(num_threads):
+        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr.daemon = True
+        thr.start()
+
+    for fn in get_files(root):
+        q.put(fn)
+
+    for _ in range(num_threads):
+        q.put(None)
+
+    while inf.n_done < num_threads:
+        time.sleep(0.1)
+
+    t2 = time.time()
+    print("\n")
+
+    log = inf.reports
+    log.sort()
+    for nbyte, nsec, fn in log[-64:]:
+        print(f"{get_spd(nbyte, nsec)}   {fn}")
+
+    print()
+    print("\n".join(inf.errors))
+
+    print(get_spd(inf.n_byte, t2 - t0))
+
+
+if __name__ == "__main__":
+    main()
+

srv/extend.md

@@ -0,0 +1,141 @@
+# hi
+this showcases my worst idea yet; *extending markdown with inline javascript*
+
+due to obvious reasons it's disabled by default, and can be enabled with `-emp`
+
+the examples are by no means correct, they're as much of a joke as this feature itself
+
+
+### sub-header
+nothing special about this one
+
+
+## except/
+this one becomes a hyperlink to ./except/ thanks to
+* the `copyparty_pre` plugin at the end of this file
+* which is invoked as a markdown filter every time the document is modified
+* which looks for headers ending with a `/` and erwrites all headers below that
+
+it is a passthrough to the markdown extension api, see https://marked.js.org/using_pro
+
+in addition to the markdown extension functions, `ctor` will be called on document init
+
+
+### these/
+and this one becomes ./except/these/
+
+
+#### ones.md
+finally ./except/these/ones.md
+
+
+### also-this.md
+whic hshoud be ./except/also-this.md
+
+
+
+
+# ok
+now for another extension type, `copyparty_post` which is called to manipulate the generated dom instead
+
+`copyparty_post` can have the following functions, all optional
+* `ctor` is called on document init
+* `render` is called when the dom is done but still in-memory
+* `render2` is called with the live browser dom as-displayed
+
+## post example
+
+the values in the `ex:` columns are linkified to `example.com/$value`
+
+| ex:foo       | bar      | ex:baz |
+| ------------ | -------- | ------ |
+| asdf         | nice     | fgsfds |
+| more one row | hi hello | aaa    |
+
+and the table can be sorted by clicking the headers
+
+the difference is that with `copyparty_pre` you'll probably break various copyparty features but if you use `copyparty_post` then future copyparty versions will probably break you
+
+
+
+
+# heres the plugins
+if there is anything below ths line in the preview then the plugin feature is disabled (good)
+
+
+
+
+```copyparty_pre
+ctor() {
+    md_plug['h'] = {
+        on: false,
+        lv: -1,
+        path: []
+    }
+},
+walkTokens(token) {
+    if (token.type == 'heading') {
+        var h = md_plug['h'],
+            is_dir = token.text.endsWith('/');
+        
+        if (h.lv >= token.depth) {
+            h.on = false;
+        }
+        if (!h.on && is_dir) {
+            h.on = true;
+            h.lv = token.depth;
+            h.path = [token.text];
+        }
+        else if (h.on && h.lv < token.depth) {
+            h.path = h.path.slice(0, token.depth - h.lv);
+            h.path.push(token.text);
+        }
+        if (!h.on)
+            return false;
+
+        var path = h.path.join('');
+        var emoji = is_dir ? '📂' : '📜';
+        token.tokens[0].text = '<a href="' + path + '">' + emoji + ' ' + path + '</a>';
+    }
+    if (token.type == 'paragraph') {
+        //console.log(JSON.parse(JSON.stringify(token.tokens)));
+        for (var a = 0; a < token.tokens.length; a++) {
+            var t = token.tokens[a];
+            if (t.type == 'text' || t.type == 'strong' || t.type == 'em') {
+                var ret = '', text = t.text;
+                for (var b = 0; b < text.length; b++)
+                    ret += (Math.random() > 0.5) ? text[b] : text[b].toUpperCase();
+                
+                t.text = ret;
+            }
+        }
+    }
+    return true;
+}
+```
+
+
+
+```copyparty_post
+render(dom) {
+    var ths = dom.querySelectorAll('th');
+    for (var a = 0; a < ths.length; a++) {
+        var th = ths[a];
+        if (th.textContent.indexOf('ex:') === 0) {
+            th.textContent = th.textContent.slice(3);
+            var nrow = 0;
+            while ((th = th.previousSibling) != null)
+                nrow++;
+            
+            var trs = ths[a].parentNode.parentNode.parentNode.querySelectorAll('tr');
+            for (var b = 1; b < trs.length; b++) {
+                var td = trs[b].childNodes[nrow];
+                td.innerHTML = '<a href="//example.com/' + td.innerHTML + '">' + td.innerHTML + '</a>';
+            }
+        }
+    }
+},
+render2(dom) {
+    window.makeSortable(dom.getElementsByTagName('table')[0]);
+}
+```

srv/test.md

@@ -1,5 +1,16 @@
 ### hello world
 
+* qwe
+* asd
+  * zxc
+  * 573
+    * one
+    * two
+    
+  * |||
+    |--|--|
+    |listed|table|
+
 ```
 [72....................................................................]
 [80............................................................................]
@@ -21,6 +32,8 @@
 l[i]=1I;(){}o0O</> var foo = "$(`bar`)"; a's'd
 ```
 
+🔍🌽.📕.🍙🔎
+
 [](#s1)
 [s1](#s1)
 [#s1](#s1)
@@ -121,6 +134,15 @@ a newline toplevel
 | a table | on the right |
 | second row | foo bar |
 
+||
+--|:-:|-:
+a table | big text in this | aaakbfddd
+second row | centred | bbb
+
+||
+--|--|--
+foo
+
 * list entry
 * [x] yes
 * [ ] no
@@ -209,3 +231,7 @@ unrelated neat stuff:
 awk '/./ {printf "%s %d\n", $0, NR; next} 1' <test.md >ln.md
 gawk '{print gensub(/([a-zA-Z\.])/,NR" \\1","1")}' <test.md >ln.md
 ```
+
+a|b|c
+--|--|--
+foo

tests/test_vfs.py

@@ -3,6 +3,7 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import time
 import json
 import shutil
 import unittest
@@ -59,8 +60,15 @@ class TestVFS(unittest.TestCase):
 
         if os.path.exists("/Volumes"):
             devname, _ = self.chkcmd("hdiutil", "attach", "-nomount", "ram://8192")
+            for _ in range(10):
+                try:
                     _, _ = self.chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
                     return "/Volumes/cptd"
+                except:
+                    print('lol macos')
+                    time.sleep(0.25)
+            
+            raise Exception("ramdisk creation failed")
 
         raise Exception("TODO support windows")