v0.11.13

.vscode/launch.json

@@ -16,12 +16,9 @@
                 "-e2ts",
                 "-mtp",
                 ".bpm=f,bin/mtag/audio-bpm.py",
-                "-a",
-                "ed:wark",
-                "-v",
-                "srv::r:aed:cnodupe",
-                "-v",
-                "dist:dist:r"
+                "-aed:wark",
+                "-vsrv::r:aed:cnodupe",
+                "-vdist:dist:r"
             ]
         },
         {
@@ -43,5 +40,13 @@
                 "${file}"
             ]
         },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
     ]
 }

.vscode/launch.py

@@ -6,10 +6,14 @@
 import re
 import os
 import sys
+
+print(sys.executable)
+
 import shlex
 import jstyleson
 import subprocess as sp
 
+
 with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
 

README.md

@@ -37,6 +37,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [other tricks](#other-tricks)
 * [searching](#searching)
     * [search configuration](#search-configuration)
+    * [database location](#database-location)
     * [metadata from audio files](#metadata-from-audio-files)
     * [file parser plugins](#file-parser-plugins)
     * [complete examples](#complete-examples)
@@ -105,6 +106,7 @@ summary: all planned features work! now please enjoy the bloatening
     * ☑ images using Pillow
     * ☑ videos using FFmpeg
     * ☑ cache eviction (max-age; maybe max-size eventually)
+  * ☑ image gallery
   * ☑ SPA (browse while uploading)
     * if you use the file-tree on the left only, not folders in the file list
 * server indexing
@@ -121,12 +123,12 @@ summary: all planned features work! now please enjoy the bloatening
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
+* MacOS: `--th-ff-jpg` may fix thumbnails using macports-FFmpeg
 
 ## general bugs
 
 * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
 * cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
-* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
 ## not my bugs
@@ -180,6 +182,8 @@ click `[-]` and `[+]` to adjust the size, and the `[a]` toggles if the tree shou
 
 it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how destructive your users are
 
+images named `folder.jpg` and `folder.png` become the thumbnail of the folder they're in
+
 
 ## zip downloads
 
@@ -296,9 +300,29 @@ the same arguments can be set as volume flags, in addition to `d2d` and `d2t` fo
 * `-v ~/music::r:cd2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 
-`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+note:
+* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+* the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
 
-the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
+you can choose to only index filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash` or the volume-flag `cdhash`, this has the following consequences:
+* initial indexing is way faster, especially when the volume is on a networked disk
+* makes it impossible to [file-search](#file-search)
+* if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
+
+if you set `--no-hash`, you can enable hashing for specific volumes using flag `cehash`
+
+
+## database location
+
+copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
+
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+* `--hist ~/.cache/copyparty -v ~/music::r:chist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
+
+note:
+* markdown edits are always stored in a local `.hist` subdirectory
+* on windows the volflag path is cyglike, so `/c/temp` means `C:\temp` but use regular paths for `--hist`
+  * you can use cygpaths for volumes too, `-v C:\Users::r` and `-v /c/users::r` both work
 
 
 ## metadata from audio files

copyparty/__main__.py

@@ -263,6 +263,9 @@ def run_argparse(argv, formatter):
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 
+    ap2 = ap.add_argument_group('appearance options')
+    ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
+
     ap2 = ap.add_argument_group('admin panel options')
     ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
     ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
@@ -274,6 +277,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
     ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
     ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
+    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
     ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
     ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval")
     ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
@@ -285,6 +289,8 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
     ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
     ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
+    ap2.add_argument("--hist", metavar="PATH", type=str, help="where to store volume state")
+    ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
     ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
     ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
     ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 11)
+VERSION = (0, 11, 13)
 CODENAME = "the grid"
-BUILD_DT = (2021, 6, 8)
+BUILD_DT = (2021, 6, 12)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -5,6 +5,8 @@ import re
 import os
 import sys
 import stat
+import base64
+import hashlib
 import threading
 
 from .__init__ import WINDOWS
@@ -22,7 +24,15 @@ class VFS(object):
         self.uadm = uadm  # users who are regular admins
         self.flags = flags  # config switches
         self.nodes = {}  # child nodes
-        self.all_vols = {vpath: self} if realpath else {}  # flattened recursive
+        self.histtab = None  # all realpath->histpath
+        self.dbv = None  # closest full/non-jump parent
+
+        if realpath:
+            self.histpath = os.path.join(realpath, ".hist")  # db / thumbcache
+            self.all_vols = {vpath: self}  # flattened recursive
+        else:
+            self.histpath = None
+            self.all_vols = None
 
     def __repr__(self):
         return "VFS({})".format(
@@ -32,9 +42,12 @@ class VFS(object):
             )
         )
 
-    def _trk(self, vol):
-        self.all_vols[vol.vpath] = vol
-        return vol
+    def get_all_vols(self, outdict):
+        if self.realpath:
+            outdict[self.vpath] = self
+
+        for v in self.nodes.values():
+            v.get_all_vols(outdict)
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""
@@ -46,19 +59,19 @@ class VFS(object):
             name, dst = dst.split("/", 1)
             if name in self.nodes:
                 # exists; do not manipulate permissions
-                return self._trk(self.nodes[name].add(src, dst))
+                return self.nodes[name].add(src, dst)
 
             vn = VFS(
-                "{}/{}".format(self.realpath, name),
+                os.path.join(self.realpath, name) if self.realpath else None,
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
                 self.uwrite,
                 self.uadm,
-                self.flags,
+                self._copy_flags(name),
             )
-            self._trk(vn)
+            vn.dbv = self.dbv or self
             self.nodes[name] = vn
-            return self._trk(vn.add(src, dst))
+            return vn.add(src, dst)
 
         if dst in self.nodes:
             # leaf exists; return as-is
@@ -67,8 +80,26 @@ class VFS(object):
         # leaf does not exist; create and keep permissions blank
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
+        vn.dbv = self.dbv or self
         self.nodes[dst] = vn
-        return self._trk(vn)
+        return vn
+
+    def _copy_flags(self, name):
+        flags = {k: v for k, v in self.flags.items()}
+        hist = flags.get("hist")
+        if hist and hist != "-":
+            flags["hist"] = "{}/{}".format(hist.rstrip("/"), name)
+
+        return flags
+
+    def bubble_flags(self):
+        if self.dbv:
+            for k, v in self.dbv.flags.items():
+                if k not in ["hist"]:
+                    self.flags[k] = v
+
+        for v in self.nodes.values():
+            v.bubble_flags()
 
     def _find(self, vpath):
         """return [vfs,remainder]"""
@@ -108,6 +139,15 @@ class VFS(object):
 
         return vn, rem
 
+    def get_dbv(self, vrem):
+        dbv = self.dbv
+        if not dbv:
+            return self, vrem
+
+        vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem]
+        vrem = "/".join([x for x in vrem if x])
+        return dbv, vrem
+
     def canonical(self, rem):
         """returns the canonical path (fully-resolved absolute fs path)"""
         rp = self.realpath
@@ -273,7 +313,8 @@ class AuthSrv(object):
         self.reload()
 
     def log(self, msg, c=0):
-        self.log_func("auth", msg, c)
+        if self.log_func:
+            self.log_func("auth", msg, c)
 
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
@@ -398,6 +439,9 @@ class AuthSrv(object):
                     raise Exception("invalid -v argument: [{}]".format(v_str))
 
                 src, dst, perms = m.groups()
+                if WINDOWS and src.startswith("/"):
+                    src = "{}:\\{}".format(src[1], src[3:])
+
                 # print("\n".join([src, dst, perms]))
                 src = fsdec(os.path.abspath(fsenc(src)))
                 dst = dst.strip("/")
@@ -451,6 +495,10 @@ class AuthSrv(object):
             v.uwrite = mwrite[dst]
             v.uadm = madm[dst]
             v.flags = mflags[dst]
+            v.dbv = None
+
+        vfs.all_vols = {}
+        vfs.get_all_vols(vfs.all_vols)
 
         missing_users = {}
         for d in [mread, mwrite]:
@@ -467,6 +515,69 @@ class AuthSrv(object):
             )
             raise Exception("invalid config")
 
+        promote = []
+        demote = []
+        for vol in vfs.all_vols.values():
+            hid = hashlib.sha512(fsenc(vol.realpath)).digest()
+            hid = base64.b32encode(hid).decode("ascii").lower()
+            vflag = vol.flags.get("hist")
+            if vflag == "-":
+                pass
+            elif vflag:
+                if WINDOWS and vflag.startswith("/"):
+                    vflag = "{}:\\{}".format(vflag[1], vflag[3:])
+                vol.histpath = vflag
+            elif self.args.hist:
+                for nch in range(len(hid)):
+                    hpath = os.path.join(self.args.hist, hid[: nch + 1])
+                    try:
+                        os.makedirs(hpath)
+                    except:
+                        pass
+
+                    powner = os.path.join(hpath, "owner.txt")
+                    try:
+                        with open(powner, "rb") as f:
+                            owner = f.read().rstrip()
+                    except:
+                        owner = None
+
+                    me = fsenc(vol.realpath).rstrip()
+                    if owner not in [None, me]:
+                        continue
+
+                    if owner is None:
+                        with open(powner, "wb") as f:
+                            f.write(me)
+
+                    vol.histpath = hpath
+                    break
+
+            vol.histpath = os.path.realpath(vol.histpath)
+            if vol.dbv:
+                if os.path.exists(os.path.join(vol.histpath, "up2k.db")):
+                    promote.append(vol)
+                    vol.dbv = None
+                else:
+                    demote.append(vol)
+
+        # discard jump-vols
+        for v in demote:
+            vfs.all_vols.pop(v.vpath)
+
+        if promote:
+            msg = [
+                "\n  the following jump-volumes were generated to assist the vfs.\n  As they contain a database (probably from v0.11.11 or older),\n  they are promoted to full volumes:"
+            ]
+            for vol in promote:
+                msg.append(
+                    "  /{}  ({})  ({})".format(vol.vpath, vol.realpath, vol.histpath)
+                )
+
+            self.log("\n\n".join(msg) + "\n", c=3)
+
+        vfs.histtab = {v.realpath: v.histpath for v in vfs.all_vols.values()}
+
         all_mte = {}
         errors = False
         for vol in vfs.all_vols.values():
@@ -476,6 +587,10 @@ class AuthSrv(object):
             if self.args.e2d or "e2ds" in vol.flags:
                 vol.flags["e2d"] = True
 
+            if self.args.no_hash:
+                if "ehash" not in vol.flags:
+                    vol.flags["dhash"] = True
+
             for k in ["e2t", "e2ts", "e2tsr"]:
                 if getattr(self.args, k):
                     vol.flags[k] = True
@@ -553,6 +668,8 @@ class AuthSrv(object):
         if errors:
             sys.exit(1)
 
+        vfs.bubble_flags()
+
         try:
             v, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == v.realpath:

copyparty/broker_mpw.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
+from copyparty.authsrv import AuthSrv
 
 import sys
 import time
@@ -34,6 +35,9 @@ class MpWorker(object):
         if not FAKE_MP:
             signal.signal(signal.SIGINT, self.signal_handler)
 
+        # starting to look like a good idea
+        self.asrv = AuthSrv(args, None, False)
+
         # instantiate all services here (TODO: inheritance?)
         self.httpsrv = HttpSrv(self, True)
         self.httpsrv.disconnect_func = self.httpdrop

copyparty/broker_thr.py

@@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals
 
 import threading
 
+from .authsrv import AuthSrv
 from .httpsrv import HttpSrv
 from .broker_util import ExceptionalQueue, try_exec
 
@@ -14,6 +15,7 @@ class BrokerThr(object):
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
+        self.asrv = hub.asrv
 
         self.mutex = threading.Lock()
 

copyparty/httpcli.py

@@ -42,7 +42,7 @@ class HttpCli(object):
         self.addr = conn.addr  # type: tuple[str, int]
         self.args = conn.args
         self.is_mp = conn.is_mp
-        self.auth = conn.auth  # type: AuthSrv
+        self.asrv = conn.asrv  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
         self.log_func = conn.log_func
@@ -154,9 +154,9 @@ class HttpCli(object):
         self.vpath = unquotep(vpath)
 
         pwd = uparam.get("pw")
-        self.uname = self.auth.iuser.get(pwd, "*")
+        self.uname = self.asrv.iuser.get(pwd, "*")
         self.rvol, self.wvol, self.avol = [[], [], []]
-        self.auth.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
+        self.asrv.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
 
         ua = self.headers.get("user-agent", "")
         self.is_rclone = ua.startswith("rclone/")
@@ -258,7 +258,14 @@ class HttpCli(object):
         return "?" + "&".join(r)
 
     def redirect(
-        self, vpath, suf="", msg="aight", flavor="go to", click=True, use302=False
+        self,
+        vpath,
+        suf="",
+        msg="aight",
+        flavor="go to",
+        click=True,
+        status=200,
+        use302=False,
     ):
         html = self.j2(
             "msg",
@@ -273,7 +280,7 @@ class HttpCli(object):
             h = {"Location": "/" + vpath, "Cache-Control": "no-cache"}
             self.reply(html, status=302, headers=h)
         else:
-            self.reply(html)
+            self.reply(html, status=status)
 
     def handle_get(self):
         if self.do_log:
@@ -314,9 +321,7 @@ class HttpCli(object):
                     self.redirect(vpath, flavor="redirecting to", use302=True)
                     return True
 
-        self.readable, self.writable = self.conn.auth.vfs.can_access(
-            self.vpath, self.uname
-        )
+        self.readable, self.writable = self.asrv.vfs.can_access(self.vpath, self.uname)
         if not self.readable and not self.writable:
             if self.vpath:
                 self.log("inaccessible: [{}]".format(self.vpath))
@@ -433,7 +438,7 @@ class HttpCli(object):
 
     def dump_to_file(self):
         reader, remains = self.get_body_reader()
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         fdir = os.path.join(vfs.realpath, rem)
 
         addr = self.ip.replace(":", ".")
@@ -443,8 +448,10 @@ class HttpCli(object):
         with open(fsenc(path), "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
+        vfs, vrem = vfs.get_dbv(rem)
+
         self.conn.hsrv.broker.put(
-            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fn
+            False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
         )
 
         return post_sz, sha_b64, remains, path
@@ -500,7 +507,7 @@ class HttpCli(object):
         if v is None:
             raise Pebkac(422, "need zip or tar keyword")
 
-        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        vn, rem = self.asrv.vfs.get(self.vpath, self.uname, True, False)
         items = self.parser.require("files", 1024 * 1024)
         if not items:
             raise Pebkac(422, "need files list")
@@ -550,13 +557,14 @@ class HttpCli(object):
             self.vpath = "/".join([self.vpath, sub]).strip("/")
             body["name"] = name
 
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
+        dbv, vrem = vfs.get_dbv(rem)
 
-        body["vtop"] = vfs.vpath
-        body["ptop"] = vfs.realpath
-        body["prel"] = rem
+        body["vtop"] = dbv.vpath
+        body["ptop"] = dbv.realpath
+        body["prel"] = vrem
         body["addr"] = self.ip
-        body["vcfg"] = vfs.flags
+        body["vcfg"] = dbv.flags
 
         if sub:
             try:
@@ -578,8 +586,14 @@ class HttpCli(object):
 
     def handle_search(self, body):
         vols = []
+        seen = {}
         for vtop in self.rvol:
-            vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
+            vfs, _ = self.asrv.vfs.get(vtop, self.uname, True, False)
+            vfs = vfs.dbv or vfs
+            if vfs in seen:
+                continue
+
+            seen[vfs] = True
             vols.append([vfs.vpath, vfs.realpath, vfs.flags])
 
         idx = self.conn.get_u2idx()
@@ -635,8 +649,8 @@ class HttpCli(object):
         except KeyError:
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
-        vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
-        ptop = vfs.realpath
+        vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
+        ptop = (vfs.dbv or vfs).realpath
 
         x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
         response = x.get()
@@ -708,7 +722,7 @@ class HttpCli(object):
         pwd = self.parser.require("cppwd", 64)
         self.parser.drop()
 
-        if pwd in self.auth.iuser:
+        if pwd in self.asrv.iuser:
             msg = "login ok"
             dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
             exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
@@ -727,7 +741,7 @@ class HttpCli(object):
         self.parser.drop()
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         sanitized = sanitize_fn(new_dir)
@@ -756,7 +770,7 @@ class HttpCli(object):
         self.parser.drop()
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         if not new_file.endswith(".md"):
@@ -780,7 +794,7 @@ class HttpCli(object):
 
     def handle_plain_upload(self):
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         files = []
@@ -817,8 +831,14 @@ class HttpCli(object):
                             raise Pebkac(400, "empty files in post")
 
                         files.append([sz, sha512_hex, p_file, fname])
+                        dbv, vrem = vfs.get_dbv(rem)
                         self.conn.hsrv.broker.put(
-                            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
+                            False,
+                            "up2k.hash_file",
+                            dbv.realpath,
+                            dbv.flags,
+                            vrem,
+                            fname,
                         )
                         self.conn.nbyte += sz
 
@@ -848,12 +868,16 @@ class HttpCli(object):
         status = "OK"
         if errmsg:
             self.log(errmsg)
-            errmsg = "ERROR: " + errmsg
             status = "ERROR"
 
         msg = "{} // {} bytes // {:.3f} MiB/s\n".format(status, sz_total, spd)
         jmsg = {"status": status, "sz": sz_total, "mbps": round(spd, 3), "files": []}
 
+        if errmsg:
+            msg += errmsg + "\n"
+            jmsg["error"] = errmsg
+            errmsg = "ERROR: " + errmsg
+
         for sz, sha512, ofn, lfn in files:
             vpath = (self.vpath + "/" if self.vpath else "") + lfn
             msg += 'sha512: {} // {} bytes // <a href="/{}">{}</a>\n'.format(
@@ -885,11 +909,21 @@ class HttpCli(object):
                 ft = "{}\n{}\n{}\n".format(ft, msg.rstrip(), errmsg)
                 f.write(ft.encode("utf-8"))
 
+        status = 400 if errmsg else 200
         if "j" in self.uparam:
-            jtxt = json.dumps(jmsg, indent=2, sort_keys=True)
-            self.reply(jtxt.encode("utf-8", "replace"), mime="application/json")
+            jtxt = json.dumps(jmsg, indent=2, sort_keys=True).encode("utf-8", "replace")
+            self.reply(jtxt, mime="application/json", status=status)
         else:
-            self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
+            self.redirect(
+                self.vpath,
+                msg=msg,
+                flavor="return to",
+                click=False,
+                status=status,
+            )
+
+        if errmsg:
+            return False
 
         self.parser.drop()
         return True
@@ -901,7 +935,7 @@ class HttpCli(object):
             raise Pebkac(400, "could not read lastmod from request")
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         # TODO:
@@ -1339,11 +1373,13 @@ class HttpCli(object):
             for y in [self.rvol, self.wvol, self.avol]
         ]
 
-        vstate = {}
         if self.avol and not self.args.no_rescan:
-            x = self.conn.hsrv.broker.put(True, "up2k.get_volstate")
-            vstate = json.loads(x.get())
-            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vstate.items()}
+            x = self.conn.hsrv.broker.put(True, "up2k.get_state")
+            vs = json.loads(x.get())
+            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()}
+        else:
+            vstate = {}
+            vs = {"scanning": None, "hashq": None, "tagq": None, "mtpq": None}
 
         html = self.j2(
             "splash",
@@ -1352,6 +1388,10 @@ class HttpCli(object):
             wvol=wvol,
             avol=avol,
             vstate=vstate,
+            scanning=vs["scanning"],
+            hashq=vs["hashq"],
+            tagq=vs["tagq"],
+            mtpq=vs["mtpq"],
             url_suf=suf,
         )
         self.reply(html.encode("utf-8"), headers=NO_STORE)
@@ -1364,9 +1404,10 @@ class HttpCli(object):
         if self.args.no_rescan:
             raise Pebkac(403, "disabled by argv")
 
-        vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True)
+        vn, _ = self.asrv.vfs.get(self.vpath, self.uname, True, True)
+
+        args = [self.asrv.vfs.all_vols, [vn.vpath]]
 
-        args = [self.auth.vfs.all_vols, [vn.vpath]]
         x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
         x = x.get()
         if not x:
@@ -1438,7 +1479,7 @@ class HttpCli(object):
             ret["k" + quotep(excl)] = sub
 
         try:
-            vn, rem = self.auth.vfs.get(top, self.uname, True, False)
+            vn, rem = self.asrv.vfs.get(top, self.uname, True, False)
             fsroot, vfs_ls, vfs_virt = vn.ls(
                 rem, self.uname, not self.args.no_scandir, incl_wo=True
             )
@@ -1479,35 +1520,51 @@ class HttpCli(object):
 
                 vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
-        vn, rem = self.auth.vfs.get(
+        vn, rem = self.asrv.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
         )
         abspath = vn.canonical(rem)
+        dbv, vrem = vn.get_dbv(rem)
 
         try:
             st = os.stat(fsenc(abspath))
         except:
             raise Pebkac(404)
 
-        if self.readable and not stat.S_ISDIR(st.st_mode):
+        if self.readable:
             if rem.startswith(".hist/up2k."):
                 raise Pebkac(403)
 
+            is_dir = stat.S_ISDIR(st.st_mode)
             th_fmt = self.uparam.get("th")
             if th_fmt is not None:
+                if is_dir:
+                    for fn in ["folder.png", "folder.jpg"]:
+                        fp = os.path.join(abspath, fn)
+                        if os.path.exists(fp):
+                            vrem = "{}/{}".format(vrem.rstrip("/"), fn)
+                            is_dir = False
+                            break
+
+                    if is_dir:
+                        return self.tx_ico("a.folder")
+
                 thp = None
                 if self.thumbcli:
-                    thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime), th_fmt)
+                    thp = self.thumbcli.get(
+                        dbv.realpath, vrem, int(st.st_mtime), th_fmt
+                    )
 
                 if thp:
                     return self.tx_file(thp)
 
                 return self.tx_ico(rem)
 
-            if abspath.endswith(".md") and "raw" not in self.uparam:
-                return self.tx_md(abspath)
+            if not is_dir:
+                if abspath.endswith(".md") and "raw" not in self.uparam:
+                    return self.tx_md(abspath)
 
-            return self.tx_file(abspath)
+                return self.tx_file(abspath)
 
         srv_info = []
 
@@ -1640,7 +1697,7 @@ class HttpCli(object):
         icur = None
         if "e2t" in vn.flags:
             idx = self.conn.get_u2idx()
-            icur = idx.get_cur(vn.realpath)
+            icur = idx.get_cur(dbv.realpath)
 
         dirs = []
         files = []
@@ -1708,6 +1765,9 @@ class HttpCli(object):
             rd = f["rd"]
             del f["rd"]
             if icur:
+                if vn != dbv:
+                    _, rd = vn.get_dbv(rd)
+
                 q = "select w from up where rd = ? and fn = ?"
                 try:
                     r = icur.execute(q, (rd, fn)).fetchone()
@@ -1748,9 +1808,13 @@ class HttpCli(object):
         j2a["files"] = dirs + files
         j2a["logues"] = logues
         j2a["taglist"] = taglist
+
         if "mte" in vn.flags:
             j2a["tag_order"] = json.dumps(vn.flags["mte"].split(","))
 
+        if self.args.css_browser:
+            j2a["css"] = self.args.css_browser
+
         html = self.j2(tpl, **j2a)
         self.reply(html.encode("utf-8", "replace"), headers=NO_STORE)
         return True

copyparty/httpconn.py

@@ -34,7 +34,7 @@ class HttpConn(object):
         self.hsrv = hsrv
 
         self.args = hsrv.args
-        self.auth = hsrv.auth
+        self.asrv = hsrv.asrv
         self.is_mp = hsrv.is_mp
         self.cert_path = hsrv.cert_path
 
@@ -71,7 +71,7 @@ class HttpConn(object):
 
     def get_u2idx(self):
         if not self.u2idx:
-            self.u2idx = U2idx(self.args, self.log_func)
+            self.u2idx = U2idx(self)
 
         return self.u2idx
 

copyparty/httpsrv.py

@@ -40,6 +40,7 @@ class HttpSrv(object):
         self.is_mp = is_mp
         self.args = broker.args
         self.log = broker.log
+        self.asrv = broker.asrv
 
         self.disconnect_func = None
         self.mutex = threading.Lock()
@@ -47,7 +48,6 @@ class HttpSrv(object):
         self.clients = {}
         self.workload = 0
         self.workload_thr_alive = False
-        self.auth = AuthSrv(self.args, self.log)
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))

copyparty/mtag.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
 import os
 import sys
 import json

copyparty/svchub.py

@@ -37,14 +37,13 @@ class SvcHub(object):
 
         self.log = self._log_disabled if args.q else self._log_enabled
 
-        # jank goes here
-        auth = AuthSrv(self.args, self.log, False)
-        if args.ls:
-            auth.dbg_ls()
-
         # initiate all services to manage
+        self.asrv = AuthSrv(self.args, self.log, False)
+        if args.ls:
+            self.asrv.dbg_ls()
+
         self.tcpsrv = TcpSrv(self)
-        self.up2k = Up2k(self, auth.vfs.all_vols)
+        self.up2k = Up2k(self)
 
         self.thumbsrv = None
         if not args.no_thumb:
@@ -54,7 +53,7 @@ class SvcHub(object):
                     msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
                     self.log("thumb", msg, c=3)
 
-                self.thumbsrv = ThumbSrv(self, auth.vfs.all_vols)
+                self.thumbsrv = ThumbSrv(self)
             else:
                 msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
                 self.log(

copyparty/th_cli.py

@@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import time
 
 from .util import Cooldown
 from .th_srv import thumb_path, THUMBABLE, FMT_FF
@@ -12,6 +11,7 @@ class ThumbCli(object):
     def __init__(self, broker):
         self.broker = broker
         self.args = broker.args
+        self.asrv = broker.asrv
 
         # cache on both sides for less broker spam
         self.cooldown = Cooldown(self.args.th_poke)
@@ -21,16 +21,19 @@ class ThumbCli(object):
         if ext not in THUMBABLE:
             return None
 
-        if self.args.no_vthumb and ext in FMT_FF:
+        is_vid = ext in FMT_FF
+        if is_vid and self.args.no_vthumb:
             return None
 
         if fmt == "j" and self.args.th_no_jpg:
             fmt = "w"
 
-        if fmt == "w" and self.args.th_no_webp:
-            fmt = "j"
+        if fmt == "w":
+            if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
+                fmt = "j"
 
-        tpath = thumb_path(ptop, rem, mtime, fmt)
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
         ret = None
         try:
             st = os.stat(tpath)

copyparty/th_srv.py

@@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import sys
 import time
 import shutil
 import base64
@@ -11,7 +10,7 @@ import threading
 import subprocess as sp
 
 from .__init__ import PY2
-from .util import fsenc, mchkcmd, Queue, Cooldown, BytesIO
+from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 
 
@@ -54,7 +53,7 @@ except:
 # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
 # ffmpeg -formats
 FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
-FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
+FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
 
 if HAVE_HEIF:
     FMT_PIL += " heif heifs heic heics"
@@ -74,7 +73,7 @@ if HAVE_FFMPEG and HAVE_FFPROBE:
     THUMBABLE.update(FMT_FF)
 
 
-def thumb_path(ptop, rem, mtime, fmt):
+def thumb_path(histpath, rem, mtime, fmt):
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
@@ -95,16 +94,15 @@ def thumb_path(ptop, rem, mtime, fmt):
     h = hashlib.sha512(fsenc(fn)).digest()[:24]
     fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
 
-    return "{}/.hist/th/{}/{}.{:x}.{}".format(
-        ptop, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
+    return "{}/th/{}/{}.{:x}.{}".format(
+        histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
     )
 
 
 class ThumbSrv(object):
-    def __init__(self, hub, vols):
+    def __init__(self, hub):
         self.hub = hub
-        self.vols = [v.realpath for v in vols.values()]
-
+        self.asrv = hub.asrv
         self.args = hub.args
         self.log_func = hub.log
 
@@ -153,7 +151,8 @@ class ThumbSrv(object):
             return not self.nthr
 
     def get(self, ptop, rem, mtime, fmt):
-        tpath = thumb_path(ptop, rem, mtime, fmt)
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
         abspath = os.path.join(ptop, rem)
         cond = threading.Condition()
         with self.mutex:
@@ -211,9 +210,9 @@ class ThumbSrv(object):
             if fun:
                 try:
                     fun(abspath, tpath)
-                except Exception as ex:
-                    msg = "{} failed on {}\n  {!r}"
-                    self.log(msg.format(fun.__name__, abspath, ex), 3)
+                except:
+                    msg = "{} failed on {}\n{}"
+                    self.log(msg.format(fun.__name__, abspath, min_ex()), 3)
                     with open(tpath, "wb") as _:
                         pass
 
@@ -245,8 +244,8 @@ class ThumbSrv(object):
             except:
                 im.thumbnail(self.res)
 
-            if im.mode not in ("RGB", "L"):
-                im = im.convert("RGB")
+            fmts = ["RGB", "L"]
+            args = {"quality": 40}
 
             if tpath.endswith(".webp"):
                 # quality 80 = pillow-default
@@ -254,15 +253,27 @@ class ThumbSrv(object):
                 # method 0 = pillow-default, fast
                 # method 4 = ffmpeg-default
                 # method 6 = max, slow
-                im.save(tpath, quality=40, method=6)
+                fmts += ["RGBA", "LA"]
+                args["method"] = 6
             else:
-                im.save(tpath, quality=40)  # default=75
+                pass  # default q = 75
+
+            if im.mode not in fmts:
+                print("conv {}".format(im.mode))
+                im = im.convert("RGB")
+
+            im.save(tpath, quality=40, method=6)
 
     def conv_ffmpeg(self, abspath, tpath):
         ret, _ = ffprobe(abspath)
 
-        dur = ret[".dur"][1] if ".dur" in ret else 4
-        seek = "{:.0f}".format(dur / 3)
+        ext = abspath.rsplit(".")[-1]
+        if ext in ["h264", "h265"]:
+            seek = []
+        else:
+            dur = ret[".dur"][1] if ".dur" in ret else 4
+            seek = "{:.0f}".format(dur / 3)
+            seek = [b"-ss", seek.encode("utf-8")]
 
         scale = "scale={0}:{1}:force_original_aspect_ratio="
         if self.args.th_no_crop:
@@ -271,19 +282,20 @@ class ThumbSrv(object):
             scale += "increase,crop={0}:{1},setsar=1:1"
 
         scale = scale.format(*list(self.res)).encode("utf-8")
+        # fmt: off
         cmd = [
             b"ffmpeg",
             b"-nostdin",
-            b"-hide_banner",
-            b"-ss",
-            seek,
-            b"-i",
-            fsenc(abspath),
-            b"-vf",
-            scale,
-            b"-vframes",
-            b"1",
+            b"-v", b"error",
+            b"-hide_banner"
         ]
+        cmd += seek
+        cmd += [
+            b"-i", fsenc(abspath),
+            b"-vf", scale,
+            b"-vframes", b"1",
+        ]
+        # fmt: on
 
         if tpath.endswith(".jpg"):
             cmd += [
@@ -300,7 +312,11 @@ class ThumbSrv(object):
 
         cmd += [fsenc(tpath)]
 
-        mchkcmd(cmd)
+        ret, sout, serr = runcmd(*cmd)
+        if ret != 0:
+            msg = ["ff: {}".format(x) for x in serr.split("\n")]
+            self.log("FFmpeg failed:\n" + "\n".join(msg), c="1;30")
+            raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
     def poke(self, tdir):
         if not self.poke_cd.poke(tdir):
@@ -319,26 +335,29 @@ class ThumbSrv(object):
         interval = self.args.th_clean
         while True:
             time.sleep(interval)
-            for vol in self.vols:
-                vol += "/.hist/th"
-                self.log("\033[Jcln {}/\033[A".format(vol))
-                self.clean(vol)
+            for vol, histpath in self.asrv.vfs.histtab.items():
+                if histpath.startswith(vol):
+                    self.log("\033[Jcln {}/\033[A".format(histpath))
+                else:
+                    self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
+
+                self.clean(histpath)
 
             self.log("\033[Jcln ok")
 
-    def clean(self, vol):
-        # self.log("cln {}".format(vol))
+    def clean(self, histpath):
+        # self.log("cln {}".format(histpath))
         maxage = self.args.th_maxage
         now = time.time()
         prev_b64 = None
         prev_fp = None
         try:
-            ents = os.listdir(vol)
+            ents = os.listdir(histpath)
         except:
             return
 
         for f in sorted(ents):
-            fp = os.path.join(vol, f)
+            fp = os.path.join(histpath, f)
             cmp = fp.lower().replace("\\", "/")
 
             # "top" or b64 prefix/full (a folder)

copyparty/u2idx.py

@@ -7,7 +7,7 @@ import time
 import threading
 from datetime import datetime
 
-from .util import u8safe, s3dec, html_escape, Pebkac
+from .util import s3dec, Pebkac, min_ex
 from .up2k import up2k_wark_from_hashlist
 
 
@@ -19,10 +19,11 @@ except:
 
 
 class U2idx(object):
-    def __init__(self, args, log_func):
-        self.args = args
-        self.log_func = log_func
-        self.timeout = args.srch_time
+    def __init__(self, conn):
+        self.log_func = conn.log_func
+        self.asrv = conn.asrv
+        self.args = conn.args
+        self.timeout = self.args.srch_time
 
         if not HAVE_SQLITE3:
             self.log("could not load sqlite3; searchign wqill be disabled")
@@ -52,18 +53,20 @@ class U2idx(object):
 
         try:
             return self.run_query(vols, uq, uv)[0]
-        except Exception as ex:
-            raise Pebkac(500, repr(ex))
+        except:
+            raise Pebkac(500, min_ex())
 
     def get_cur(self, ptop):
         cur = self.cur.get(ptop)
         if cur:
             return cur
 
-        cur = _open(ptop)
-        if not cur:
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
+        if not os.path.exists(db_path):
             return None
 
+        cur = sqlite3.connect(db_path).cursor()
         self.cur[ptop] = cur
         return cur
 
@@ -242,6 +245,7 @@ class U2idx(object):
                 hit["tags"] = tags
 
             ret.extend(sret)
+            # print("[{}] {}".format(ptop, sret))
 
         done_flag.append(True)
         self.active_id = None
@@ -262,9 +266,3 @@ class U2idx(object):
 
         if identifier == self.active_id:
             self.active_cur.connection.interrupt()
-
-
-def _open(ptop):
-    db_path = os.path.join(ptop, ".hist", "up2k.db")
-    if os.path.exists(db_path):
-        return sqlite3.connect(db_path).cursor()

copyparty/up2k.py

@@ -48,8 +48,9 @@ class Up2k(object):
         * ~/.config flatfiles for active jobs
     """
 
-    def __init__(self, hub, all_vols):
+    def __init__(self, hub):
         self.hub = hub
+        self.asrv = hub.asrv
         self.args = hub.args
         self.log_func = hub.log
 
@@ -60,6 +61,8 @@ class Up2k(object):
         self.mutex = threading.Lock()
         self.hashq = Queue()
         self.tagq = Queue()
+        self.n_hashq = 0
+        self.n_tagq = 0
         self.volstate = {}
         self.registry = {}
         self.entags = {}
@@ -94,15 +97,17 @@ class Up2k(object):
             self.log("could not initialize sqlite3, will use in-memory registry only")
 
         if self.args.no_fastboot:
-            self.deferred_init(all_vols)
+            self.deferred_init()
         else:
             t = threading.Thread(
-                target=self.deferred_init, args=(all_vols,), name="up2k-deferred-init"
+                target=self.deferred_init,
+                name="up2k-deferred-init",
             )
             t.daemon = True
             t.start()
 
-    def deferred_init(self, all_vols):
+    def deferred_init(self):
+        all_vols = self.asrv.vfs.all_vols
         have_e2d = self.init_indexes(all_vols)
 
         if have_e2d:
@@ -126,8 +131,28 @@ class Up2k(object):
     def log(self, msg, c=0):
         self.log_func("up2k", msg + "\033[K", c)
 
-    def get_volstate(self):
-        return json.dumps(self.volstate, indent=4)
+    def get_state(self):
+        mtpq = 0
+        q = "select count(w) from mt where k = 't:mtp'"
+        got_lock = self.mutex.acquire(timeout=0.5)
+        if got_lock:
+            for cur in self.cur.values():
+                try:
+                    mtpq += cur.execute(q).fetchone()[0]
+                except:
+                    pass
+            self.mutex.release()
+        else:
+            mtpq = "?"
+
+        ret = {
+            "volstate": self.volstate,
+            "scanning": hasattr(self, "pp"),
+            "hashq": self.n_hashq,
+            "tagq": self.n_tagq,
+            "mtpq": mtpq,
+        }
+        return json.dumps(ret, indent=4)
 
     def rescan(self, all_vols, scan_vols):
         if hasattr(self, "pp"):
@@ -184,25 +209,27 @@ class Up2k(object):
                 self.log(msg, c=3)
 
         live_vols = []
-        for vol in vols:
-            try:
-                os.listdir(vol.realpath)
-            except:
-                self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
-                self.log("cannot access " + vol.realpath, c=1)
-                continue
+        with self.mutex:
+            # only need to protect register_vpath but all in one go feels right
+            for vol in vols:
+                try:
+                    os.listdir(vol.realpath)
+                except:
+                    self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
+                    self.log("cannot access " + vol.realpath, c=1)
+                    continue
 
-            if scan_vols and vol.vpath not in scan_vols:
-                continue
+                if scan_vols and vol.vpath not in scan_vols:
+                    continue
 
-            if not self.register_vpath(vol.realpath, vol.flags):
-                # self.log("db not enabled for {}".format(m, vol.realpath))
-                continue
+                if not self.register_vpath(vol.realpath, vol.flags):
+                    # self.log("db not enable for {}".format(m, vol.realpath))
+                    continue
 
-            live_vols.append(vol)
+                live_vols.append(vol)
 
-            if vol.vpath not in self.volstate:
-                self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
+                if vol.vpath not in self.volstate:
+                    self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
 
         vols = live_vols
         need_vac = {}
@@ -294,7 +321,8 @@ class Up2k(object):
         return have_e2d
 
     def register_vpath(self, ptop, flags):
-        db_path = os.path.join(ptop, ".hist", "up2k.db")
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
         if ptop in self.registry:
             try:
                 return [self.cur[ptop], db_path]
@@ -314,7 +342,7 @@ class Up2k(object):
             self.log(" ".join(sorted(a)) + "\033[0m")
 
         reg = {}
-        path = os.path.join(ptop, ".hist", "up2k.snap")
+        path = os.path.join(histpath, "up2k.snap")
         if "e2d" in flags and os.path.exists(path):
             with gzip.GzipFile(path, "rb") as f:
                 j = f.read().decode("utf-8")
@@ -338,7 +366,7 @@ class Up2k(object):
             return None
 
         try:
-            os.mkdir(os.path.join(ptop, ".hist"))
+            os.makedirs(histpath)
         except:
             pass
 
@@ -355,6 +383,7 @@ class Up2k(object):
     def _build_file_index(self, vol, all_vols):
         do_vac = False
         top = vol.realpath
+        nohash = "dhash" in vol.flags
         with self.mutex:
             cur, _ = self.register_vpath(top, vol.flags)
 
@@ -369,7 +398,7 @@ class Up2k(object):
             if WINDOWS:
                 excl = [x.replace("/", "\\") for x in excl]
 
-            n_add = self._build_dir(dbw, top, set(excl), top)
+            n_add = self._build_dir(dbw, top, set(excl), top, nohash)
             n_rm = self._drop_lost(dbw[0], top)
             if dbw[1]:
                 self.log("commit {} new files".format(dbw[1]))
@@ -377,24 +406,25 @@ class Up2k(object):
 
             return True, n_add or n_rm or do_vac
 
-    def _build_dir(self, dbw, top, excl, cdir):
+    def _build_dir(self, dbw, top, excl, cdir, nohash):
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
-        histdir = os.path.join(top, ".hist")
+        histpath = self.asrv.vfs.histtab[top]
         ret = 0
         g = statdir(self.log, not self.args.no_scandir, False, cdir)
         for iname, inf in sorted(g):
             abspath = os.path.join(cdir, iname)
             lmod = int(inf.st_mtime)
+            sz = inf.st_size
             if stat.S_ISDIR(inf.st_mode):
-                if abspath in excl or abspath == histdir:
+                if abspath in excl or abspath == histpath:
                     continue
                 # self.log(" dir: {}".format(abspath))
-                ret += self._build_dir(dbw, top, excl, abspath)
+                ret += self._build_dir(dbw, top, excl, abspath, nohash)
             else:
                 # self.log("file: {}".format(abspath))
                 rp = abspath[len(top) :].replace("\\", "/").strip("/")
                 rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
-                sql = "select * from up where rd = ? and fn = ?"
+                sql = "select w, mt, sz from up where rd = ? and fn = ?"
                 try:
                     c = dbw[0].execute(sql, (rd, fn))
                 except:
@@ -403,18 +433,18 @@ class Up2k(object):
                 in_db = list(c.fetchall())
                 if in_db:
                     self.pp.n -= 1
-                    _, dts, dsz, _, _ = in_db[0]
+                    dw, dts, dsz = in_db[0]
                     if len(in_db) > 1:
                         m = "WARN: multiple entries: [{}] => [{}] |{}|\n{}"
                         rep_db = "\n".join([repr(x) for x in in_db])
                         self.log(m.format(top, rp, len(in_db), rep_db))
                         dts = -1
 
-                    if dts == lmod and dsz == inf.st_size:
+                    if dts == lmod and dsz == sz and (nohash or dw[0] != "#"):
                         continue
 
                     m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format(
-                        top, rp, dts, lmod, dsz, inf.st_size
+                        top, rp, dts, lmod, dsz, sz
                     )
                     self.log(m)
                     self.db_rm(dbw[0], rd, fn)
@@ -423,17 +453,22 @@ class Up2k(object):
                     in_db = None
 
                 self.pp.msg = "a{} {}".format(self.pp.n, abspath)
-                if inf.st_size > 1024 * 1024:
-                    self.log("file: {}".format(abspath))
 
-                try:
-                    hashes = self._hashlist_from_file(abspath)
-                except Exception as ex:
-                    self.log("hash: {} @ [{}]".format(repr(ex), abspath))
-                    continue
+                if nohash:
+                    wark = up2k_wark_from_metadata(self.salt, sz, lmod, rd, fn)
+                else:
+                    if sz > 1024 * 1024:
+                        self.log("file: {}".format(abspath))
 
-                wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
-                self.db_add(dbw[0], wark, rd, fn, lmod, inf.st_size)
+                    try:
+                        hashes = self._hashlist_from_file(abspath)
+                    except Exception as ex:
+                        self.log("hash: {} @ [{}]".format(repr(ex), abspath))
+                        continue
+
+                    wark = up2k_wark_from_hashlist(self.salt, sz, hashes)
+
+                self.db_add(dbw[0], wark, rd, fn, lmod, sz)
                 dbw[1] += 1
                 ret += 1
                 td = time.time() - dbw[2]
@@ -928,7 +963,7 @@ class Up2k(object):
     def _create_v3(self, cur):
         """
         collision in 2^(n/2) files where n = bits (6 bits/ch)
-          10*6/2 = 2^30 =       1'073'741'824, 24.1mb idx
+          10*6/2 = 2^30 =       1'073'741'824, 24.1mb idx  1<<(3*10)
           12*6/2 = 2^36 =      68'719'476'736, 24.8mb idx
           16*6/2 = 2^48 = 281'474'976'710'656, 26.1mb idx
         """
@@ -976,9 +1011,10 @@ class Up2k(object):
         return self._orz(db_path)
 
     def handle_json(self, cj):
-        if not self.register_vpath(cj["ptop"], cj["vcfg"]):
-            if cj["ptop"] not in self.registry:
-                raise Pebkac(410, "location unavailable")
+        with self.mutex:
+            if not self.register_vpath(cj["ptop"], cj["vcfg"]):
+                if cj["ptop"] not in self.registry:
+                    raise Pebkac(410, "location unavailable")
 
         cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
@@ -1219,6 +1255,7 @@ class Up2k(object):
 
         if "e2t" in self.flags[ptop]:
             self.tagq.put([ptop, wark, rd, fn])
+            self.n_tagq += 1
 
         return True
 
@@ -1344,11 +1381,12 @@ class Up2k(object):
                 for k, reg in self.registry.items():
                     self._snap_reg(prev, k, reg, discard_interval)
 
-    def _snap_reg(self, prev, k, reg, discard_interval):
+    def _snap_reg(self, prev, ptop, reg, discard_interval):
         now = time.time()
+        histpath = self.asrv.vfs.histtab[ptop]
         rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
         if rm:
-            m = "dropping {} abandoned uploads in {}".format(len(rm), k)
+            m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
             vis = [self._vis_job_progress(x) for x in rm]
             self.log("\n".join([m] + vis))
             for job in rm:
@@ -1366,21 +1404,21 @@ class Up2k(object):
                 except:
                     pass
 
-        path = os.path.join(k, ".hist", "up2k.snap")
+        path = os.path.join(histpath, "up2k.snap")
         if not reg:
-            if k not in prev or prev[k] is not None:
-                prev[k] = None
+            if ptop not in prev or prev[ptop] is not None:
+                prev[ptop] = None
                 if os.path.exists(fsenc(path)):
                     os.unlink(fsenc(path))
             return
 
         newest = max(x["poke"] for _, x in reg.items()) if reg else 0
         etag = [len(reg), newest]
-        if etag == prev.get(k, None):
+        if etag == prev.get(ptop, None):
             return
 
         try:
-            os.mkdir(os.path.join(k, ".hist"))
+            os.makedirs(histpath)
         except:
             pass
 
@@ -1392,14 +1430,21 @@ class Up2k(object):
         atomic_move(path2, path)
 
         self.log("snap: {} |{}|".format(path, len(reg.keys())))
-        prev[k] = etag
+        prev[ptop] = etag
 
     def _tagger(self):
+        with self.mutex:
+            self.n_tagq += 1
+
         while True:
+            with self.mutex:
+                self.n_tagq -= 1
+
             ptop, wark, rd, fn = self.tagq.get()
             if "e2t" not in self.flags[ptop]:
                 continue
 
+            # self.log("\n  " + repr([ptop, rd, fn]))
             abspath = os.path.join(ptop, rd, fn)
             tags = self.mtag.get(abspath)
             ntags1 = len(tags)
@@ -1425,8 +1470,16 @@ class Up2k(object):
             self.log("tagged {} ({}+{})".format(abspath, ntags1, len(tags) - ntags1))
 
     def _hasher(self):
+        with self.mutex:
+            self.n_hashq += 1
+
         while True:
+            with self.mutex:
+                self.n_hashq -= 1
+            # self.log("hashq {}".format(self.n_hashq))
+
             ptop, rd, fn = self.hashq.get()
+            # self.log("hashq {} pop {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
             if "e2d" not in self.flags[ptop]:
                 continue
 
@@ -1439,8 +1492,11 @@ class Up2k(object):
                 self.idx_wark(ptop, wark, rd, fn, inf.st_mtime, inf.st_size)
 
     def hash_file(self, ptop, flags, rd, fn):
-        self.register_vpath(ptop, flags)
-        self.hashq.put([ptop, rd, fn])
+        with self.mutex:
+            self.register_vpath(ptop, flags)
+            self.hashq.put([ptop, rd, fn])
+            self.n_hashq += 1
+        # self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
 
 
 def up2k_chunksize(filesize):
@@ -1462,9 +1518,12 @@ def up2k_wark_from_hashlist(salt, filesize, hashes):
     ident.extend(hashes)
     ident = "\n".join(ident)
 
-    hasher = hashlib.sha512()
-    hasher.update(ident.encode("utf-8"))
-    digest = hasher.digest()[:32]
+    wark = hashlib.sha512(ident.encode("utf-8")).digest()
+    wark = base64.urlsafe_b64encode(wark)
+    return wark.decode("ascii")[:43]
 
-    wark = base64.urlsafe_b64encode(digest)
-    return wark.decode("utf-8").rstrip("=")
+
+def up2k_wark_from_metadata(salt, sz, lastmod, rd, fn):
+    ret = fsenc("{}\n{}\n{}\n{}\n{}".format(salt, lastmod, sz, rd, fn))
+    ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest())
+    return "#{}".format(ret[:42].decode("ascii"))

copyparty/util.py

@@ -254,6 +254,17 @@ def trace(*args, **kwargs):
     nuprint(msg)
 
 
+def min_ex():
+    et, ev, tb = sys.exc_info()
+    tb = traceback.extract_tb(tb, 2)
+    ex = [
+        "{} @ {} <{}>: {}".format(fp.split(os.sep)[-1], ln, fun, txt)
+        for fp, ln, fun, txt in tb
+    ]
+    ex.append("{}: {}".format(et.__name__, ev))
+    return "\n".join(ex)
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -568,8 +579,10 @@ def read_header(sr):
             else:
                 continue
 
-        sr.unrecv(ret[ofs + 4 :])
-        return ret[:ofs].decode("utf-8", "surrogateescape").split("\r\n")
+        if len(ret) > ofs + 4:
+            sr.unrecv(ret[ofs + 4 :])
+
+        return ret[:ofs].decode("utf-8", "surrogateescape").lstrip("\r\n").split("\r\n")
 
 
 def humansize(sz, terse=False):
@@ -985,8 +998,8 @@ def guess_mime(url, fallback="application/octet-stream"):
 def runcmd(*argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
-    stdout = stdout.decode("utf-8")
-    stderr = stderr.decode("utf-8")
+    stdout = stdout.decode("utf-8", "replace")
+    stderr = stderr.decode("utf-8", "replace")
     return [p.returncode, stdout, stderr]
 
 

copyparty/web/baguettebox.js

@@ -0,0 +1,583 @@
+/*!
+ * baguetteBox.js
+ * @author  feimosi
+ * @version 1.11.1-mod
+ * @url https://github.com/feimosi/baguetteBox.js
+ */
+
+window.baguetteBox = (function () {
+    'use strict';
+
+    var options = {},
+        defaults = {
+            captions: true,
+            buttons: 'auto',
+            noScrollbars: false,
+            bodyClass: 'baguetteBox-open',
+            titleTag: false,
+            async: false,
+            preload: 2,
+            animation: 'slideIn',
+            afterShow: null,
+            afterHide: null,
+            onChange: null,
+        },
+        overlay, slider, previousButton, nextButton, closeButton,
+        currentGallery = [],
+        currentIndex = 0,
+        isOverlayVisible = false,
+        touch = {},  // start-pos
+        touchFlag = false,  // busy
+        regex = /.+\.(gif|jpe?g|png|webp)/i,
+        data = {},  // all galleries
+        imagesElements = [],
+        documentLastFocus = null;
+
+    var overlayClickHandler = function (event) {
+        if (event.target.id.indexOf('baguette-img') !== -1) {
+            hideOverlay();
+        }
+    };
+
+    var touchstartHandler = function (event) {
+        touch.count++;
+        if (touch.count > 1) {
+            touch.multitouch = true;
+        }
+        touch.startX = event.changedTouches[0].pageX;
+        touch.startY = event.changedTouches[0].pageY;
+    };
+    var touchmoveHandler = function (event) {
+        if (touchFlag || touch.multitouch) {
+            return;
+        }
+        event.preventDefault ? event.preventDefault() : event.returnValue = false;
+        var touchEvent = event.touches[0] || event.changedTouches[0];
+        if (touchEvent.pageX - touch.startX > 40) {
+            touchFlag = true;
+            showPreviousImage();
+        } else if (touchEvent.pageX - touch.startX < -40) {
+            touchFlag = true;
+            showNextImage();
+        } else if (touch.startY - touchEvent.pageY > 100) {
+            hideOverlay();
+        }
+    };
+    var touchendHandler = function () {
+        touch.count--;
+        if (touch.count <= 0) {
+            touch.multitouch = false;
+        }
+        touchFlag = false;
+    };
+    var contextmenuHandler = function () {
+        touchendHandler();
+    };
+
+    var trapFocusInsideOverlay = function (event) {
+        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(event.target))) {
+            event.stopPropagation();
+            initFocus();
+        }
+    };
+
+    function run(selector, userOptions) {
+        buildOverlay();
+        removeFromCache(selector);
+        return bindImageClickListeners(selector, userOptions);
+    }
+
+    function bindImageClickListeners(selector, userOptions) {
+        var galleryNodeList = document.querySelectorAll(selector);
+        var selectorData = {
+            galleries: [],
+            nodeList: galleryNodeList
+        };
+        data[selector] = selectorData;
+
+        [].forEach.call(galleryNodeList, function (galleryElement) {
+            if (userOptions && userOptions.filter) {
+                regex = userOptions.filter;
+            }
+
+            var tagsNodeList = [];
+            if (galleryElement.tagName === 'A') {
+                tagsNodeList = [galleryElement];
+            } else {
+                tagsNodeList = galleryElement.getElementsByTagName('a');
+            }
+
+            tagsNodeList = [].filter.call(tagsNodeList, function (element) {
+                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1) {
+                    return regex.test(element.href);
+                }
+            });
+            if (tagsNodeList.length === 0) {
+                return;
+            }
+
+            var gallery = [];
+            [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
+                var imageElementClickHandler = function (event) {
+                    if (event && event.ctrlKey)
+                        return true;
+
+                    event.preventDefault ? event.preventDefault() : event.returnValue = false;
+                    prepareOverlay(gallery, userOptions);
+                    showOverlay(imageIndex);
+                };
+                var imageItem = {
+                    eventHandler: imageElementClickHandler,
+                    imageElement: imageElement
+                };
+                bind(imageElement, 'click', imageElementClickHandler);
+                gallery.push(imageItem);
+            });
+            selectorData.galleries.push(gallery);
+        });
+
+        return selectorData.galleries;
+    }
+
+    function clearCachedData() {
+        for (var selector in data) {
+            if (data.hasOwnProperty(selector)) {
+                removeFromCache(selector);
+            }
+        }
+    }
+
+    function removeFromCache(selector) {
+        if (!data.hasOwnProperty(selector)) {
+            return;
+        }
+        var galleries = data[selector].galleries;
+        [].forEach.call(galleries, function (gallery) {
+            [].forEach.call(gallery, function (imageItem) {
+                unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
+            });
+
+            if (currentGallery === gallery) {
+                currentGallery = [];
+            }
+        });
+
+        delete data[selector];
+    }
+
+    function buildOverlay() {
+        overlay = ebi('baguetteBox-overlay');
+        if (overlay) {
+            slider = ebi('baguetteBox-slider');
+            previousButton = ebi('previous-button');
+            nextButton = ebi('next-button');
+            closeButton = ebi('close-button');
+            return;
+        }
+        overlay = mknod('div');
+        overlay.setAttribute('role', 'dialog');
+        overlay.id = 'baguetteBox-overlay';
+        document.getElementsByTagName('body')[0].appendChild(overlay);
+
+        slider = mknod('div');
+        slider.id = 'baguetteBox-slider';
+        overlay.appendChild(slider);
+
+        previousButton = mknod('button');
+        previousButton.setAttribute('type', 'button');
+        previousButton.id = 'previous-button';
+        previousButton.setAttribute('aria-label', 'Previous');
+        previousButton.innerHTML = '&lt;';
+        overlay.appendChild(previousButton);
+
+        nextButton = mknod('button');
+        nextButton.setAttribute('type', 'button');
+        nextButton.id = 'next-button';
+        nextButton.setAttribute('aria-label', 'Next');
+        nextButton.innerHTML = '&gt;';
+        overlay.appendChild(nextButton);
+
+        closeButton = mknod('button');
+        closeButton.setAttribute('type', 'button');
+        closeButton.id = 'close-button';
+        closeButton.setAttribute('aria-label', 'Close');
+        closeButton.innerHTML = '&times;';
+        overlay.appendChild(closeButton);
+
+        previousButton.className = nextButton.className = closeButton.className = 'baguetteBox-button';
+
+        bindEvents();
+    }
+
+    function keyDownHandler(event) {
+        switch (event.keyCode) {
+            case 37: // Left
+                showPreviousImage();
+                break;
+            case 39: // Right
+                showNextImage();
+                break;
+            case 27: // Esc
+                hideOverlay();
+                break;
+            case 36: // Home
+                showFirstImage(event);
+                break;
+            case 35: // End
+                showLastImage(event);
+                break;
+        }
+    }
+
+    var passiveSupp = false;
+    try {
+        var opts = {
+            get passive() {
+                passiveSupp = true;
+                return false;
+            }
+        };
+        window.addEventListener('test', null, opts);
+        window.removeEventListener('test', null, opts);
+    }
+    catch (ex) {
+        passiveSupp = false;
+    }
+    var passiveEvent = passiveSupp ? { passive: false } : null;
+    var nonPassiveEvent = passiveSupp ? { passive: true } : null;
+
+    function bindEvents() {
+        bind(overlay, 'click', overlayClickHandler);
+        bind(previousButton, 'click', showPreviousImage);
+        bind(nextButton, 'click', showNextImage);
+        bind(closeButton, 'click', hideOverlay);
+        bind(slider, 'contextmenu', contextmenuHandler);
+        bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        bind(overlay, 'touchend', touchendHandler);
+        bind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function unbindEvents() {
+        unbind(overlay, 'click', overlayClickHandler);
+        unbind(previousButton, 'click', showPreviousImage);
+        unbind(nextButton, 'click', showNextImage);
+        unbind(closeButton, 'click', hideOverlay);
+        unbind(slider, 'contextmenu', contextmenuHandler);
+        unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        unbind(overlay, 'touchend', touchendHandler);
+        unbind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function prepareOverlay(gallery, userOptions) {
+        if (currentGallery === gallery) {
+            return;
+        }
+        currentGallery = gallery;
+        setOptions(userOptions);
+        slider.innerHTML = '';
+        imagesElements.length = 0;
+
+        var imagesFiguresIds = [];
+        var imagesCaptionsIds = [];
+        for (var i = 0, fullImage; i < gallery.length; i++) {
+            fullImage = mknod('div');
+            fullImage.className = 'full-image';
+            fullImage.id = 'baguette-img-' + i;
+            imagesElements.push(fullImage);
+
+            imagesFiguresIds.push('baguetteBox-figure-' + i);
+            imagesCaptionsIds.push('baguetteBox-figcaption-' + i);
+            slider.appendChild(imagesElements[i]);
+        }
+        overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
+        overlay.setAttribute('aria-describedby', imagesCaptionsIds.join(' '));
+    }
+
+    function setOptions(newOptions) {
+        if (!newOptions) {
+            newOptions = {};
+        }
+        for (var item in defaults) {
+            options[item] = defaults[item];
+            if (typeof newOptions[item] !== 'undefined') {
+                options[item] = newOptions[item];
+            }
+        }
+        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
+            options.animation === 'slideIn' ? '' : 'none');
+
+        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1)) {
+            options.buttons = false;
+        }
+
+        previousButton.style.display = nextButton.style.display = (options.buttons ? '' : 'none');
+    }
+
+    function showOverlay(chosenImageIndex) {
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'hidden';
+            document.body.style.overflowY = 'scroll';
+        }
+        if (overlay.style.display === 'block') {
+            return;
+        }
+
+        bind(document, 'keydown', keyDownHandler);
+        currentIndex = chosenImageIndex;
+        touch = {
+            count: 0,
+            startX: null,
+            startY: null
+        };
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+
+        updateOffset();
+        overlay.style.display = 'block';
+        // Fade in overlay
+        setTimeout(function () {
+            overlay.className = 'visible';
+            if (options.bodyClass && document.body.classList) {
+                document.body.classList.add(options.bodyClass);
+            }
+            if (options.afterShow) {
+                options.afterShow();
+            }
+        }, 50);
+        if (options.onChange) {
+            options.onChange(currentIndex, imagesElements.length);
+        }
+        documentLastFocus = document.activeElement;
+        initFocus();
+        isOverlayVisible = true;
+    }
+
+    function initFocus() {
+        if (options.buttons) {
+            previousButton.focus();
+        } else {
+            closeButton.focus();
+        }
+    }
+
+    function hideOverlay(e) {
+        ev(e);
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'auto';
+            document.body.style.overflowY = 'auto';
+        }
+        if (overlay.style.display === 'none') {
+            return;
+        }
+
+        unbind(document, 'keydown', keyDownHandler);
+        // Fade out and hide the overlay
+        overlay.className = '';
+        setTimeout(function () {
+            overlay.style.display = 'none';
+            if (options.bodyClass && document.body.classList) {
+                document.body.classList.remove(options.bodyClass);
+            }
+            if (options.afterHide) {
+                options.afterHide();
+            }
+            documentLastFocus && documentLastFocus.focus();
+            isOverlayVisible = false;
+        }, 500);
+    }
+
+    function loadImage(index, callback) {
+        var imageContainer = imagesElements[index];
+        var galleryItem = currentGallery[index];
+
+        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined') {
+            return;  // out-of-bounds or gallery dirty
+        }
+
+        if (imageContainer.getElementsByTagName('img')[0]) {
+            // image is loaded, cb and bail
+            if (callback) {
+                callback();
+            }
+            return;
+        }
+
+        var imageElement = galleryItem.imageElement,
+            imageSrc = imageElement.href,
+            thumbnailElement = imageElement.getElementsByTagName('img')[0],
+            imageCaption = typeof options.captions === 'function' ?
+                options.captions.call(currentGallery, imageElement) :
+                imageElement.getAttribute('data-caption') || imageElement.title;
+
+        var figure = mknod('figure');
+        figure.id = 'baguetteBox-figure-' + index;
+        figure.innerHTML = '<div class="baguetteBox-spinner">' +
+            '<div class="baguetteBox-double-bounce1"></div>' +
+            '<div class="baguetteBox-double-bounce2"></div>' +
+            '</div>';
+
+        if (options.captions && imageCaption) {
+            var figcaption = mknod('figcaption');
+            figcaption.id = 'baguetteBox-figcaption-' + index;
+            figcaption.innerHTML = imageCaption;
+            figure.appendChild(figcaption);
+        }
+        imageContainer.appendChild(figure);
+
+        var image = mknod('img');
+        image.onload = function () {
+            // Remove loader element
+            var spinner = document.querySelector('#baguette-img-' + index + ' .baguetteBox-spinner');
+            figure.removeChild(spinner);
+            if (!options.async && callback) {
+                callback();
+            }
+        };
+        image.setAttribute('src', imageSrc);
+        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (options.titleTag && imageCaption) {
+            image.title = imageCaption;
+        }
+        figure.appendChild(image);
+
+        if (options.async && callback) {
+            callback();
+        }
+    }
+
+    function showNextImage(e) {
+        ev(e);
+        return show(currentIndex + 1);
+    }
+
+    function showPreviousImage(e) {
+        ev(e);
+        return show(currentIndex - 1);
+    }
+
+    function showFirstImage(event) {
+        if (event) {
+            event.preventDefault();
+        }
+        return show(0);
+    }
+
+    function showLastImage(event) {
+        if (event) {
+            event.preventDefault();
+        }
+        return show(currentGallery.length - 1);
+    }
+
+    /**
+     * Move the gallery to a specific index
+     * @param `index` {number} - the position of the image
+     * @param `gallery` {array} - gallery which should be opened, if omitted assumes the currently opened one
+     * @return {boolean} - true on success or false if the index is invalid
+     */
+    function show(index, gallery) {
+        if (!isOverlayVisible && index >= 0 && index < gallery.length) {
+            prepareOverlay(gallery, options);
+            showOverlay(index);
+            return true;
+        }
+        if (index < 0) {
+            if (options.animation) {
+                bounceAnimation('left');
+            }
+            return false;
+        }
+        if (index >= imagesElements.length) {
+            if (options.animation) {
+                bounceAnimation('right');
+            }
+            return false;
+        }
+
+        currentIndex = index;
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+        updateOffset();
+
+        if (options.onChange) {
+            options.onChange(currentIndex, imagesElements.length);
+        }
+
+        return true;
+    }
+
+    /**
+     * Triggers the bounce animation
+     * @param {('left'|'right')} direction - Direction of the movement
+     */
+    function bounceAnimation(direction) {
+        slider.className = 'bounce-from-' + direction;
+        setTimeout(function () {
+            slider.className = '';
+        }, 400);
+    }
+
+    function updateOffset() {
+        var offset = -currentIndex * 100 + '%';
+        if (options.animation === 'fadeIn') {
+            slider.style.opacity = 0;
+            setTimeout(function () {
+                slider.style.transform = 'translate3d(' + offset + ',0,0)';
+                slider.style.opacity = 1;
+            }, 400);
+        } else {
+            slider.style.transform = 'translate3d(' + offset + ',0,0)';
+        }
+    }
+
+    function preloadNext(index) {
+        if (index - currentIndex >= options.preload) {
+            return;
+        }
+        loadImage(index + 1, function () {
+            preloadNext(index + 1);
+        });
+    }
+
+    function preloadPrev(index) {
+        if (currentIndex - index >= options.preload) {
+            return;
+        }
+        loadImage(index - 1, function () {
+            preloadPrev(index - 1);
+        });
+    }
+
+    function bind(element, event, callback, options) {
+        element.addEventListener(event, callback, options);
+    }
+
+    function unbind(element, event, callback, options) {
+        element.removeEventListener(event, callback, options);
+    }
+
+    function destroyPlugin() {
+        unbindEvents();
+        clearCachedData();
+        unbind(document, 'keydown', keyDownHandler);
+        document.getElementsByTagName('body')[0].removeChild(ebi('baguetteBox-overlay'));
+        data = {};
+        currentGallery = [];
+        currentIndex = 0;
+    }
+
+    return {
+        run: run,
+        show: show,
+        showNext: showNextImage,
+        showPrevious: showPreviousImage,
+        hide: hideOverlay,
+        destroy: destroyPlugin
+    };
+})();

copyparty/web/browser.css

@@ -644,7 +644,6 @@ input[type="checkbox"]:checked+label {
 }
 #treeul a+a {
 	width: calc(100% - 2em);
-	background: #333;
 	line-height: 1em;
 }
 #treeul a+a:hover {
@@ -751,9 +750,12 @@ input[type="checkbox"]:checked+label {
 	font-family: monospace, monospace;
 	line-height: 2em;
 }
-#griden.on+#thumbs {
+#thumbs {
 	opacity: .3;
 }
+#griden.on+#thumbs {
+	opacity: 1;
+}
 #ghead {
 	background: #3c3c3c;
 	border: 1px solid #444;
@@ -798,6 +800,13 @@ html.light #ghead {
 	padding: .2em .3em;
 	display: block;
 }
+#ggrid span.dir:before {
+	content: '📂';
+	line-height: 0;
+	font-size: 2em;
+	display: inline-block;
+	margin: -.7em .1em -.5em -.3em;
+}
 #ggrid a:hover {
 	background: #444;
 	border-color: #555;
@@ -910,6 +919,7 @@ html.light #files {
 }
 html.light #files thead th {
 	background: #eee;
+	border-radius: 0;
 }
 html.light #files tr td {
 	border-top: 1px solid #ddd;
@@ -1021,4 +1031,161 @@ html.light #tree::-webkit-scrollbar {
 }
 #tree::-webkit-scrollbar-thumb {
 	background: #da0;
-}
+}
+
+
+
+#baguetteBox-overlay {
+	display: none;
+	opacity: 0;
+	position: fixed;
+	overflow: hidden;
+	top: 0;
+	left: 0;
+	width: 100%;
+	height: 100%;
+	z-index: 1000000;
+	background: rgba(0, 0, 0, 0.8);
+	transition: opacity .3s ease;
+}
+#baguetteBox-overlay.visible {
+	opacity: 1;
+}
+#baguetteBox-overlay .full-image {
+    display: inline-block;
+    position: relative;
+    width: 100%;
+    height: 100%;
+    text-align: center;
+}
+#baguetteBox-overlay .full-image figure {
+	display: inline;
+	margin: 0;
+	height: 100%;
+}
+#baguetteBox-overlay .full-image img {
+	display: inline-block;
+	width: auto;
+	height: auto;
+	max-height: 100%;
+	max-width: 100%;
+	vertical-align: middle;
+	box-shadow: 0 0 8px rgba(0, 0, 0, 0.6);
+}
+#baguetteBox-overlay .full-image figcaption {
+	display: block;
+	position: absolute;
+	bottom: 0;
+	width: 100%;
+	text-align: center;
+	line-height: 1.8;
+	white-space: normal;
+	color: #ccc;
+}
+#baguetteBox-overlay figcaption a {
+	background: rgba(0, 0, 0, 0.6);
+	border-radius: .4em;
+	padding: .3em .6em;
+}
+#baguetteBox-overlay .full-image:before {
+	content: "";
+	display: inline-block;
+	height: 50%;
+	width: 1px;
+	margin-right: -1px;
+}
+#baguetteBox-slider {
+	position: absolute;
+	left: 0;
+	top: 0;
+	height: 100%;
+	width: 100%;
+	white-space: nowrap;
+	transition: left .2s ease, transform .2s ease;
+}
+#baguetteBox-slider.bounce-from-right {
+	animation: bounceFromRight .4s ease-out;
+}
+#baguetteBox-slider.bounce-from-left {
+	animation: bounceFromLeft .4s ease-out;
+}
+@keyframes bounceFromRight {
+	0% {margin-left: 0}
+	50% {margin-left: -30px}
+	100% {margin-left: 0}
+}
+@keyframes bounceFromLeft {
+	0% {margin-left: 0}
+	50% {margin-left: 30px}
+	100% {margin-left: 0}
+}
+.baguetteBox-button#next-button,
+.baguetteBox-button#previous-button {
+	top: 50%;
+	top: calc(50% - 30px);
+	width: 44px;
+	height: 60px;
+} 
+.baguetteBox-button {
+	position: absolute;
+	cursor: pointer;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	border: 0;
+	border-radius: 15%;
+	background: rgba(50, 50, 50, 0.5);
+	color: #ddd;
+	font: 1.6em sans-serif;
+	transition: background-color .3s ease;
+}
+.baguetteBox-button:focus,
+.baguetteBox-button:hover {
+	background: rgba(50, 50, 50, 0.9);
+}
+#next-button {
+	right: 2%;
+}
+#previous-button {
+	left: 2%;
+}
+#close-button {
+	top: 20px;
+	right: 2%;
+	width: 30px;
+	height: 30px;
+}
+.baguetteBox-button svg {
+	position: absolute;
+	left: 0;
+	top: 0;
+}
+.baguetteBox-spinner {
+	width: 40px;
+	height: 40px;
+	display: inline-block;
+	position: absolute;
+	top: 50%;
+	left: 50%;
+	margin-top: -20px;
+	margin-left: -20px;
+}
+.baguetteBox-double-bounce1,
+.baguetteBox-double-bounce2 {
+	width: 100%;
+	height: 100%;
+	border-radius: 50%;
+	background-color: #fff;
+	opacity: .6;
+	position: absolute;
+	top: 0;
+	left: 0;
+	animation: bounce 2s infinite ease-in-out;
+}
+.baguetteBox-double-bounce2 {
+	animation-delay: -1s;
+}
+@keyframes bounce {
+	0%, 100% {transform: scale(0)}
+	50% {transform: scale(1)}
+}

copyparty/web/browser.html

@@ -8,6 +8,9 @@
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
+    {%- if css %}
+    <link rel="stylesheet" type="text/css" media="screen" href="{{ css }}{{ ts }}">
+    {%- endif %}
 </head>
 
 <body>

copyparty/web/browser.js

@@ -38,7 +38,7 @@ var have_webp = null;
 	img.onerror = function () {
 		have_webp = false;
 	};
-	img.src = "data:image/webp;base64,UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA";
+	img.src = "data:image/webp;base64,UklGRhoAAABXRUJQVlA4TA0AAAAvAAAAEAcQERGIiP4HAA==";
 })();
 
 
@@ -48,7 +48,6 @@ function MPlayer() {
 	this.au = null;
 	this.au_native = null;
 	this.au_ogvjs = null;
-	this.cover_url = '';
 	this.tracks = {};
 	this.order = [];
 
@@ -710,8 +709,9 @@ function autoplay_blocked(seek) {
 
 
 var thegrid = (function () {
-	var lfiles = ebi('files');
-	var gfiles = document.createElement('div');
+	var lfiles = ebi('files'),
+		gfiles = document.createElement('div');
+
 	gfiles.setAttribute('id', 'gfiles');
 	gfiles.style.display = 'none';
 	gfiles.innerHTML = (
@@ -733,7 +733,8 @@ var thegrid = (function () {
 		'en': bcfg_get('griden', false),
 		'sel': bcfg_get('gridsel', false),
 		'sz': fcfg_get('gridsz', 10),
-		'isdirty': true
+		'isdirty': true,
+		'bbox': null
 	};
 
 	ebi('thumbs').onclick = function (e) {
@@ -858,14 +859,15 @@ var thegrid = (function () {
 				href = esc(ao.getAttribute('href')),
 				ref = ao.getAttribute('id'),
 				isdir = href.split('?')[0].slice(-1)[0] == '/',
+				ac = isdir ? ' class="dir"' : '',
 				ihref = href;
 
-			if (isdir) {
-				ihref = '/.cpr/ico/folder'
-			}
-			else if (r.thumbs) {
+			if (r.thumbs) {
 				ihref += (ihref.indexOf('?') === -1 ? '?' : '&') + 'th=' + (have_webp ? 'w' : 'j');
 			}
+			else if (isdir) {
+				ihref = '/.cpr/ico/folder';
+			}
 			else {
 				var ar = href.split('?')[0].split('.');
 				if (ar.length > 1)
@@ -886,14 +888,42 @@ var thegrid = (function () {
 			}
 
 			html.push('<a href="' + href + '" ref="' + ref + '"><img src="' +
-				ihref + '" /><span>' + ao.innerHTML + '</span></a>');
+				ihref + '" /><span' + ac + '>' + ao.innerHTML + '</span></a>');
 		}
 		lfiles.style.display = 'none';
 		gfiles.style.display = 'block';
 		ebi('ggrid').innerHTML = html.join('\n');
+		r.bagit();
 		r.loadsel();
 	}
 
+	r.bagit = function () {
+		if (!window.baguetteBox)
+			return;
+
+		if (r.bbox)
+			baguetteBox.destroy();
+
+		r.bbox = baguetteBox.run('#ggrid', {
+			captions: function (g) {
+				var idx = -1,
+					h = '' + g;
+
+				for (var a = 0; a < r.bbox.length; a++)
+					if (r.bbox[a].imageElement == g)
+						idx = a;
+
+				return '<a download href="' + h +
+					'">' + (idx + 1) + ' / ' + r.bbox.length + ' -- ' +
+					esc(uricom_dec(h.split('/').slice(-1)[0])[0]) + '</a>';
+			}
+		})[0];
+	};
+
+	setTimeout(function () {
+		import_js('/.cpr/baguettebox.js', r.bagit);
+	}, 1);
+
 	if (r.en) {
 		loadgrid();
 	}

copyparty/web/splash.css

@@ -26,10 +26,23 @@ a {
 	border-radius: .2em;
 	padding: .2em .8em;
 }
-td, th {
+table {
+	border-collapse: collapse;
+}
+.vols td,
+.vols th {
 	padding: .3em .6em;
 	text-align: left;
 }
+.num {
+	border-right: 1px solid #bbb;
+}
+.num td {
+	padding: .1em .7em .1em 0;
+}
+.num td:first-child {
+	text-align: right;
+}
 .btns {
 	margin: 1em 0;
 }
@@ -57,4 +70,7 @@ html.dark input {
 	border-radius: .5em;
 	padding: .5em .7em;
 	margin: 0 .5em 0 0;
+}
+html.dark .num {
+	border-color: #777;
 }

copyparty/web/splash.html

@@ -15,16 +15,25 @@
 
         {%- if avol %}
         <h1>admin panel:</h1>
-        <table>
-            <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
-            <tbody>
-                {% for mp in avol %}
-                {%- if mp in vstate and vstate[mp] %}
-                <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
-                {%- endif %}
-                {% endfor %}
-            </tbody>
-        </table>
+        <table><tr><td> <!-- hehehe -->
+            <table class="num">
+                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
+                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
+                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
+                <tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+            </table>
+        </td><td>
+            <table class="vols">
+                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+                <tbody>
+                    {% for mp in avol %}
+                    {%- if mp in vstate and vstate[mp] %}
+                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                    {%- endif %}
+                    {% endfor %}
+                </tbody>
+            </table>
+        </td></tr></table>
         <div class="btns">
             <a href="{{ avol[0] }}?stack">dump stack</a>
         </div>

docs/README.md

@@ -0,0 +1,11 @@
+## [`minimal-up2k.html`](minimal-up2k.html)
+* save as `.epilogue.html` inside a folder to [simplify the ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+## [`browser.css`](browser.css)
+* example for `--css-browser`
+
+## [`rclone.md`](rclone.md)
+* notes on using rclone as a fuse client/server
+
+## [`example.conf`](example.conf)
+* example config file for `-c` which never really happened

docs/browser.css

@@ -0,0 +1,29 @@
+html {
+    background: url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
+}
+#files th {
+    background: rgba(32, 32, 32, 0.9) !important;
+}
+#ops,
+#treeul,
+#files td {
+    background: rgba(32, 32, 32, 0.3) !important;
+}
+
+
+html.light {
+    background: url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
+}
+html.light #files th {
+    background: rgba(255, 255, 255, 0.9) !important;
+}
+html.light #ops,
+html.light #treeul,
+html.light #files td {
+    background: rgba(248, 248, 248, 0.8) !important;
+}
+
+
+#files * {
+    background: transparent !important;
+}

scripts/make-sfx.sh

@@ -208,7 +208,7 @@ find | grep -E '\.css$' | while IFS= read -r f; do
 	}
 	!/\}$/ {printf "%s",$0;next}
 	1
-	' <$f | gsed 's/;\}$/}/' >t
+	' <$f | sed 's/;\}$/}/' >t
 	tmv "$f"
 done
 find | grep -E '\.(js|html)$' | while IFS= read -r f; do
@@ -223,7 +223,7 @@ command -v pigz &&
 	pk='gzip'
 
 echo "$pk"
-find | grep -E '\.(js|css)$' | while IFS= read -r f; do
+find | grep -E '\.(js|css)$' | grep -vF /deps/ | while IFS= read -r f; do
 	echo -n .
 	$pk "$f"
 done

scripts/run-tests.sh

@@ -3,10 +3,13 @@ set -ex
 
 pids=()
 for py in python{2,3}; do
-    $py -m unittest discover -s tests >/dev/null &
+    nice $py -m unittest discover -s tests >/dev/null &
     pids+=($!)
 done
 
+python3 scripts/test/smoketest.py &
+pids+=($!)
+
 for pid in ${pids[@]}; do
     wait $pid
 done

scripts/test/smoketest.py

@@ -0,0 +1,209 @@
+import os
+import sys
+import time
+import shlex
+import shutil
+import signal
+import tempfile
+import requests
+import threading
+import subprocess as sp
+
+
+CPP = []
+
+
+class Cpp(object):
+    def __init__(self, args):
+        args = [sys.executable, "-m", "copyparty"] + args
+        print(" ".join([shlex.quote(x) for x in args]))
+
+        self.ls_pre = set(list(os.listdir()))
+        self.p = sp.Popen(args)
+        # , stdout=sp.PIPE, stderr=sp.PIPE)
+
+        self.t = threading.Thread(target=self._run)
+        self.t.daemon = True
+        self.t.start()
+
+    def _run(self):
+        self.so, self.se = self.p.communicate()
+
+    def stop(self, wait):
+        if wait:
+            os.kill(self.p.pid, signal.SIGINT)
+            self.t.join(timeout=2)
+        else:
+            self.p.kill()  # macos py3.8
+
+    def clean(self):
+        t = os.listdir()
+        for f in t:
+            if f not in self.ls_pre and f.startswith("up."):
+                os.unlink(f)
+
+    def await_idle(self, ub, timeout):
+        req = ["scanning</td><td>False", "hash-q</td><td>0", "tag-q</td><td>0"]
+        lim = int(timeout * 10)
+        u = ub + "?h"
+        for n in range(lim):
+            try:
+                time.sleep(0.1)
+                r = requests.get(u, timeout=0.1)
+                for x in req:
+                    if x not in r.text:
+                        print("ST: {}/{} miss {}".format(n, lim, x))
+                        raise Exception()
+                print("ST: idle")
+                return
+            except:
+                pass
+
+
+def tc1():
+    ub = "http://127.0.0.1:4321/"
+    td = os.path.join("srv", "smoketest")
+    try:
+        shutil.rmtree(td)
+    except:
+        if os.path.exists(td):
+            raise
+
+    for _ in range(10):
+        try:
+            os.mkdir(td)
+        except:
+            time.sleep(0.1)  # win10
+
+    assert os.path.exists(td)
+
+    vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
+    if not os.path.exists(vidp):
+        cmd = "ffmpeg -f lavfi -i testsrc=48x32:3 -t 1 -c:v libx264 -tune animation -preset veryslow -crf 69"
+        sp.check_call(cmd.split(" ") + [vidp])
+
+    with open(vidp, "rb") as f:
+        ovid = f.read()
+
+    args = [
+        "-p4321",
+        "-e2dsa",
+        "-e2tsr",
+        "--no-mutagen",
+        "--th-ff-jpg",
+        "--hist",
+        os.path.join(td, "dbm"),
+    ]
+    pdirs = []
+    hpaths = {}
+
+    for d1 in ["r", "w", "a"]:
+        pdirs.append("{}/{}".format(td, d1))
+        pdirs.append("{}/{}/j".format(td, d1))
+        for d2 in ["r", "w", "a"]:
+            d = os.path.join(td, d1, "j", d2)
+            pdirs.append(d)
+            os.makedirs(d)
+
+    pdirs = [x.replace("\\", "/") for x in pdirs]
+    udirs = [x.split("/", 2)[2] for x in pdirs]
+    perms = [x.rstrip("j/")[-1] for x in pdirs]
+    for pd, ud, p in zip(pdirs, udirs, perms):
+        if ud[-1] == "j":
+            continue
+
+        hp = None
+        if pd.endswith("st/a"):
+            hp = hpaths[ud] = os.path.join(td, "db1")
+        elif pd[:-1].endswith("a/j/"):
+            hpaths[ud] = os.path.join(td, "dbm")
+            hp = None
+        else:
+            hp = "-"
+            hpaths[ud] = os.path.join(pd, ".hist")
+
+        arg = "{}:{}:{}".format(pd, ud, p, hp)
+        if hp:
+            arg += ":chist=" + hp
+
+        args += ["-v", arg]
+
+    # return
+    cpp = Cpp(args)
+    CPP.append(cpp)
+    cpp.await_idle(ub, 3)
+
+    for d in udirs:
+        vid = ovid + "\n{}".format(d).encode("utf-8")
+        try:
+            requests.post(ub + d, data={"act": "bput"}, files={"f": ("a.h264", vid)})
+        except:
+            pass
+
+    cpp.clean()
+
+    # GET permission
+    for d, p in zip(udirs, perms):
+        u = "{}{}/a.h264".format(ub, d)
+        r = requests.get(u)
+        ok = bool(r)
+        if ok != (p in ["a"]):
+            raise Exception("get {} with perm {} at {}".format(ok, p, u))
+
+    # stat filesystem
+    for d, p in zip(pdirs, perms):
+        u = "{}/a.h264".format(d)
+        ok = os.path.exists(u)
+        if ok != (p in ["a", "w"]):
+            raise Exception("stat {} with perm {} at {}".format(ok, p, u))
+
+    # GET thumbnail, vreify contents
+    for d, p in zip(udirs, perms):
+        u = "{}{}/a.h264?th=j".format(ub, d)
+        r = requests.get(u)
+        ok = bool(r and r.content[:3] == b"\xff\xd8\xff")
+        if ok != (p in ["a"]):
+            raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
+
+    # check tags
+    cpp.await_idle(ub, 5)
+    for d, p in zip(udirs, perms):
+        u = "{}{}?ls".format(ub, d)
+        r = requests.get(u)
+        j = r.json() if r else False
+        tag = None
+        if j:
+            for f in j["files"]:
+                tag = tag or f["tags"].get("res")
+
+        r_ok = bool(j)
+        w_ok = bool(r_ok and j.get("files"))
+
+        if not r_ok or w_ok != (p in ["a"]):
+            raise Exception("ls {} with perm {} at {}".format(ok, p, u))
+
+        if (tag and p != "a") or (not tag and p == "a"):
+            raise Exception("tag {} with perm {} at {}".format(tag, p, u))
+
+        if tag is not None and tag != "48x32":
+            raise Exception("tag [{}] at {}".format(tag, u))
+
+    cpp.stop(True)
+
+
+def run(tc):
+    try:
+        tc()
+    finally:
+        try:
+            CPP[0].stop(False)
+        except:
+            pass
+
+
+def main():
+    run(tc1)
+
+
+if __name__ == "__main__":
+    main()

tests/test_httpcli.py

@@ -37,6 +37,9 @@ class Cfg(Namespace):
             nih=True,
             mtp=[],
             mte="a",
+            hist=None,
+            no_hash=False,
+            css_browser=None,
             **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
         )
 
@@ -99,7 +102,7 @@ class TestHttpCli(unittest.TestCase):
             pprint.pprint(vcfg)
 
             self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
-            self.auth = AuthSrv(self.args, self.log)
+            self.asrv = AuthSrv(self.args, self.log)
             vfiles = [x for x in allfiles if x.startswith(top)]
             for fp in vfiles:
                 rok, wok = self.can_rw(fp)
@@ -188,12 +191,12 @@ class TestHttpCli(unittest.TestCase):
     def put(self, url):
         buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
         buf = buf.format(url, len(url) + 4).encode("utf-8")
-        conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
         HttpCli(conn).run()
         return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
 
     def curl(self, url, binary=False):
-        conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
         HttpCli(conn).run()
         if binary:
             h, b = conn.s._reply.split(b"\r\n\r\n", 1)

tests/test_vfs.py

@@ -18,8 +18,14 @@ from copyparty import util
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
         ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
-        ex["mtp"] = []
-        ex["mte"] = "a"
+        ex2 = {
+            "mtp": [],
+            "mte": "a",
+            "hist": None,
+            "no_hash": False,
+            "css_browser": None,
+        }
+        ex.update(ex2)
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
 
 
@@ -113,13 +119,13 @@ class TestVFS(unittest.TestCase):
         n = vfs.nodes["a"]
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(n.vpath, "a")
-        self.assertEqual(n.realpath, td + "/a")
+        self.assertEqual(n.realpath, os.path.join(td, "a"))
         self.assertEqual(n.uread, ["*", "k"])
         self.assertEqual(n.uwrite, ["k"])
         n = n.nodes["ac"]
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(n.vpath, "a/ac")
-        self.assertEqual(n.realpath, td + "/a/ac")
+        self.assertEqual(n.realpath, os.path.join(td, "a", "ac"))
         self.assertEqual(n.uread, ["*", "k"])
         self.assertEqual(n.uwrite, ["k"])
         n = n.nodes["acb"]

tests/util.py

@@ -60,7 +60,7 @@ def get_ramdisk():
 
     if os.path.exists("/Volumes"):
         # hdiutil eject /Volumes/cptd/
-        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://65536")
+        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://131072")
         devname = devname.strip()
         print("devname: [{}]".format(devname))
         for _ in range(10):
@@ -110,12 +110,12 @@ class VHttpSrv(object):
 
 
 class VHttpConn(object):
-    def __init__(self, args, auth, log, buf):
+    def __init__(self, args, asrv, log, buf):
         self.s = VSock(buf)
         self.sr = Unrecv(self.s)
         self.addr = ("127.0.0.1", "42069")
         self.args = args
-        self.auth = auth
+        self.asrv = asrv
         self.is_mp = False
         self.log_func = log
         self.log_src = "a"