v0.7.3

.gitattributes

@@ -1,4 +1,6 @@
 * text eol=lf
 
+*.reg text eol=crlf
+
 *.png binary
 *.gif binary

.vscode/launch.json

@@ -12,10 +12,12 @@
                 //"-nw",
                 "-ed",
                 "-emp",
+                "-e2d",
+                "-e2s",
                 "-a",
                 "ed:wark",
                 "-v",
-                "srv::r:aed"
+                "srv::r:aed:cnodupe"
             ]
         },
         {

.vscode/tasks.json

@@ -0,0 +1,10 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "pre",
+            "command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
+            "type": "shell"
+        }
+    ]
+}

README.md

@@ -13,6 +13,17 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * code standard: `black`
 
 
+## quickstart
+
+download [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and you're all set!
+
+running the sfx without arguments (for example doubleclicking it on Windows) will let anyone access the current folder; see `-h` for help if you want accounts and volumes etc
+
+you may also want these, especially on servers:
+* [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
+* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for legit https)
+
+
 ## notes
 
 * iPhone/iPad: use Firefox to download files
@@ -126,13 +137,15 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
-* up2k handle filename too long
-* up2k fails on empty files? alert then stuck
+* reduce up2k roundtrips
+  * start from a chunk index and just go
+  * terminate client on bad data
 * drop onto folders
-* look into android thumbnail cache file format
+* `os.copy_file_range` for up2k cloning
+* up2k partials ui
 * support pillow-simd
 * cache sha512 chunks on client
-* symlink existing files on upload
 * comment field
+* ~~look into android thumbnail cache file format~~ bad idea
 * figure out the deal with pixel3a not being connectable as hotspot
   * pixel3a having unpredictable 3sec latency in general :||||

bin/README.md

@@ -34,3 +34,8 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 * does the same thing except more correct, `samba` approves
 * **supports Linux** -- expect `18 MiB/s` (wait what)
 * **supports Macos** -- probably
+
+
+
+# copyparty-fuse-streaming.py
+* pretend this doesn't exist

bin/copyparty-fuse.py

@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
 mount a copyparty server (local or remote) as a filesystem
 
 usage:
-  python copyparty-fuse.py ./music http://192.168.1.69:3923/
+  python copyparty-fuse.py http://192.168.1.69:3923/  ./music
 
 dependencies:
   python3 -m pip install --user fusepy
@@ -20,6 +20,10 @@ dependencies:
   + on Macos: https://osxfuse.github.io/
   + on Windows: https://github.com/billziss-gh/winfsp/releases/latest
 
+note:
+  you probably want to run this on windows clients:
+  https://github.com/9001/copyparty/blob/master/contrib/explorer-nothumbs-nofoldertypes.reg
+
 get server cert:
   awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
 """
@@ -100,7 +104,7 @@ def rice_tid():
 
 
 def fancy_log(msg):
-    print("{} {}\n".format(rice_tid(), msg), end="")
+    print("{:10.6f} {} {}\n".format(time.time() % 900, rice_tid(), msg), end="")
 
 
 def null_log(msg):
@@ -159,7 +163,7 @@ class RecentLog(object):
         thr.start()
 
     def put(self, msg):
-        msg = "{} {}\n".format(rice_tid(), msg)
+        msg = "{:10.6f} {} {}\n".format(time.time() % 900, rice_tid(), msg)
         if self.f:
             fmsg = " ".join([datetime.utcnow().strftime("%H%M%S.%f"), str(msg)])
             self.f.write(fmsg.encode("utf-8"))
@@ -367,7 +371,7 @@ class Gateway(object):
         ret = []
         remainder = b""
         ptn = re.compile(
-            r'^<tr><td>(-|DIR)</td><td><a[^>]* href="([^"]+)"[^>]*>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$'
+            r'^<tr><td>(-|DIR|<a [^<]+</a>)</td><td><a[^>]* href="([^"]+)"[^>]*>([^<]+)</a></td><td>([^<]+)</td><td>[^<]+</td><td>([^<]+)</td></tr>$'
         )
 
         while True:
@@ -405,7 +409,7 @@ class Gateway(object):
                     info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
                     # python cannot strptime(1959-01-01) on windows
 
-                if ftype == "-":
+                if ftype != "DIR":
                     ret.append([fname, self.stat_file(ts, sz), 0])
                 else:
                     ret.append([fname, self.stat_dir(ts, sz), 0])
@@ -658,8 +662,18 @@ class CPPF(Operations):
 
         else:
             if get2 - get1 <= 1024 * 1024:
-                h_ofs = get1 - 256 * 1024
-                h_end = get2 + 1024 * 1024
+                # unless the request is for the last n bytes of the file,
+                # grow the start to cache some stuff around the range
+                if get2 < file_sz - 1:
+                    h_ofs = get1 - 1024 * 256
+                else:
+                    h_ofs = get1 - 1024 * 32
+
+                # likewise grow the end unless start is 0
+                if get1 > 0:
+                    h_end = get2 + 1024 * 1024
+                else:
+                    h_end = get2 + 1024 * 64
             else:
                 # big enough, doesn't need pads
                 h_ofs = get1
@@ -705,6 +719,7 @@ class CPPF(Operations):
             self.dircache.append(cn)
             self.clean_dircache()
 
+        # import pprint; pprint.pprint(ret)
         return ret
 
     def readdir(self, path, fh=None):
@@ -802,7 +817,11 @@ class CPPF(Operations):
                 # dbg("=" + repr(cache_stat))
                 return cache_stat
 
-        info("=ENOENT ({})".format(hexler(path)))
+        fun = info
+        if MACOS and path.split('/')[-1].startswith('._'):
+            fun = dbg
+        
+        fun("=ENOENT ({})".format(hexler(path)))
         raise FuseOSError(errno.ENOENT)
 
     access = None

contrib/README.md

@@ -9,6 +9,9 @@
 * assumes the webserver and copyparty is running on the same server/IP
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
+### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
+disables thumbnails and folder-type detection in windows explorer, makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+
 # OS integration
 init-scripts to start copyparty as a service
 * [`systemd/copyparty.service`](systemd/copyparty.service)

contrib/explorer-nothumbs-nofoldertypes.reg

@@ -0,0 +1,31 @@
+Windows Registry Editor Version 5.00
+
+; this will do 3 things, all optional:
+;  1) disable thumbnails
+;  2) delete all existing folder type settings/detections
+;  3) disable folder type detection (force default columns)
+;
+; this makes the file explorer way faster,
+; especially on slow/networked locations
+
+
+; =====================================================================
+; 1) disable thumbnails
+
+[HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced]
+"IconsOnly"=dword:00000001
+
+
+; =====================================================================
+; 2) delete all existing folder type settings/detections
+
+[-HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\Bags]
+
+[-HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\BagMRU]
+
+
+; =====================================================================
+; 3) disable folder type detection
+
+[HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\Bags\AllFolders\Shell]
+"FolderType"="NotSpecified"

copyparty/__main__.py

@@ -105,17 +105,22 @@ def main():
         epilog=dedent(
             """
             -a takes username:password,
-            -v takes src:dst:permset:permset:... where "permset" is
-               accesslevel followed by username (no separator)
+            -v takes src:dst:permset:permset:cflag:cflag:...
+               where "permset" is accesslevel followed by username (no separator)
+               and "cflag" is config flags to set on this volume
             
+            list of cflags:
+              cnodupe rejects existing files (instead of symlinking them)
+
             example:\033[35m
-              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed  \033[36m
+              -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe  \033[36m
               mount current directory at "/" with
                * r (read-only) for everyone
                * a (read+write) for ed
               mount ../inc at "/dump" with
                * w (write-only) for everyone
-               * a (read+write) for ed  \033[0m
+               * a (read+write) for ed
+               * reject duplicate files  \033[0m
             
             if no accounts or volumes are configured,
             current folder will be read/write for everyone
@@ -125,9 +130,10 @@ def main():
             """
         ),
     )
+    # fmt: off
     ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind")
-    ap.add_argument("-p", metavar="PORT", type=int, default=3923, help="port to bind")
+    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
     ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
     ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
     ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
@@ -135,11 +141,25 @@ def main():
     ap.add_argument("-q", action="store_true", help="quiet")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap.add_argument("-e2d", action="store_true", help="enable up2k database")
+    ap.add_argument("-e2s", action="store_true", help="enable up2k db-scanner")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     al = ap.parse_args()
+    # fmt: on
+
+    al.i = al.i.split(",")
+    try:
+        if "-" in al.p:
+            lo, hi = [int(x) for x in al.p.split("-")]
+            al.p = list(range(lo, hi + 1))
+        else:
+            al.p = [int(x) for x in al.p.split(",")]
+    except:
+        raise Exception("invalid value for -p")
 
     SvcHub(al).run()
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 6, 2)
-CODENAME = "CHRISTMAAAAAS"
-BUILD_DT = (2020, 12, 14)
+VERSION = (0, 7, 3)
+CODENAME = "keeping track"
+BUILD_DT = (2021, 2, 3)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -12,11 +12,12 @@ from .util import undot, Pebkac, fsdec, fsenc
 class VFS(object):
     """single level in the virtual fs"""
 
-    def __init__(self, realpath, vpath, uread=[], uwrite=[]):
+    def __init__(self, realpath, vpath, uread=[], uwrite=[], flags={}):
         self.realpath = realpath  # absolute path on host filesystem
         self.vpath = vpath  # absolute path in the virtual filesystem
         self.uread = uread  # users who can read this
         self.uwrite = uwrite  # users who can write this
+        self.flags = flags  # config switches
         self.nodes = {}  # child nodes
 
     def add(self, src, dst):
@@ -36,6 +37,7 @@ class VFS(object):
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
                 self.uwrite,
+                self.flags,
             )
             self.nodes[name] = vn
             return vn.add(src, dst)
@@ -128,11 +130,10 @@ class VFS(object):
 class AuthSrv(object):
     """verifies users against given paths"""
 
-    def __init__(self, args, log_func):
-        self.log_func = log_func
+    def __init__(self, args, log_func, warn_anonwrite=True):
         self.args = args
-
-        self.warn_anonwrite = True
+        self.log_func = log_func
+        self.warn_anonwrite = warn_anonwrite
 
         if WINDOWS:
             self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -161,7 +162,7 @@ class AuthSrv(object):
 
         yield prev, True
 
-    def _parse_config_file(self, fd, user, mread, mwrite, mount):
+    def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount):
         vol_src = None
         vol_dst = None
         for ln in [x.decode("utf-8").strip() for x in fd]:
@@ -191,6 +192,7 @@ class AuthSrv(object):
                 mount[vol_dst] = vol_src
                 mread[vol_dst] = []
                 mwrite[vol_dst] = []
+                mflags[vol_dst] = {}
                 continue
 
             lvl, uname = ln.split(" ")
@@ -198,6 +200,9 @@ class AuthSrv(object):
                 mread[vol_dst].append(uname)
             if lvl in "wa":
                 mwrite[vol_dst].append(uname)
+            if lvl == "c":
+                # config option, currently switches only
+                mflags[vol_dst][uname] = True
 
     def reload(self):
         """
@@ -210,6 +215,7 @@ class AuthSrv(object):
         user = {}  # username:password
         mread = {}  # mountpoint:[username]
         mwrite = {}  # mountpoint:[username]
+        mflags = {}  # mountpoint:[flag]
         mount = {}  # dst:src (mountpoint:realpath)
 
         if self.args.a:
@@ -232,9 +238,13 @@ class AuthSrv(object):
                 mount[dst] = src
                 mread[dst] = []
                 mwrite[dst] = []
+                mflags[dst] = {}
 
                 perms = perms.split(":")
                 for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
+                    if lvl == "c":
+                        # config option, currently switches only
+                        mflags[dst][uname] = True
                     if uname == "":
                         uname = "*"
                     if lvl in "ra":
@@ -245,14 +255,15 @@ class AuthSrv(object):
         if self.args.c:
             for cfg_fn in self.args.c:
                 with open(cfg_fn, "rb") as f:
-                    self._parse_config_file(f, user, mread, mwrite, mount)
+                    self._parse_config_file(f, user, mread, mwrite, mflags, mount)
 
+        self.all_writable = []
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(os.path.abspath("."), "", [], [])
+            vfs = VFS(os.path.abspath("."), "")
 
         maxdepth = 0
         for dst in sorted(mount.keys(), key=lambda x: (x.count("/"), len(x))):
@@ -262,12 +273,18 @@ class AuthSrv(object):
 
             if dst == "":
                 # rootfs was mapped; fully replaces the default CWD vfs
-                vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst])
+                vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst], mflags[dst])
                 continue
 
             v = vfs.add(mount[dst], dst)
             v.uread = mread[dst]
             v.uwrite = mwrite[dst]
+            v.flags = mflags[dst]
+            if v.uwrite:
+                self.all_writable.append(v)
+
+        if vfs.uwrite and vfs not in self.all_writable:
+            self.all_writable.append(vfs)
 
         missing_users = {}
         for d in [mread, mwrite]:

copyparty/broker_mpw.py

@@ -73,7 +73,7 @@ class MpWorker(object):
                 if PY2:
                     sck = pickle.loads(sck)  # nosec
 
-                self.log("%s %s" % addr, "-" * 4 + "C-qpop")
+                self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:

copyparty/broker_thr.py

@@ -28,7 +28,7 @@ class BrokerThr(object):
     def put(self, want_retval, dest, *args):
         if dest == "httpconn":
             sck, addr = args
-            self.log("%s %s" % addr, "-" * 4 + "C-qpop")
+            self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
             self.httpsrv.accept(sck, addr)
 
         else:

copyparty/httpcli.py

@@ -28,6 +28,7 @@ class HttpCli(object):
         self.conn = conn
         self.s = conn.s
         self.sr = conn.sr
+        self.ip = conn.addr[0]
         self.addr = conn.addr
         self.args = conn.args
         self.auth = conn.auth
@@ -42,7 +43,7 @@ class HttpCli(object):
         self.log_func(self.log_src, msg)
 
     def _check_nonfatal(self, ex):
-        return ex.code in [404]
+        return ex.code < 400 or ex.code == 404
 
     def _assert_safe_rem(self, rem):
         # sanity check to prevent any disasters
@@ -85,7 +86,8 @@ class HttpCli(object):
 
         v = self.headers.get("x-forwarded-for", None)
         if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
-            self.log_src = self.conn.set_rproxy(v.split(",")[0])
+            self.ip = v.split(",")[0]
+            self.log_src = self.conn.set_rproxy(self.ip)
 
         self.uname = "*"
         if "cookie" in self.headers:
@@ -305,7 +307,7 @@ class HttpCli(object):
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
         fdir = os.path.join(vfs.realpath, rem)
 
-        addr = self.conn.addr[0].replace(":", ".")
+        addr = self.ip.replace(":", ".")
         fn = "put-{:.6f}-{}.bin".format(time.time(), addr)
         path = os.path.join(fdir, fn)
 
@@ -384,9 +386,11 @@ class HttpCli(object):
 
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
 
-        body["vdir"] = self.vpath
-        body["rdir"] = os.path.join(vfs.realpath, rem)
-        body["addr"] = self.addr[0]
+        body["vtop"] = vfs.vpath
+        body["ptop"] = vfs.realpath
+        body["prel"] = rem
+        body["addr"] = self.ip
+        body["flag"] = vfs.flags
 
         x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body)
         response = x.get()
@@ -408,7 +412,10 @@ class HttpCli(object):
         except KeyError:
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
-        x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", wark, chash)
+        vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        ptop = vfs.realpath
+
+        x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
         response = x.get()
         chunksize, cstart, path, lastmod = response
 
@@ -453,8 +460,8 @@ class HttpCli(object):
 
                 self.log("clone {} done".format(cstart[0]))
 
-        x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", wark, chash)
-        num_left = x.get()
+        x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", ptop, wark, chash)
+        num_left, path = x.get()
 
         if not WINDOWS and num_left == 0:
             times = (int(time.time()), int(lastmod))
@@ -575,7 +582,7 @@ class HttpCli(object):
                     if not os.path.isdir(fsenc(fdir)):
                         raise Pebkac(404, "that folder does not exist")
 
-                    suffix = ".{:.6f}-{}".format(time.time(), self.addr[0])
+                    suffix = ".{:.6f}-{}".format(time.time(), self.ip)
                     open_args = {"fdir": fdir, "suffix": suffix}
                 else:
                     open_args = {}
@@ -637,7 +644,7 @@ class HttpCli(object):
                         "\n".join(
                             unicode(x)
                             for x in [
-                                ":".join(unicode(x) for x in self.addr),
+                                ":".join(unicode(x) for x in [self.ip, self.addr[1]]),
                                 msg.rstrip(),
                             ]
                         )
@@ -686,7 +693,7 @@ class HttpCli(object):
             return True
 
         fp = os.path.join(vfs.realpath, rem)
-        srv_lastmod = -1
+        srv_lastmod = srv_lastmod3 = -1
         try:
             st = os.stat(fsenc(fp))
             srv_lastmod = st.st_mtime
@@ -790,6 +797,8 @@ class HttpCli(object):
                 editions[ext or "plain"] = [fs_path, st.st_size]
             except:
                 pass
+            if not self.vpath.startswith(".cpr/"):
+                break
 
         if not editions:
             raise Pebkac(404)
@@ -884,6 +893,7 @@ class HttpCli(object):
 
             logtail += " [\033[36m{}-{}\033[0m]".format(lower, upper)
 
+        use_sendfile = False
         if decompress:
             open_func = gzip.open
             open_args = [fsenc(fs_path), "rb"]
@@ -893,6 +903,8 @@ class HttpCli(object):
             open_func = open
             # 512 kB is optimal for huge files, use 64k
             open_args = [fsenc(fs_path), "rb", 64 * 1024]
+            if hasattr(os, "sendfile"):
+                use_sendfile = not self.args.no_sendfile
 
         #
         # send reply
@@ -915,24 +927,13 @@ class HttpCli(object):
 
         ret = True
         with open_func(*open_args) as f:
-            remains = upper - lower
-            f.seek(lower)
-            while remains > 0:
-                # time.sleep(0.01)
-                buf = f.read(4096)
-                if not buf:
-                    break
+            if use_sendfile:
+                remains = sendfile_kern(lower, upper, f, self.s)
+            else:
+                remains = sendfile_py(lower, upper, f, self.s)
 
-                if remains < len(buf):
-                    buf = buf[:remains]
-
-                try:
-                    self.s.sendall(buf)
-                    remains -= len(buf)
-                except:
-                    logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
-                    ret = False
-                    break
+        if remains > 0:
+            logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
 
         spd = self._spd((upper - lower) - remains)
         self.log("{},  {}".format(logmsg, spd))
@@ -1028,6 +1029,10 @@ class HttpCli(object):
             if abspath.endswith(".md") and "raw" not in self.uparam:
                 return self.tx_md(abspath)
 
+            bad = "{0}.hist{0}up2k.".format(os.sep)
+            if abspath.endswith(bad + "db") or abspath.endswith(bad + "snap"):
+                raise Pebkac(403)
+
             return self.tx_file(abspath)
 
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)

copyparty/httpconn.py

@@ -65,6 +65,7 @@ class HttpConn(object):
             color = 34
             self.rproxy = ip
 
+        self.ip = ip
         self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
         return self.log_src
 

copyparty/httpsrv.py

@@ -38,7 +38,7 @@ class HttpSrv(object):
 
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        self.log("%s %s" % addr, "-" * 5 + "C-cthr")
+        self.log("%s %s" % addr, "\033[1;30m|%sC-cthr\033[0m" % ("-" * 5,))
         thr = threading.Thread(target=self.thr_client, args=(sck, addr))
         thr.daemon = True
         thr.start()
@@ -66,11 +66,11 @@ class HttpSrv(object):
                 thr.start()
 
         try:
-            self.log("%s %s" % addr, "-" * 6 + "C-crun")
+            self.log("%s %s" % addr, "\033[1;30m|%sC-crun\033[0m" % ("-" * 6,))
             cli.run()
 
         finally:
-            self.log("%s %s" % addr, "-" * 7 + "C-done")
+            self.log("%s %s" % addr, "\033[1;30m|%sC-cdone\033[0m" % ("-" * 7,))
             try:
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()

copyparty/svchub.py

@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
 import calendar
 
 from .__init__ import PY2, WINDOWS, MACOS, VT100
+from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
 from .util import mp
@@ -38,6 +39,10 @@ class SvcHub(object):
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
+        if self.args.e2d and self.args.e2s:
+            auth = AuthSrv(self.args, self.log, False)
+            self.up2k.build_indexes(auth.all_writable)
+
         # decide which worker impl to use
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker

copyparty/tcpsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import time
 import socket
+import select
 
 from .util import chkcmd, Counter
 
@@ -23,55 +24,75 @@ class TcpSrv(object):
 
         ip = "127.0.0.1"
         eps = {ip: "local only"}
-        if self.args.i != ip:
-            eps = self.detect_interfaces(self.args.i) or {self.args.i: "external"}
+        nonlocals = [x for x in self.args.i if x != ip]
+        if nonlocals:
+            eps = self.detect_interfaces(self.args.i)
+            if not eps:
+                for x in nonlocals:
+                    eps[x] = "external"
 
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
-            self.log(
-                "tcpsrv",
-                "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
-                    ip, self.args.p, desc
-                ),
-            )
+            for port in sorted(self.args.p):
+                self.log(
+                    "tcpsrv",
+                    "available @ http://{}:{}/  (\033[33m{}\033[0m)".format(
+                        ip, port, desc
+                    ),
+                )
 
-        self.srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.srv = []
+        for ip in self.args.i:
+            for port in self.args.p:
+                self.srv.append(self._listen(ip, port))
+
+    def _listen(self, ip, port):
+        srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         try:
-            self.srv.bind((self.args.i, self.args.p))
+            srv.bind((ip, port))
+            return srv
         except (OSError, socket.error) as ex:
             if ex.errno == 98:
                 raise Exception(
-                    "\033[1;31mport {} is busy on interface {}\033[0m".format(
-                        self.args.p, self.args.i
-                    )
+                    "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
                 )
 
             if ex.errno == 99:
                 raise Exception(
-                    "\033[1;31minterface {} does not exist\033[0m".format(self.args.i)
+                    "\033[1;31minterface {} does not exist\033[0m".format(ip)
                 )
 
     def run(self):
-        self.srv.listen(self.args.nc)
-
-        self.log("tcpsrv", "listening @ {0}:{1}".format(self.args.i, self.args.p))
+        for srv in self.srv:
+            srv.listen(self.args.nc)
+            ip, port = srv.getsockname()
+            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
 
         while True:
-            self.log("tcpsrv", "-" * 1 + "C-ncli")
+            self.log("tcpsrv", "\033[1;30m|%sC-ncli\033[0m" % ("-" * 1,))
             if self.num_clients.v >= self.args.nc:
                 time.sleep(0.1)
                 continue
 
-            self.log("tcpsrv", "-" * 2 + "C-acc1")
-            sck, addr = self.srv.accept()
-            self.log("%s %s" % addr, "-" * 3 + "C-acc2")
-            self.num_clients.add()
-            self.hub.broker.put(False, "httpconn", sck, addr)
+            self.log("tcpsrv", "\033[1;30m|%sC-acc1\033[0m" % ("-" * 2,))
+            ready, _, _ = select.select(self.srv, [], [])
+            for srv in ready:
+                sck, addr = srv.accept()
+                sip, sport = srv.getsockname()
+                self.log(
+                    "%s %s" % addr,
+                    "\033[1;30m|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                        "-" * 3, sip, sport % 8, sport
+                    ),
+                )
+                self.num_clients.add()
+                self.hub.broker.put(False, "httpconn", sck, addr)
 
     def shutdown(self):
         self.log("tcpsrv", "ok bye")
 
-    def detect_interfaces(self, listen_ip):
+    def detect_interfaces(self, listen_ips):
         eps = {}
 
         # get all ips and their interfaces
@@ -85,8 +106,9 @@ class TcpSrv(object):
             for ln in ip_addr.split("\n"):
                 try:
                     ip, dev = r.match(ln.rstrip()).groups()
-                    if listen_ip in ["0.0.0.0", ip]:
-                        eps[ip] = dev
+                    for lip in listen_ips:
+                        if lip in ["0.0.0.0", ip]:
+                            eps[ip] = dev
                 except:
                     pass
 
@@ -113,11 +135,12 @@ class TcpSrv(object):
 
         s.close()
 
-        if default_route and listen_ip in ["0.0.0.0", default_route]:
-            desc = "\033[32mexternal"
-            try:
-                eps[default_route] += ", " + desc
-            except:
-                eps[default_route] = desc
+        for lip in listen_ips:
+            if default_route and lip in ["0.0.0.0", default_route]:
+                desc = "\033[32mexternal"
+                try:
+                    eps[default_route] += ", " + desc
+                except:
+                    eps[default_route] = desc
 
         return eps

copyparty/up2k.py

@@ -6,6 +6,9 @@ import os
 import re
 import time
 import math
+import json
+import gzip
+import stat
 import shutil
 import base64
 import hashlib
@@ -13,7 +16,15 @@ import threading
 from copy import deepcopy
 
 from .__init__ import WINDOWS
-from .util import Pebkac, Queue, fsenc, sanitize_fn, ren_open
+from .util import Pebkac, Queue, fsdec, fsenc, sanitize_fn, ren_open, atomic_move
+
+HAVE_SQLITE3 = False
+try:
+    import sqlite3
+
+    HAVE_SQLITE3 = True
+except:
+    pass
 
 
 class Up2k(object):
@@ -22,20 +33,21 @@ class Up2k(object):
       * documentation
       * registry persistence
         * ~/.config flatfiles for active jobs
-        * wark->path database for finished uploads
     """
 
     def __init__(self, broker):
         self.broker = broker
         self.args = broker.args
         self.log = broker.log
+        self.persist = self.args.e2d
 
         # config
         self.salt = "hunter2"  # TODO: config
 
         # state
-        self.registry = {}
         self.mutex = threading.Lock()
+        self.registry = {}
+        self.db = {}
 
         if WINDOWS:
             # usually fails to set lastmod too quickly
@@ -44,53 +56,306 @@ class Up2k(object):
             thr.daemon = True
             thr.start()
 
+        if self.persist:
+            thr = threading.Thread(target=self._snapshot)
+            thr.daemon = True
+            thr.start()
+
         # static
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$")
 
+        if self.persist and not HAVE_SQLITE3:
+            m = "could not initialize sqlite3, will use in-memory registry only"
+            self.log("up2k", m)
+
+    def _vis_job_progress(self, job):
+        perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
+        path = os.path.join(job["ptop"], job["prel"], job["name"])
+        return "{:5.1f}% {}".format(perc, path)
+
+    def _vis_reg_progress(self, reg):
+        ret = []
+        for _, job in reg.items():
+            ret.append(self._vis_job_progress(job))
+
+        return ret
+
+    def register_vpath(self, ptop):
+        with self.mutex:
+            if ptop in self.registry:
+                return None
+
+            reg = {}
+            path = os.path.join(ptop, ".hist", "up2k.snap")
+            if self.persist and os.path.exists(path):
+                with gzip.GzipFile(path, "rb") as f:
+                    j = f.read().decode("utf-8")
+
+                reg = json.loads(j)
+                for _, job in reg.items():
+                    job["poke"] = time.time()
+
+                m = "loaded snap {} |{}|".format(path, len(reg.keys()))
+                m = [m] + self._vis_reg_progress(reg)
+                self.log("up2k", "\n".join(m))
+
+            self.registry[ptop] = reg
+            if not self.persist or not HAVE_SQLITE3:
+                return None
+
+            try:
+                os.mkdir(os.path.join(ptop, ".hist"))
+            except:
+                pass
+
+            db_path = os.path.join(ptop, ".hist", "up2k.db")
+            if ptop in self.db:
+                # self.db[ptop].close()
+                return None
+
+            try:
+                db = self._open_db(db_path)
+                self.db[ptop] = db
+                return db
+            except Exception as ex:
+                m = "failed to open [{}]: {}".format(ptop, repr(ex))
+                self.log("up2k", m)
+
+            return None
+
+    def build_indexes(self, writeables):
+        tops = [d.realpath for d in writeables]
+        for top in tops:
+            db = self.register_vpath(top)
+            if db:
+                # can be symlink so don't `and d.startswith(top)``
+                excl = set([d for d in tops if d != top])
+                self._build_dir([db, 0], top, excl, top)
+                self._drop_lost(db, top)
+                db.commit()
+
+    def _build_dir(self, dbw, top, excl, cdir):
+        try:
+            inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
+        except Exception as ex:
+            self.log("up2k", "listdir: " + repr(ex))
+            return
+
+        histdir = os.path.join(top, ".hist")
+        for inode in inodes:
+            abspath = os.path.join(cdir, inode)
+            try:
+                inf = os.stat(fsenc(abspath))
+            except Exception as ex:
+                self.log("up2k", "stat: " + repr(ex))
+                continue
+
+            if stat.S_ISDIR(inf.st_mode):
+                if abspath in excl or abspath == histdir:
+                    continue
+                # self.log("up2k", " dir: {}".format(abspath))
+                self._build_dir(dbw, top, excl, abspath)
+            else:
+                # self.log("up2k", "file: {}".format(abspath))
+                rp = abspath[len(top) :].replace("\\", "/").strip("/")
+                c = dbw[0].execute("select * from up where rp = ?", (rp,))
+                in_db = list(c.fetchall())
+                if in_db:
+                    _, dts, dsz, _ = in_db[0]
+                    if len(in_db) > 1:
+                        m = "WARN: multiple entries: [{}] => [{}] ({})"
+                        self.log("up2k", m.format(top, rp, len(in_db)))
+                        dts = -1
+
+                    if dts == inf.st_mtime and dsz == inf.st_size:
+                        continue
+
+                    m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format(
+                        top, rp, dts, inf.st_mtime, dsz, inf.st_size
+                    )
+                    self.log("up2k", m)
+                    self.db_rm(dbw[0], rp)
+                    dbw[1] += 1
+                    in_db = None
+
+                self.log("up2k", "file: {}".format(abspath))
+                try:
+                    hashes = self._hashlist_from_file(abspath)
+                except Exception as ex:
+                    self.log("up2k", "hash: " + repr(ex))
+                    continue
+
+                wark = self._wark_from_hashlist(inf.st_size, hashes)
+                self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size)
+                dbw[1] += 1
+                if dbw[1] > 1024:
+                    dbw[0].commit()
+                    dbw[1] = 0
+
+    def _drop_lost(self, db, top):
+        rm = []
+        c = db.execute("select * from up")
+        for dwark, dts, dsz, drp in c:
+            abspath = os.path.join(top, drp)
+            try:
+                if not os.path.exists(fsenc(abspath)):
+                    rm.append(drp)
+            except Exception as ex:
+                self.log("up2k", "stat-rm: " + repr(ex))
+
+        if not rm:
+            return
+
+        self.log("up2k", "forgetting {} deleted files".format(len(rm)))
+        for rp in rm:
+            self.db_rm(db, rp)
+
+    def _open_db(self, db_path):
+        conn = sqlite3.connect(db_path, check_same_thread=False)
+        try:
+            c = conn.execute(r"select * from kv where k = 'sver'")
+            rows = c.fetchall()
+            if rows:
+                ver = rows[0][1]
+            else:
+                self.log("up2k", "WARN: no sver in kv, DB corrupt?")
+                ver = "unknown"
+
+            if ver == "1":
+                try:
+                    nfiles = next(conn.execute("select count(w) from up"))[0]
+                    self.log("up2k", "found DB at {} |{}|".format(db_path, nfiles))
+                    return conn
+                except Exception as ex:
+                    m = "WARN: could not list files, DB corrupt?\n  " + repr(ex)
+                    self.log("up2k", m)
+
+            m = "REPLACING unsupported DB (v.{}) at {}".format(ver, db_path)
+            self.log("up2k", m)
+            conn.close()
+            os.unlink(db_path)
+            conn = sqlite3.connect(db_path, check_same_thread=False)
+        except:
+            pass
+
+        # sqlite is variable-width only, no point in using char/nchar/varchar
+        for cmd in [
+            r"create table kv (k text, v text)",
+            r"create table up (w text, mt int, sz int, rp text)",
+            r"insert into kv values ('sver', '1')",
+            r"create index up_w on up(w)",
+        ]:
+            conn.execute(cmd)
+
+        conn.commit()
+        self.log("up2k", "created DB at {}".format(db_path))
+        return conn
+
     def handle_json(self, cj):
+        self.register_vpath(cj["ptop"])
         cj["name"] = sanitize_fn(cj["name"])
+        cj["poke"] = time.time()
         wark = self._get_wark(cj)
         now = time.time()
+        job = None
         with self.mutex:
-            # TODO use registry persistence here to symlink any matching wark
-            if wark in self.registry:
-                job = self.registry[wark]
-                if job["rdir"] != cj["rdir"] or job["name"] != cj["name"]:
-                    src = os.path.join(job["rdir"], job["name"])
-                    dst = os.path.join(cj["rdir"], cj["name"])
+            db = self.db.get(cj["ptop"], None)
+            reg = self.registry[cj["ptop"]]
+            if db:
+                cur = db.execute(r"select * from up where w = ?", (wark,))
+                for _, dtime, dsize, dp_rel in cur:
+                    dp_abs = os.path.join(cj["ptop"], dp_rel).replace("\\", "/")
+                    # relying on path.exists to return false on broken symlinks
+                    if os.path.exists(fsenc(dp_abs)):
+                        try:
+                            prel, name = dp_rel.rsplit("/", 1)
+                        except:
+                            prel = ""
+                            name = dp_rel
+
+                        job = {
+                            "name": name,
+                            "prel": prel,
+                            "vtop": cj["vtop"],
+                            "ptop": cj["ptop"],
+                            "flag": cj["flag"],
+                            "size": dsize,
+                            "lmod": dtime,
+                            "hash": [],
+                            "need": [],
+                        }
+                        break
+
+                if job and wark in reg:
+                    del reg[wark]
+
+            if job or wark in reg:
+                job = job or reg[wark]
+                if job["prel"] == cj["prel"] and job["name"] == cj["name"]:
+                    # ensure the files haven't been deleted manually
+                    names = [job[x] for x in ["name", "tnam"] if x in job]
+                    for fn in names:
+                        path = os.path.join(job["ptop"], job["prel"], fn)
+                        try:
+                            if os.path.getsize(path) > 0:
+                                # upload completed or both present
+                                break
+                        except:
+                            # missing; restart
+                            job = None
+                            break
+                else:
+                    # file contents match, but not the path
+                    src = os.path.join(job["ptop"], job["prel"], job["name"])
+                    dst = os.path.join(cj["ptop"], cj["prel"], cj["name"])
+                    vsrc = os.path.join(job["vtop"], job["prel"], job["name"])
+                    vsrc = vsrc.replace("\\", "/")  # just for prints anyways
                     if job["need"]:
                         self.log("up2k", "unfinished:\n  {0}\n  {1}".format(src, dst))
-                        err = "partial upload exists at a different location; please resume uploading here instead:\n{0}{1} ".format(
-                            job["vdir"], job["name"]
-                        )
+                        err = "partial upload exists at a different location; please resume uploading here instead:\n"
+                        err += vsrc + " "
+                        raise Pebkac(400, err)
+                    elif "nodupe" in job["flag"]:
+                        self.log("up2k", "dupe-reject:\n  {0}\n  {1}".format(src, dst))
+                        err = "upload rejected, file already exists:\n " + vsrc + " "
                         raise Pebkac(400, err)
                     else:
                         # symlink to the client-provided name,
                         # returning the previous upload info
                         job = deepcopy(job)
-                        job["rdir"] = cj["rdir"]
-                        job["name"] = self._untaken(cj["rdir"], cj["name"], now, cj["addr"])
-                        dst = os.path.join(job["rdir"], job["name"])
+                        for k in ["ptop", "vtop", "prel"]:
+                            job[k] = cj[k]
+
+                        pdir = os.path.join(cj["ptop"], cj["prel"])
+                        job["name"] = self._untaken(pdir, cj["name"], now, cj["addr"])
+                        dst = os.path.join(job["ptop"], job["prel"], job["name"])
                         os.unlink(fsenc(dst))  # TODO ed pls
                         self._symlink(src, dst)
-            else:
+
+            if not job:
                 job = {
                     "wark": wark,
                     "t0": now,
-                    "addr": cj["addr"],
-                    "vdir": cj["vdir"],
-                    "rdir": cj["rdir"],
-                    # client-provided, sanitized by _get_wark:
-                    "name": cj["name"],
-                    "size": cj["size"],
-                    "lmod": cj["lmod"],
                     "hash": deepcopy(cj["hash"]),
+                    "need": [],
                 }
+                # client-provided, sanitized by _get_wark: name, size, lmod
+                for k in [
+                    "addr",
+                    "vtop",
+                    "ptop",
+                    "prel",
+                    "flag",
+                    "name",
+                    "size",
+                    "lmod",
+                    "poke",
+                ]:
+                    job[k] = cj[k]
 
                 # one chunk may occur multiple times in a file;
                 # filter to unique values for the list of missing chunks
                 # (preserve order to reduce disk thrashing)
-                job["need"] = []
                 lut = {}
                 for k in cj["hash"]:
                     if k not in lut:
@@ -139,40 +404,58 @@ class Up2k(object):
                     lsrc = "../" * (len(lsrc) - 1) + "/".join(lsrc)
             os.symlink(fsenc(lsrc), fsenc(ldst))
         except (AttributeError, OSError) as ex:
-            self.log("up2k", "cannot symlink; creating copy")
+            self.log("up2k", "cannot symlink; creating copy: " + repr(ex))
             shutil.copy2(fsenc(src), fsenc(dst))
 
-    def handle_chunk(self, wark, chash):
+    def handle_chunk(self, ptop, wark, chash):
         with self.mutex:
-            job = self.registry.get(wark)
+            job = self.registry[ptop].get(wark, None)
             if not job:
-                raise Pebkac(404, "unknown wark")
+                raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
                 raise Pebkac(200, "already got that but thanks??")
 
             nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
             if not nchunk:
-                raise Pebkac(404, "unknown chunk")
+                raise Pebkac(400, "unknown chunk")
+
+        job["poke"] = time.time()
 
         chunksize = self._get_chunksize(job["size"])
         ofs = [chunksize * x for x in nchunk]
 
-        path = os.path.join(job["rdir"], job["name"])
+        path = os.path.join(job["ptop"], job["prel"], job["tnam"])
 
         return [chunksize, ofs, path, job["lmod"]]
 
-    def confirm_chunk(self, wark, chash):
+    def confirm_chunk(self, ptop, wark, chash):
         with self.mutex:
-            job = self.registry[wark]
+            job = self.registry[ptop][wark]
+            pdir = os.path.join(job["ptop"], job["prel"])
+            src = os.path.join(pdir, job["tnam"])
+            dst = os.path.join(pdir, job["name"])
+
             job["need"].remove(chash)
             ret = len(job["need"])
+            if ret > 0:
+                return ret, src
 
-            if WINDOWS and ret == 0:
-                path = os.path.join(job["rdir"], job["name"])
-                self.lastmod_q.put([path, (int(time.time()), int(job["lmod"]))])
+            atomic_move(src, dst)
 
-            return ret
+            if WINDOWS:
+                self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))])
+
+            db = self.db.get(job["ptop"], None)
+            if db:
+                rp = os.path.join(job["prel"], job["name"]).replace("\\", "/")
+                self.db_rm(db, rp)
+                self.db_add(db, job["wark"], rp, job["lmod"], job["size"])
+                db.commit()
+                del self.registry[ptop][wark]
+                # in-memory registry is reserved for unfinished uploads
+
+            return ret, dst
 
     def _get_chunksize(self, filesize):
         chunksize = 1024 * 1024
@@ -186,6 +469,13 @@ class Up2k(object):
                 chunksize += stepsize
                 stepsize *= mul
 
+    def db_rm(self, db, rp):
+        db.execute("delete from up where rp = ?", (rp,))
+
+    def db_add(self, db, wark, rp, ts, sz):
+        v = (wark, ts, sz, rp)
+        db.execute("insert into up values (?,?,?,?)", v)
+
     def _get_wark(self, cj):
         if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024:  # 16TiB
             raise Pebkac(400, "name or numchunks not according to spec")
@@ -202,9 +492,13 @@ class Up2k(object):
         except:
             cj["lmod"] = int(time.time())
 
-        # server-reproducible file identifier, independent of name or location
-        ident = [self.salt, str(cj["size"])]
-        ident.extend(cj["hash"])
+        wark = self._wark_from_hashlist(cj["size"], cj["hash"])
+        return wark
+
+    def _wark_from_hashlist(self, filesize, hashes):
+        """ server-reproducible file identifier, independent of name or location """
+        ident = [self.salt, str(filesize)]
+        ident.extend(hashes)
         ident = "\n".join(ident)
 
         hasher = hashlib.sha512()
@@ -214,11 +508,40 @@ class Up2k(object):
         wark = base64.urlsafe_b64encode(digest)
         return wark.decode("utf-8").rstrip("=")
 
+    def _hashlist_from_file(self, path):
+        fsz = os.path.getsize(path)
+        csz = self._get_chunksize(fsz)
+        ret = []
+        with open(path, "rb", 512 * 1024) as f:
+            while fsz > 0:
+                hashobj = hashlib.sha512()
+                rem = min(csz, fsz)
+                fsz -= rem
+                while rem > 0:
+                    buf = f.read(min(rem, 64 * 1024))
+                    if not buf:
+                        raise Exception("EOF at " + str(f.tell()))
+
+                    hashobj.update(buf)
+                    rem -= len(buf)
+
+                digest = hashobj.digest()[:32]
+                digest = base64.urlsafe_b64encode(digest)
+                ret.append(digest.decode("utf-8").rstrip("="))
+
+        return ret
+
     def _new_upload(self, job):
-        self.registry[job["wark"]] = job
+        self.registry[job["ptop"]][job["wark"]] = job
+        pdir = os.path.join(job["ptop"], job["prel"])
+        job["name"] = self._untaken(pdir, job["name"], job["t0"], job["addr"])
+        # if len(job["name"].split(".")) > 8:
+        #    raise Exception("aaa")
+
+        tnam = job["name"] + ".PARTIAL"
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
-        with ren_open(job["name"], "wb", fdir=job["rdir"], suffix=suffix) as f:
-            f, job["name"] = f["orz"]
+        with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f:
+            f, job["tnam"] = f["orz"]
             f.seek(job["size"] - 1)
             f.write(b"e")
 
@@ -235,3 +558,58 @@ class Up2k(object):
                     os.utime(fsenc(path), times)
                 except:
                     self.log("lmod", "failed to utime ({}, {})".format(path, times))
+
+    def _snapshot(self):
+        persist_interval = 30  # persist unfinished uploads index every 30 sec
+        discard_interval = 21600  # drop unfinished uploads after 6 hours inactivity
+        prev = {}
+        while True:
+            time.sleep(persist_interval)
+            with self.mutex:
+                for k, reg in self.registry.items():
+                    self._snap_reg(prev, k, reg, discard_interval)
+
+    def _snap_reg(self, prev, k, reg, discard_interval):
+        now = time.time()
+        rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
+        if rm:
+            m = "dropping {} abandoned uploads in {}".format(len(rm), k)
+            vis = [self._vis_job_progress(x) for x in rm]
+            self.log("up2k", "\n".join([m] + vis))
+            for job in rm:
+                del reg[job["wark"]]
+                try:
+                    # remove the filename reservation
+                    path = os.path.join(job["ptop"], job["prel"], job["name"])
+                    if os.path.getsize(path) == 0:
+                        os.unlink(path)
+
+                    if len(job["hash"]) == len(job["need"]):
+                        # PARTIAL is empty, delete that too
+                        path = os.path.join(job["ptop"], job["prel"], job["tnam"])
+                        os.unlink(path)
+                except:
+                    pass
+
+        path = os.path.join(k, ".hist", "up2k.snap")
+        if not reg:
+            if k not in prev or prev[k] is not None:
+                prev[k] = None
+                if os.path.exists(path):
+                    os.unlink(path)
+            return
+
+        newest = max(x["poke"] for _, x in reg.items()) if reg else 0
+        etag = [len(reg), newest]
+        if etag == prev.get(k, None):
+            return
+
+        path2 = "{}.{}".format(path, os.getpid())
+        j = json.dumps(reg, indent=2, sort_keys=True).encode("utf-8")
+        with gzip.GzipFile(path2, "wb") as f:
+            f.write(j)
+
+        atomic_move(path2, path)
+
+        self.log("up2k", "snap: {} |{}|".format(path, len(reg.keys())))
+        prev[k] = etag

copyparty/util.py

@@ -6,6 +6,7 @@ import os
 import sys
 import time
 import base64
+import select
 import struct
 import hashlib
 import platform
@@ -548,6 +549,16 @@ else:
     fsdec = w8dec
 
 
+def atomic_move(src, dst):
+    if not PY2:
+        os.replace(src, dst)
+    else:
+        if os.path.exists(dst):
+            os.unlink(dst)
+
+        os.rename(src, dst)
+
+
 def read_socket(sr, total_size):
     remains = total_size
     while remains > 0:
@@ -591,6 +602,46 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
+def sendfile_py(lower, upper, f, s):
+    remains = upper - lower
+    f.seek(lower)
+    while remains > 0:
+        # time.sleep(0.01)
+        buf = f.read(min(4096, remains))
+        if not buf:
+            return remains
+
+        try:
+            s.sendall(buf)
+            remains -= len(buf)
+        except:
+            return remains
+
+    return 0
+
+
+def sendfile_kern(lower, upper, f, s):
+    out_fd = s.fileno()
+    in_fd = f.fileno()
+    ofs = lower
+    while ofs < upper:
+        try:
+            req = min(2 ** 30, upper - ofs)
+            select.select([], [out_fd], [], 10)
+            n = os.sendfile(out_fd, in_fd, ofs, req)
+        except Exception as ex:
+            # print("sendfile: " + repr(ex))
+            n = 0
+        
+        if n <= 0:
+            return upper - ofs
+        
+        ofs += n
+        # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
+
+    return 0
+
+
 def unescape_cookie(orig):
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""
@@ -671,3 +722,6 @@ class Pebkac(Exception):
     def __init__(self, code, msg=None):
         super(Pebkac, self).__init__(msg or HTTPCODE[code])
         self.code = code
+
+    def __repr__(self):
+        return "Pebkac({}, {})".format(self.code, repr(self.args))

copyparty/web/browser.js

@@ -8,7 +8,14 @@ function dbg(msg) {
 
 function ev(e) {
 	e = e || window.event;
-	e.preventDefault ? e.preventDefault() : (e.returnValue = false);
+
+	if (e.preventDefault)
+		e.preventDefault()
+
+	if (e.stopPropagation)
+		e.stopPropagation();
+
+	e.returnValue = false;
 	return e;
 }
 
@@ -315,8 +322,12 @@ var vbar = (function () {
 		var rect = pbar.pcan.getBoundingClientRect();
 		var x = e.clientX - rect.left;
 		var mul = x * 1.0 / rect.width;
+		var seek = mp.au.duration * mul;
+		console.log('seek: ' + seek);
+		if (!isFinite(seek))
+			return;
 
-		mp.au.currentTime = mp.au.duration * mul;
+		mp.au.currentTime = seek;
 
 		if (mp.au === mp.au_native)
 			// hack: ogv.js breaks on .play() during playback
@@ -443,7 +454,8 @@ function play(tid, call_depth) {
 	mp.au.tid = tid;
 	mp.au.src = url;
 	mp.au.volume = mp.expvol();
-	setclass('trk' + tid, 'play act');
+	var oid = 'trk' + tid;
+	setclass(oid, 'play act');
 
 	try {
 		if (hack_attempt_play)
@@ -452,7 +464,11 @@ function play(tid, call_depth) {
 		if (mp.au.paused)
 			autoplay_blocked();
 
-		location.hash = 'trk' + tid;
+		var o = ebi(oid);
+		o.setAttribute('id', 'thx_js');
+		location.hash = oid;
+		o.setAttribute('id', oid);
+
 		pbar.drawbuf();
 		return true;
 	}

copyparty/web/up2k.js

@@ -201,6 +201,7 @@ function up2k_init(have_crypto) {
 
     var parallel_uploads = cfg_get('nthread');
     var multitask = bcfg_get('multitask', true);
+    var ask_up = bcfg_get('ask_up', true);
 
     var col_hashing = '#00bbff';
     var col_hashed = '#004466';
@@ -262,6 +263,7 @@ function up2k_init(have_crypto) {
 
         more_one_file();
         var bad_files = [];
+        var good_files = [];
         for (var a = 0; a < files.length; a++) {
             var fobj = files[a];
             if (is_itemlist) {
@@ -275,9 +277,32 @@ function up2k_init(have_crypto) {
                     throw 1;
             }
             catch (ex) {
-                bad_files.push([a, fobj.name]);
+                bad_files.push(fobj.name);
                 continue;
             }
+            good_files.push(fobj);
+        }
+
+        if (bad_files.length > 0) {
+            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, files.length);
+            for (var a = 0; a < bad_files.length; a++)
+                msg += '-- ' + bad_files[a] + '\n';
+
+            if (files.length - bad_files.length <= 1 && /(android)/i.test(navigator.userAgent))
+                msg += '\nFirefox-Android has a bug which prevents selecting multiple files. Try selecting one file at a time. For more info, see firefox bug 1456557';
+
+            alert(msg);
+        }
+
+        var msg = ['upload these ' + good_files.length + ' files?'];
+        for (var a = 0; a < good_files.length; a++)
+            msg.push(good_files[a].name);
+
+        if (ask_up && !confirm(msg.join('\n')))
+            return;
+
+        for (var a = 0; a < good_files.length; a++) {
+            var fobj = good_files[a];
             var now = new Date().getTime();
             var lmod = fobj.lastModified || now;
             var entry = {
@@ -307,17 +332,6 @@ function up2k_init(have_crypto) {
             st.files.push(entry);
             st.todo.hash.push(entry);
         }
-
-        if (bad_files.length > 0) {
-            var msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, files.length);
-            for (var a = 0; a < bad_files.length; a++)
-                msg += '-- ' + bad_files[a][1] + '\n';
-
-            if (files.length - bad_files.length <= 1 && /(android)/i.test(navigator.userAgent))
-                msg += '\nFirefox-Android has a bug which prevents selecting multiple files. Try selecting one file at a time. For more info, see firefox bug 1456557';
-
-            alert(msg);
-        }
     }
     ebi('u2btn').addEventListener('drop', gotfile, false);
 
@@ -336,16 +350,17 @@ function up2k_init(have_crypto) {
     //
 
     function handshakes_permitted() {
-        return multitask || (
-            st.todo.upload.length == 0 &&
-            st.busy.upload.length == 0);
+        var lim = multitask ? 1 : 0;
+        return lim >=
+            st.todo.upload.length +
+            st.busy.upload.length;
     }
 
     function hashing_permitted() {
-        return multitask || (
-            handshakes_permitted() &&
-            st.todo.handshake.length == 0 &&
-            st.busy.handshake.length == 0);
+        var lim = multitask ? 1 : 0;
+        return handshakes_permitted() && lim >=
+            st.todo.handshake.length +
+            st.busy.handshake.length;
     }
 
     var tasker = (function () {
@@ -670,8 +685,12 @@ function up2k_init(have_crypto) {
             else {
                 var err = "";
                 var rsp = (xhr.responseText + '');
-                if (rsp.indexOf('partial upload exists') !== -1) {
-                    err = rsp.slice(5);
+                if (rsp.indexOf('partial upload exists') !== -1 ||
+                    rsp.indexOf('file already exists') !== -1) {
+                    err = rsp;
+                    var ofs = err.lastIndexOf(' : ');
+                    if (ofs > 0)
+                        err = err.slice(0, ofs);
                 }
                 if (err != "") {
                     ebi('f{0}t'.format(t.n)).innerHTML = "ERROR";
@@ -821,6 +840,11 @@ function up2k_init(have_crypto) {
         bcfg_set('multitask', multitask);
     }
 
+    function tgl_ask_up() {
+        ask_up = !ask_up;
+        bcfg_set('ask_up', ask_up);
+    }
+
     function nop(ev) {
         ev.preventDefault();
         this.click();
@@ -837,6 +861,7 @@ function up2k_init(have_crypto) {
 
     ebi('nthread').addEventListener('input', bumpthread, false);
     ebi('multitask').addEventListener('click', tgl_multitask, false);
+    ebi('ask_up').addEventListener('click', tgl_ask_up, false);
 
     var nodes = ebi('u2conf').getElementsByTagName('a');
     for (var a = nodes.length - 1; a >= 0; a--)

copyparty/web/upload.css

@@ -194,6 +194,12 @@
 #u2conf input+a {
 	background: #d80;
 }
+#u2conf input[type="checkbox"]+label {
+	color: #f5a;
+}
+#u2conf input[type="checkbox"]:checked+label {
+	color: #fc5;
+}
 #u2foot {
 	color: #fff;
 	font-style: italic;

copyparty/web/upload.html

@@ -43,10 +43,14 @@
                         <input class="txtbox" id="nthread" value="2" />
                         <a href="#" id="nthread_add">+</a>
                     </td>
-                    <td rowspan="2">
+                    <td rowspan="2" style="padding-left:1.5em">
                         <input type="checkbox" id="multitask" />
                         <label for="multitask">hash while<br />uploading</label>
                     </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="ask_up" />
+                        <label for="ask_up">ask for<br />confirmation</label>
+                    </td>
                 </tr>
             </table>
 

docs/notes.sh

@@ -3,6 +3,14 @@ echo not a script
 exit 1
 
 
+##
+## delete all partial uploads
+##  (supports linux/macos, probably windows+msys2)
+
+gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f -- "$f"; done
+gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done
+
+
 ##
 ## create a test payload
 

scripts/fusefuzz.py

@@ -4,10 +4,10 @@ import os
 import time
 
 """
-mkdir -p /dev/shm/fusefuzz/{r,v}
-PYTHONPATH=.. python3 -m copyparty -v /dev/shm/fusefuzz/r::r -i 127.0.0.1
-../bin/copyparty-fuse.py /dev/shm/fusefuzz/v http://127.0.0.1:3923/ 2 0
-(d="$PWD"; cd /dev/shm/fusefuzz && "$d"/fusefuzz.py)
+td=/dev/shm/; [ -e $td ] || td=$HOME; mkdir -p $td/fusefuzz/{r,v}
+PYTHONPATH=.. python3 -m copyparty -v $td/fusefuzz/r::r -i 127.0.0.1
+../bin/copyparty-fuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
+(d="$PWD"; cd $td/fusefuzz && "$d"/fusefuzz.py)
 """
 
 

scripts/make-pypi-release.sh

@@ -3,12 +3,15 @@ set -e
 echo
 
 # osx support
-command -v gtar  >/dev/null &&
-command -v gfind >/dev/null && {
-	tar()  { gtar  "$@"; }
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
 }
 
 which md5sum 2>/dev/null >/dev/null &&

scripts/make-sfx.sh

@@ -18,13 +18,16 @@ echo
 #   (the fancy markdown editor)
 
 
-command -v gtar  >/dev/null &&
-command -v gfind >/dev/null && {
-	tar()  { gtar  "$@"; }
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
 	unexpand() { gunexpand "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
 }
 
 [ -e copyparty/__main__.py ] || cd ..
@@ -115,7 +118,7 @@ git describe --tags >/dev/null 2>/dev/null && {
 		exit 1
 	}
 
-	dt="$(git log -1 --format=%cd --date=format:'%Y,%m,%d' | sed 's/,0?/, /g')"
+	dt="$(git log -1 --format=%cd --date=format:'%Y,%m,%d' | sed -E 's/,0?/, /g')"
 	printf 'git %3s: \033[36m%s\033[0m\n' ver "$ver" dt "$dt"
 	sed -ri '
 		s/^(VERSION =)(.*)/#\1\2\n\1 ('"$t_ver"')/;

scripts/make-tgz-release.sh

@@ -2,12 +2,16 @@
 set -e
 echo
 
-command -v gtar  >/dev/null &&
-command -v gfind >/dev/null && {
-	tar()  { gtar  "$@"; }
+# osx support
+# port install gnutar findutils gsed coreutils
+gtar=$(command -v gtar || command -v gnutar) || true
+[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
+	tar()  { $gtar "$@"; }
 	sed()  { gsed  "$@"; }
 	find() { gfind "$@"; }
 	sort() { gsort "$@"; }
+	command -v grealpath >/dev/null &&
+		realpath() { grealpath "$@"; }
 }
 
 which md5sum 2>/dev/null >/dev/null &&
@@ -16,15 +20,15 @@ which md5sum 2>/dev/null >/dev/null &&
 
 ver="$1"
 
-[[ "x$ver" == x ]] &&
+[ "x$ver" = x ] &&
 {
 	echo "need argument 1:  version"
 	echo
 	exit 1
 }
 
-[[ -e copyparty/__main__.py ]] || cd ..
-[[ -e copyparty/__main__.py ]] ||
+[ -e copyparty/__main__.py ] || cd ..
+[ -e copyparty/__main__.py ] ||
 {
 	echo "run me from within the project root folder"
 	echo
@@ -35,8 +39,8 @@ mkdir -p dist
 zip_path="$(pwd)/dist/copyparty-$ver.zip"
 tgz_path="$(pwd)/dist/copyparty-$ver.tar.gz"
 
-[[ -e "$zip_path" ]] ||
-[[ -e "$tgz_path" ]] &&
+[ -e "$zip_path" ] ||
+[ -e "$tgz_path" ] &&
 {
 	echo "found existing archives for this version"
 	echo "  $zip_path"