v0.11.4

bin/README.md

@@ -45,3 +45,18 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 # [`mtag/`](mtag/)
 * standalone programs which perform misc. file analysis
 * copyparty can Popen programs like these during file indexing to collect additional metadata
+
+
+# [`dbtool.py`](dbtool.py)
+upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty recommends to wipe the DB and reindex because it now collects additional metadata during analysis, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
+
+for that example (upgrading to v0.11.0), first move the old db aside, launch copyparty, let it rebuild the db until the point where it starts running mtp (colored messages as it adds the mtp tags), then CTRL-C and patch in the old mtp tags from the old db instead
+
+so assuming you have `-mtp` parsers to provide the tags `key` and `.bpm`:
+
+```
+~/bin/dbtool.py -ls up2k.db
+~/bin/dbtool.py -src up2k.db.v0.10.22 up2k.db -cmp
+~/bin/dbtool.py -src up2k.db.v0.10.22 up2k.db -rm-mtp-flag -copy key
+~/bin/dbtool.py -src up2k.db.v0.10.22 up2k.db -rm-mtp-flag -copy .bpm -vac
+```

bin/dbtool.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import sqlite3
+import argparse
+
+DB_VER = 3
+
+
+def die(msg):
+    print("\033[31m\n" + msg + "\n\033[0m")
+    sys.exit(1)
+
+
+def read_ver(db):
+    for tab in ["ki", "kv"]:
+        try:
+            c = db.execute(r"select v from {} where k = 'sver'".format(tab))
+        except:
+            continue
+
+        rows = c.fetchall()
+        if rows:
+            return int(rows[0][0])
+
+    return "corrupt"
+
+
+def ls(db):
+    nfiles = next(db.execute("select count(w) from up"))[0]
+    ntags = next(db.execute("select count(w) from mt"))[0]
+    print(f"{nfiles} files")
+    print(f"{ntags} tags\n")
+
+    print("number of occurences for each tag,")
+    print(" 'x' = file has no tags")
+    print(" 't:mtp' = the mtp flag (file not mtp processed yet)")
+    print()
+    for k, nk in db.execute("select k, count(k) from mt group by k order by k"):
+        print(f"{nk:9} {k}")
+
+
+def compare(n1, d1, n2, d2, verbose):
+    nt = next(d1.execute("select count(w) from up"))[0]
+    n = 0
+    miss = 0
+    for w, rd, fn in d1.execute("select w, rd, fn from up"):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} files in {n1} against {n2}\033[0m"
+            print(m)
+
+        q = "select w from up where substr(w,1,16) = ?"
+        hit = d2.execute(q, (w[:16],)).fetchone()
+        if not hit:
+            miss += 1
+            if verbose:
+                print(f"file in {n1} missing in {n2}: [{w}] {rd}/{fn}")
+
+    print(f" {miss} files in {n1} missing in {n2}\n")
+
+    nt = next(d1.execute("select count(w) from mt"))[0]
+    n = 0
+    miss = {}
+    nmiss = 0
+    for w, k, v in d1.execute("select * from mt"):
+        n += 1
+        if n % 100_000 == 0:
+            m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
+            print(m)
+
+        v2 = d2.execute("select v from mt where w = ? and +k = ?", (w, k)).fetchone()
+        if v2:
+            v2 = v2[0]
+
+        # if v != v2 and v2 and k in [".bpm", "key"] and n2 == "src":
+        #    print(f"{w} [{rd}/{fn}] {k} = [{v}] / [{v2}]")
+
+        if v2 is not None:
+            if k.startswith("."):
+                try:
+                    diff = abs(float(v) - float(v2))
+                    if diff > float(v) / 0.9:
+                        v2 = None
+                    else:
+                        v2 = v
+                except:
+                    pass
+
+            if v != v2:
+                v2 = None
+
+        if v2 is None:
+            nmiss += 1
+            try:
+                miss[k] += 1
+            except:
+                miss[k] = 1
+
+            if verbose:
+                q = "select rd, fn from up where substr(w,1,16) = ?"
+                rd, fn = d1.execute(q, (w,)).fetchone()
+                print(f"missing in {n2}: [{w}] [{rd}/{fn}] {k} = {v}")
+
+    for k, v in sorted(miss.items()):
+        if v:
+            print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
+
+    print(f"in total, {nmiss} missing tags in {n2}\n")
+
+
+def copy_mtp(d1, d2, tag, rm):
+    nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
+    n = 0
+    ndone = 0
+    for w, k, v in d1.execute("select * from mt where k = ?", (tag,)):
+        n += 1
+        if n % 25_000 == 0:
+            m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
+            print(m)
+
+        hit = d2.execute("select v from mt where w = ? and +k = ?", (w, k)).fetchone()
+        if hit:
+            hit = hit[0]
+
+        if hit != v:
+            ndone += 1
+            if hit is not None:
+                d2.execute("delete from mt where w = ? and +k = ?", (w, k))
+
+            d2.execute("insert into mt values (?,?,?)", (w, k, v))
+            if rm:
+                d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w,))
+
+    d2.commit()
+    print(f"copied {ndone} {tag} tags over")
+
+
+def main():
+    os.system("")
+    print()
+    
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db", help="database to work on")
+    ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
+
+    ap2 = ap.add_argument_group("informational / read-only stuff")
+    ap2.add_argument("-v", action="store_true", help="verbose")
+    ap2.add_argument("-ls", action="store_true", help="list summary for db")
+    ap2.add_argument("-cmp", action="store_true", help="compare databases")
+
+    ap2 = ap.add_argument_group("options which modify target db")
+    ap2.add_argument("-copy", metavar="TAG", type=str, help="mtp tag to copy over")
+    ap2.add_argument(
+        "-rm-mtp-flag",
+        action="store_true",
+        help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
+    )
+    ap2.add_argument("-vac", action="store_true", help="optimize DB")
+
+    ar = ap.parse_args()
+
+    for v in [ar.db, ar.src]:
+        if v and not os.path.exists(v):
+            die("database must exist")
+
+    db = sqlite3.connect(ar.db)
+    ds = sqlite3.connect(ar.src) if ar.src else None
+
+    for d, n in [[ds, "src"], [db, "dst"]]:
+        if not d:
+            continue
+
+        ver = read_ver(d)
+        if ver == "corrupt":
+            die("{} database appears to be corrupt, sorry")
+
+        if ver != DB_VER:
+            m = f"{n} db is version {ver}, this tool only supports version {DB_VER}, please upgrade it with copyparty first"
+            die(m)
+
+    if ar.ls:
+        ls(db)
+
+    if ar.cmp:
+        if not ds:
+            die("need src db to compare against")
+
+        compare("src", ds, "dst", db, ar.v)
+        compare("dst", db, "src", ds, ar.v)
+
+    if ar.copy:
+        copy_mtp(ds, db, ar.copy, ar.rm_mtp_flag)
+
+
+if __name__ == "__main__":
+    main()

copyparty/__main__.py

@@ -261,7 +261,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
     ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
     ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
-    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=1800, help="cleanup interval")
+    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval")
     ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
 
     ap2 = ap.add_argument_group('database options')

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 1)
+VERSION = (0, 11, 4)
 CODENAME = "the grid"
-BUILD_DT = (2021, 5, 29)
+BUILD_DT = (2021, 6, 1)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -135,7 +135,7 @@ class VFS(object):
             #
             return os.path.realpath(rp)
 
-    def ls(self, rem, uname, scandir, lstat=False):
+    def ls(self, rem, uname, scandir, incl_wo=False, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
@@ -143,12 +143,12 @@ class VFS(object):
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
-                if (
-                    uname in vn2.uread
-                    or "*" in vn2.uread
-                    or uname in vn2.uwrite
-                    or "*" in vn2.uwrite
-                ):
+                ok = uname in vn2.uread or "*" in vn2.uread
+
+                if not ok and incl_wo:
+                    ok = uname in vn2.uwrite or "*" in vn2.uwrite
+
+                if ok:
                     virt_vis[name] = vn2
 
             # no vfs nodes in the list of real inodes
@@ -162,7 +162,7 @@ class VFS(object):
         rel is a unix-style user-defined vpath (not vfs-related)
         """
 
-        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, False, lstat)
         rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
         rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
 

copyparty/httpcli.py

@@ -600,8 +600,9 @@ class HttpCli(object):
             taglist = {}
         else:
             # search by query params
-            self.log("qj: " + repr(body))
-            hits, taglist = idx.search(vols, body)
+            q = body["q"]
+            self.log("qj: " + q)
+            hits, taglist = idx.search(vols, q)
             msg = len(hits)
 
         idx.p_end = time.time()
@@ -1310,20 +1311,23 @@ class HttpCli(object):
 
     def tx_mounts(self):
         suf = self.urlq(rm=["h"])
-        rvol = [x + "/" if x else x for x in self.rvol]
-        wvol = [x + "/" if x else x for x in self.wvol]
+        rvol, wvol, avol = [
+            [("/" + x).rstrip("/") + "/" for x in y]
+            for y in [self.rvol, self.wvol, self.avol]
+        ]
 
         vstate = {}
         if self.avol and not self.args.no_rescan:
             x = self.conn.hsrv.broker.put(True, "up2k.get_volstate")
             vstate = json.loads(x.get())
+            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vstate.items()}
 
         html = self.j2(
             "splash",
             this=self,
             rvol=rvol,
             wvol=wvol,
-            avol=self.avol,
+            avol=avol,
             vstate=vstate,
             url_suf=suf,
         )
@@ -1396,7 +1400,9 @@ class HttpCli(object):
 
         try:
             vn, rem = self.auth.vfs.get(top, self.uname, True, False)
-            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
+            fsroot, vfs_ls, vfs_virt = vn.ls(
+                rem, self.uname, not self.args.no_scandir, True
+            )
         except:
             vfs_ls = []
             vfs_virt = {}
@@ -1561,7 +1567,9 @@ class HttpCli(object):
             if v is not None:
                 return self.tx_zip(k, v, vn, rem, [], self.args.ed)
 
-        fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
+        fsroot, vfs_ls, vfs_virt = vn.ls(
+            rem, self.uname, not self.args.no_scandir, True
+        )
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
         vfs_ls.extend(vfs_virt.keys())

copyparty/th_srv.py

@@ -316,10 +316,10 @@ class ThumbSrv(object):
             time.sleep(interval)
             for vol in self.vols:
                 vol += "/.hist/th"
-                self.log("cln {}/".format(vol))
+                self.log("\033[Jcln {}/\033[A".format(vol))
                 self.clean(vol)
 
-            self.log("cln ok")
+            self.log("\033[Jcln ok")
 
     def clean(self, vol):
         # self.log("cln {}".format(vol))

copyparty/u2idx.py

@@ -47,11 +47,11 @@ class U2idx(object):
         fhash = body["hash"]
         wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
 
-        uq = "substr(w,1,16) = ? and w = ?"
+        uq = "where substr(w,1,16) = ? and w = ?"
         uv = [wark[:16], wark]
 
         try:
-            return self.run_query(vols, uq, uv, {})[0]
+            return self.run_query(vols, uq, uv)[0]
         except Exception as ex:
             raise Pebkac(500, repr(ex))
 
@@ -67,37 +67,120 @@ class U2idx(object):
         self.cur[ptop] = cur
         return cur
 
-    def search(self, vols, body):
+    def search(self, vols, uq):
         """search by query params"""
         if not HAVE_SQLITE3:
             return []
 
-        qobj = {}
-        _conv_sz(qobj, body, "sz_min", "up.sz >= ?")
-        _conv_sz(qobj, body, "sz_max", "up.sz <= ?")
-        _conv_dt(qobj, body, "dt_min", "up.mt >= ?")
-        _conv_dt(qobj, body, "dt_max", "up.mt <= ?")
-        for seg, dk in [["path", "up.rd"], ["name", "up.fn"]]:
-            if seg in body:
-                _conv_txt(qobj, body, seg, dk)
+        q = ""
+        va = []
+        joins = ""
+        is_key = True
+        is_size = False
+        is_date = False
+        kw_key = ["(", ")", "and ", "or ", "not "]
+        kw_val = ["==", "=", "!=", ">", ">=", "<", "<=", "like "]
+        ptn_mt = re.compile(r"^\.?[a-z]+$")
+        mt_ctr = 0
+        mt_keycmp = "substr(up.w,1,16)"
+        mt_keycmp2 = None
 
-        uq, uv = _sqlize(qobj)
+        while True:
+            uq = uq.strip()
+            if not uq:
+                break
 
-        qobj = {}
-        if "tags" in body:
-            _conv_txt(qobj, body, "tags", "mt.v")
+            ok = False
+            for kw in kw_key + kw_val:
+                if uq.startswith(kw):
+                    is_key = kw in kw_key
+                    uq = uq[len(kw) :]
+                    ok = True
+                    q += kw
+                    break
 
-        if "adv" in body:
-            _conv_adv(qobj, body, "adv")
+            if ok:
+                continue
+
+            v, uq = (uq + " ").split(" ", 1)
+            if is_key:
+                is_key = False
+
+                if v == "size":
+                    v = "up.sz"
+                    is_size = True
+
+                elif v == "date":
+                    v = "up.mt"
+                    is_date = True
+
+                elif v == "path":
+                    v = "up.rd"
+
+                elif v == "name":
+                    v = "up.fn"
+
+                elif v == "tags" or ptn_mt.match(v):
+                    mt_ctr += 1
+                    mt_keycmp2 = "mt{}.w".format(mt_ctr)
+                    joins += "inner join mt mt{} on {} = {} ".format(
+                        mt_ctr, mt_keycmp, mt_keycmp2
+                    )
+                    if v == "tags":
+                        v = "mt{0}.v".format(mt_ctr)
+                    else:
+                        v = "mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
+
+                else:
+                    raise Pebkac(400, "invalid key [" + v + "]")
+
+                q += v + " "
+                continue
+
+            head = ""
+            tail = ""
+
+            if is_date:
+                is_date = False
+                v = v.upper().rstrip("Z").replace(",", " ").replace("T", " ")
+                while "  " in v:
+                    v = v.replace("  ", " ")
+
+                for fmt in [
+                    "%Y-%m-%d %H:%M:%S",
+                    "%Y-%m-%d %H:%M",
+                    "%Y-%m-%d %H",
+                    "%Y-%m-%d",
+                ]:
+                    try:
+                        v = datetime.strptime(v, fmt).timestamp()
+                        break
+                    except:
+                        pass
+
+            elif is_size:
+                is_size = False
+                v = int(float(v) * 1024 * 1024)
+
+            else:
+                if v.startswith("*"):
+                    head = "'%'||"
+                    v = v[1:]
+
+                if v.endswith("*"):
+                    tail = "||'%'"
+                    v = v[:-1]
+
+            q += " {}?{} ".format(head, tail)
+            va.append(v)
+            is_key = True
 
         try:
-            return self.run_query(vols, uq, uv, qobj)
+            return self.run_query(vols, joins + "where " + q, va)
         except Exception as ex:
             raise Pebkac(500, repr(ex))
 
-    def run_query(self, vols, uq, uv, targs):
-        self.log("qs: {} {} ,  {}".format(uq, repr(uv), repr(targs)))
-
+    def run_query(self, vols, uq, uv):
         done_flag = []
         self.active_id = "{:.6f}_{}".format(
             time.time(), threading.current_thread().ident
@@ -112,35 +195,14 @@ class U2idx(object):
         thr.daemon = True
         thr.start()
 
-        if not targs:
-            if not uq:
-                q = "select * from up"
-                v = ()
-            else:
-                q = "select * from up where " + uq
-                v = tuple(uv)
+        if not uq:
+            q = "select * from up"
+            v = ()
         else:
-            q = "select up.* from up"
-            keycmp = "substr(up.w,1,16)"
-            where = []
-            v = []
-            ctr = 0
-            for tq, tv in sorted(targs.items()):
-                ctr += 1
-                tq = tq.split("\n")[0]
-                keycmp2 = "mt{}.w".format(ctr)
-                q += " inner join mt mt{} on {} = {}".format(ctr, keycmp, keycmp2)
-                keycmp = keycmp2
-                where.append(tq.replace("mt.", keycmp[:-1]))
-                v.append(tv)
+            q = "select up.* from up " + uq
+            v = tuple(uv)
 
-            if uq:
-                where.append(uq)
-                v.extend(uv)
-
-            q += " where " + (" and ".join(where))
-
-        # self.log("q2: {} {}".format(q, repr(v)))
+        self.log("qs: {!r} {!r}".format(q, v))
 
         ret = []
         lim = 1000
@@ -163,7 +225,7 @@ class U2idx(object):
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
 
-                rp = "/".join([vtop, rd, fn])
+                rp = "/".join([x for x in [vtop, rd, fn] if x])
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
 
             for hit in sret:
@@ -204,78 +266,3 @@ def _open(ptop):
     db_path = os.path.join(ptop, ".hist", "up2k.db")
     if os.path.exists(db_path):
         return sqlite3.connect(db_path).cursor()
-
-
-def _conv_sz(q, body, k, sql):
-    if k in body:
-        q[sql] = int(float(body[k]) * 1024 * 1024)
-
-
-def _conv_dt(q, body, k, sql):
-    if k not in body:
-        return
-
-    v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ")
-    while "  " in v:
-        v = v.replace("  ", " ")
-
-    for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]:
-        try:
-            ts = datetime.strptime(v, fmt).timestamp()
-            break
-        except:
-            ts = None
-
-    if ts:
-        q[sql] = ts
-
-
-def _conv_txt(q, body, k, sql):
-    for v in body[k].split(" "):
-        inv = ""
-        if v.startswith("-"):
-            inv = "not"
-            v = v[1:]
-
-        if not v:
-            continue
-
-        head = "'%'||"
-        if v.startswith("^"):
-            head = ""
-            v = v[1:]
-
-        tail = "||'%'"
-        if v.endswith("$"):
-            tail = ""
-            v = v[:-1]
-
-        qk = "{} {} like {}?{}".format(sql, inv, head, tail)
-        q[qk + "\n" + v] = u8safe(v)
-
-
-def _conv_adv(q, body, k):
-    ptn = re.compile(r"^(\.?[a-z]+) *(==?|!=|<=?|>=?) *(.*)$")
-
-    parts = body[k].split(" ")
-    parts = [x.strip() for x in parts if x.strip()]
-
-    for part in parts:
-        m = ptn.match(part)
-        if not m:
-            p = html_escape(part)
-            raise Pebkac(400, "invalid argument [" + p + "]")
-
-        k, op, v = m.groups()
-        qk = "mt.k = '{}' and mt.v {} ?".format(k, op)
-        q[qk + "\n" + v] = u8safe(v)
-
-
-def _sqlize(qobj):
-    keys = []
-    values = []
-    for k, v in sorted(qobj.items()):
-        keys.append(k.split("\n")[0])
-        values.append(v)
-
-    return " and ".join(keys), values

copyparty/up2k.py

@@ -181,17 +181,20 @@ class Up2k(object):
         for vol in vols:
             try:
                 os.listdir(vol.realpath)
-                if not self.register_vpath(vol.realpath, vol.flags):
-                    raise Exception()
-
-                if vol.vpath in scan_vols or not scan_vols:
-                    live_vols.append(vol)
-
-                if vol.vpath not in self.volstate:
-                    self.volstate[vol.vpath] = "OFFLINE (not initialized)"
             except:
+                self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
+                self.log("cannot access " + vol.realpath, c=1)
+                continue
+
+            if not self.register_vpath(vol.realpath, vol.flags):
                 # self.log("db not enabled for {}".format(m, vol.realpath))
-                pass
+                continue
+
+            if vol.vpath in scan_vols or not scan_vols:
+                live_vols.append(vol)
+
+            if vol.vpath not in self.volstate:
+                self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
 
         vols = live_vols
         need_vac = {}
@@ -585,7 +588,8 @@ class Up2k(object):
 
         del self.pp
         for k in list(self.volstate.keys()):
-            self.volstate[k] = "online, idle"
+            if "OFFLINE" not in self.volstate[k]:
+                self.volstate[k] = "online, idle"
 
     def _run_one_mtp(self, ptop):
         entags = self.entags[ptop]
@@ -1240,12 +1244,15 @@ class Up2k(object):
         return wark
 
     def _hashlist_from_file(self, path):
+        pp = self.pp if hasattr(self, "pp") else None
         fsz = os.path.getsize(fsenc(path))
         csz = up2k_chunksize(fsz)
         ret = []
         with open(fsenc(path), "rb", 512 * 1024) as f:
             while fsz > 0:
-                self.pp.msg = "{} MB, {}".format(int(fsz / 1024 / 1024), path)
+                if pp:
+                    pp.msg = "{} MB, {}".format(int(fsz / 1024 / 1024), path)
+
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
                 fsz -= rem

copyparty/web/browser.css

@@ -529,6 +529,17 @@ input[type="checkbox"]:checked+label {
 	height: 1em;
 	margin: .2em 0 -1em 1.6em;
 }
+#tq_raw {
+	width: calc(100% - 2em);
+	margin: .3em 0 0 1.4em;
+}
+#tq_raw td+td {
+	width: 100%;
+}
+#op_search #q_raw {
+	width: 100%;
+	display: block;
+}
 #files td div span {
 	color: #fff;
 	padding: 0 .4em;

copyparty/web/browser.js

@@ -826,6 +826,11 @@ var thegrid = (function () {
 			ths[a].onclick = r.sel ? seltgl : null;
 			ths[a].setAttribute('class', ebi(ths[a].getAttribute('ref')).parentNode.parentNode.getAttribute('class'));
 		}
+		var uns = QS('#ggrid a[ref="unsearch"]');
+		if (uns)
+			uns.onclick = function () {
+				ebi('unsearch').click();
+			};
 	}
 
 	function loadgrid() {
@@ -836,9 +841,9 @@ var thegrid = (function () {
 			return r.loadsel();
 
 		var html = [];
-		var tr = lfiles.tBodies[0].rows;
-		for (var a = 0; a < tr.length; a++) {
-			var ao = tr[a].cells[1].firstChild,
+		var files = QSA('#files>tbody>tr>td:nth-child(2) a[id]');
+		for (var a = 0, aa = files.length; a < aa; a++) {
+			var ao = files[a],
 				href = esc(ao.getAttribute('href')),
 				ref = ao.getAttribute('id'),
 				isdir = href.split('?')[0].slice(-1)[0] == '/',
@@ -1026,6 +1031,7 @@ document.onkeydown = function (e) {
 	for (var a = 0; a < trs.length; a += 2) {
 		html.push('<table>' + (trs[a].concat(trs[a + 1])).join('\n') + '</table>');
 	}
+	html.push('<table id="tq_raw"><tr><td>raw</td><td><input id="q_raw" type="text" name="q" /></td></tr></table>');
 	ebi('srch_form').innerHTML = html.join('\n');
 
 	var o = QSA('#op_search input');
@@ -1050,33 +1056,83 @@ document.onkeydown = function (e) {
 			var chk = ebi(id.slice(0, -1) + 'c');
 			chk.checked = ((v + '').length > 0);
 		}
+
+		if (id != "q_raw")
+			encode_query();
+
 		clearTimeout(search_timeout);
 		if (Date.now() - search_in_progress > 30 * 1000)
 			search_timeout = setTimeout(do_search, 200);
 	}
 
+	function encode_query() {
+		var q = '';
+		for (var a = 0; a < sconf.length; a++) {
+			for (var b = 1; b < sconf[a].length; b++) {
+				var k = sconf[a][b][0],
+					chk = 'srch_' + k + 'c',
+					tvs = ebi('srch_' + k + 'v').value.split(/ /g);
+
+				if (!ebi(chk).checked)
+					continue;
+
+				for (var c = 0; c < tvs.length; c++) {
+					var tv = tvs[c];
+					if (!tv.length)
+						break;
+
+					q += ' and ';
+
+					if (k == 'adv') {
+						q += tv.replace(/ /g, " and ").replace(/([=!><]=?)/, " $1 ");
+						continue;
+					}
+
+					if (k.length == 3) {
+						q += k.replace(/sz/, 'size').replace(/dt/, 'date').replace(/l$/, ' >= ').replace(/u$/, ' <= ') + tv;
+						continue;
+					}
+
+					if (k == 'path' || k == 'name' || k == 'tags') {
+						var not = ' ';
+						if (tv.slice(0, 1) == '-') {
+							tv = tv.slice(1);
+							not = ' not ';
+						}
+
+						if (tv.slice(0, 1) == '^') {
+							tv = tv.slice(1);
+						}
+						else {
+							tv = '*' + tv;
+						}
+
+						if (tv.slice(-1) == '$') {
+							tv = tv.slice(0, -1);
+						}
+						else {
+							tv += '*';
+						}
+
+						q += k + not + 'like ' + tv;
+					}
+				}
+			}
+		}
+		ebi('q_raw').value = q.slice(5);
+	}
+
 	function do_search() {
 		search_in_progress = Date.now();
 		srch_msg(false, "searching...");
 		clearTimeout(search_timeout);
 
-		var params = {},
-			o = QSA('#op_search input[type="text"]');
-
-		for (var a = 0; a < o.length; a++) {
-			var chk = ebi(o[a].getAttribute('id').slice(0, -1) + 'c');
-			if (!chk.checked)
-				continue;
-
-			params[o[a].getAttribute('name')] = o[a].value;
-		}
-		// ebi('srch_q').textContent = JSON.stringify(params, null, 4);
 		var xhr = new XMLHttpRequest();
 		xhr.open('POST', '/?srch', true);
 		xhr.setRequestHeader('Content-Type', 'text/plain');
 		xhr.onreadystatechange = xhr_search_results;
 		xhr.ts = Date.now();
-		xhr.send(JSON.stringify(params));
+		xhr.send(JSON.stringify({ "q": ebi('q_raw').value }));
 	}
 
 	function xhr_search_results() {

copyparty/web/splash.html

@@ -20,13 +20,13 @@
             <tbody>
                 {% for mp in avol %}
                 {%- if mp in vstate and vstate[mp] %}
-                <tr><td>/{{ mp }}</td><td><a href="/{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
                 {%- endif %}
                 {% endfor %}
             </tbody>
         </table>
         <div class="btns">
-            <a href="/{{ avol[0] }}?stack">dump stack</a>
+            <a href="{{ avol[0] }}?stack">dump stack</a>
         </div>
         {%- endif %}
 
@@ -34,7 +34,7 @@
         <h1>you can browse these:</h1>
         <ul>
             {% for mp in rvol %}
-            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
         {%- endif %}
@@ -43,7 +43,7 @@
         <h1>you can upload to:</h1>
         <ul>
             {% for mp in wvol %}
-            <li><a href="/{{ mp }}{{ url_suf }}">/{{ mp }}</a></li>
+            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
             {% endfor %}
         </ul>
         {%- endif %}

docs/notes.sh

@@ -80,6 +80,13 @@ command -v gdate && date() { gdate "$@"; }; while true; do t=$(date +%s.%N); (ti
 var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.querySelectorAll('#u2tab .prog a').forEach((x) => {t.push(b+encodeURI(x.getAttribute("href")))}); console.log(t.join("\n"));
 
 
+##
+## bash oneliners
+
+# get the size and video-id of all youtube vids in folder, assuming filename ends with -id.ext, and create a copyparty search query
+find -maxdepth 1 -printf '%s %p\n' | sort -n | awk '!/-([0-9a-zA-Z_-]{11})\.(mkv|mp4|webm)$/{next} {sub(/\.[^\.]+$/,"");n=length($0);v=substr($0,n-10);print $1, v}' | tee /dev/stderr | awk 'BEGIN {p="("} {printf("%s name like *-%s.* ",p,$2);p="or"} END {print ")\n"}' | cat >&2
+
+
 ##
 ## sqlite3 stuff
 

tests/test_httpcli.py

@@ -32,6 +32,8 @@ class Cfg(Namespace):
             no_zip=False,
             no_scandir=False,
             no_sendfile=True,
+            no_rescan=True,
+            ihead=False,
             nih=True,
             mtp=[],
             mte="a",

tests/util.py

@@ -91,7 +91,10 @@ class VHttpConn(object):
         self.auth = auth
         self.log_func = log
         self.log_src = "a"
+        self.lf_url = None
         self.hsrv = VHttpSrv()
         self.nbyte = 0
         self.workload = 0
+        self.ico = None
+        self.thumbcli = None
         self.t0 = time.time()