v0.10.5

README.md

@@ -97,6 +97,7 @@ summary: it works! you can use it! (but technically not even close to beta)
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
+* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
 

copyparty/__main__.py

@@ -177,11 +177,14 @@ def sighandler(signal=None, frame=None):
     print("\n".join(msg))
 
 
-def main():
+def main(argv=None):
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
 
+    if argv is None:
+        argv = sys.argv
+
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
@@ -194,13 +197,13 @@ def main():
     deprecated = [["-e2s", "-e2ds"]]
     for dk, nk in deprecated:
         try:
-            idx = sys.argv.index(dk)
+            idx = argv.index(dk)
         except:
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
         print(msg.format(dk, nk))
-        sys.argv[idx] = nk
+        argv[idx] = nk
         time.sleep(2)
 
     ap = argparse.ArgumentParser(
@@ -290,7 +293,7 @@ def main():
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     
-    al = ap.parse_args()
+    al = ap.parse_args(args=argv[1:])
     # fmt: on
 
     # propagate implications

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 10, 2)
+VERSION = (0, 10, 5)
 CODENAME = "zip it"
-BUILD_DT = (2021, 3, 27)
+BUILD_DT = (2021, 3, 31)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/broker_mp.py

@@ -51,7 +51,7 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if True:
+        if not self.args.q:
             thr = threading.Thread(target=self.debug_load_balancer)
             thr.daemon = True
             thr.start()

copyparty/star.py

@@ -11,9 +11,20 @@ class QFile(object):
 
     def __init__(self):
         self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
 
     def write(self, buf):
-        self.q.put(buf)
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
 
 
 class StreamTar(object):
@@ -38,7 +49,7 @@ class StreamTar(object):
     def gen(self):
         while True:
             buf = self.qfile.q.get()
-            if buf is None:
+            if not buf:
                 break
 
             self.co += len(buf)
@@ -81,4 +92,4 @@ class StreamTar(object):
             self.ser(self.errf)
 
         self.tar.close()
-        self.qfile.q.put(None)
+        self.qfile.write(None)

copyparty/up2k.py

@@ -1310,6 +1310,7 @@ class Up2k(object):
                     self.log("no cursor to write tags with??", c=1)
                     continue
 
+                # TODO is undef if vol 404 on startup
                 entags = self.entags[ptop]
                 if not entags:
                     self.log("no entags okay.jpg", c=3)

copyparty/web/browser.js

@@ -753,7 +753,7 @@ document.onkeydown = function (e) {
 		clearTimeout(search_timeout);
 		var now = new Date().getTime();
 		if (now - search_in_progress > 30 * 1000)
-			search_timeout = setTimeout(do_search, 100);
+			search_timeout = setTimeout(do_search, 200);
 	}
 
 	function do_search() {
@@ -796,6 +796,8 @@ document.onkeydown = function (e) {
 		var res = JSON.parse(this.responseText),
 			tagord = res.tag_order;
 
+		sortfiles(res.hits);
+
 		var ofiles = ebi('files');
 		if (ofiles.getAttribute('ts') > this.ts)
 			return;
@@ -814,7 +816,7 @@ document.onkeydown = function (e) {
 
 		var html = mk_files_header(tagord);
 		html.push('<tbody>');
-		html.push('<tr><td>-</td><td colspan="42"><a href="#" id="unsearch">close search results</a></td></tr>');
+		html.push('<tr><td>-</td><td colspan="42"><a href="#" id="unsearch">! close search results</a></td></tr>');
 		for (var a = 0; a < res.hits.length; a++) {
 			var r = res.hits[a],
 				ts = parseInt(r.ts),
@@ -833,7 +835,7 @@ document.onkeydown = function (e) {
 					v = r.tags[k] || "";
 
 				if (k == ".dur") {
-					var sv = s2ms(v);
+					var sv = v ? s2ms(v) : "";
 					nodes[nodes.length - 1] += '</td><td sortv="' + v + '">' + sv;
 					continue;
 				}
@@ -1085,35 +1087,8 @@ var treectl = (function () {
 		}
 
 		ebi('srv_info').innerHTML = '<span>' + res.srvinf + '</span>';
-		var nodes = res.dirs.concat(res.files),
-			sopts = jread('fsort', []);
-
-		try {
-			for (var a = sopts.length - 1; a >= 0; a--) {
-				var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
-				if (!name)
-					continue;
-
-				if (name.indexOf('tags/') == -1) {
-					nodes.sort(function (v1, v2) {
-						if (!v1[name]) return -1 * rev;
-						if (!v2[name]) return 1 * rev;
-						return rev * (typ == 'int' ? (v1[name] - v2[name]) : (v1[name].localeCompare(v2[name])));
-					});
-				}
-				else {
-					name = name.slice(5);
-					nodes.sort(function (v1, v2) {
-						if (!v1.tags[name]) return -1 * rev;
-						if (!v2.tags[name]) return 1 * rev;
-						return rev * (typ == 'int' ? (v1.tags[name] - v2.tags[name]) : (v1.tags[name].localeCompare(v2.tags[name])));
-					});
-				}
-			}
-		}
-		catch (ex) {
-			console.log("failed to apply sort config: " + ex);
-		}
+		var nodes = res.dirs.concat(res.files);
+		nodes = sortfiles(nodes);
 
 		var top = this.top;
 		var html = mk_files_header(res.taglist);
@@ -1128,7 +1103,7 @@ var treectl = (function () {
 					v = (r.tags || {})[k] || "";
 
 				if (k == ".dur") {
-					var sv = s2ms(v);
+					var sv = v ? s2ms(v) : "";
 					ln[ln.length - 1] += '</td><td sortv="' + v + '">' + sv;
 					continue;
 				}
@@ -1411,8 +1386,8 @@ var filecols = (function () {
 		if (!min)
 			for (var a = 0, aa = rows.length; a < aa; a++) {
 				var c = rows[a].cells[i];
-				if (c)
-					var v = c.textContent = s2ms(c.textContent);
+				if (c && c.textContent)
+					c.textContent = s2ms(c.textContent);
 			}
 	}
 	catch (ex) { }
@@ -1483,8 +1458,11 @@ var mukey = (function () {
 	}
 
 	function render() {
-		var ci = find_file_col('Key'),
-			i = ci[0],
+		var ci = find_file_col('Key');
+		if (!ci)
+			return;
+
+		var i = ci[0],
 			min = ci[1],
 			rows = ebi('files').tBodies[0].rows;
 
@@ -1533,7 +1511,10 @@ var mukey = (function () {
 
 
 function addcrc() {
-	var links = document.querySelectorAll('#files>tbody>tr>td:nth-child(2)>a');
+	var links = document.querySelectorAll(
+		'#files>tbody>tr>td:nth-child(2)>' + (
+			ebi('unsearch') ? 'div>a:last-child' : 'a'));
+
 	for (var a = 0, aa = links.length; a < aa; a++)
 		if (!links[a].getAttribute('id'))
 			links[a].setAttribute('id', 'f-' + crc32(links[a].textContent));
@@ -1634,7 +1615,7 @@ var msel = (function () {
 		var names = [];
 		var links = document.querySelectorAll('#files tbody tr.sel td:nth-child(2) a');
 		for (var a = 0, aa = links.length; a < aa; a++)
-			names.push(links[a].getAttribute('href').split('/').slice(-1));
+			names.push(links[a].getAttribute('href').replace(/\/$/, "").split('/').slice(-1));
 
 		return names;
 	}

copyparty/web/util.js

@@ -113,6 +113,75 @@ function crc32(str) {
 };
 
 
+function sortfiles(nodes) {
+    var sopts = jread('fsort', [["lead", -1, ""], ["href", 1, ""]]);
+
+    try {
+        var is_srch = false;
+        if (nodes[0]['rp']) {
+            is_srch = true;
+            for (var b = 0, bb = nodes.length; b < bb; b++)
+                nodes[b].ext = nodes[b].rp.split('.').pop();
+            for (var b = 0; b < sopts.length; b++)
+                if (sopts[b][0] == 'href')
+                    sopts[b][0] = 'rp';
+        }
+        for (var a = sopts.length - 1; a >= 0; a--) {
+            var name = sopts[a][0], rev = sopts[a][1], typ = sopts[a][2];
+            if (!name)
+                continue;
+
+            if (name.indexOf('tags/') === 0) {
+                name = name.slice(5);
+                for (var b = 0, bb = nodes.length; b < bb; b++)
+                    nodes[b]._sv = nodes[b].tags[name];
+            }
+            else {
+                for (var b = 0, bb = nodes.length; b < bb; b++) {
+                    var v = nodes[b][name];
+
+                    if ((v + '').indexOf('<a ') === 0)
+                        v = v.split('>')[1];
+                    else if (name == "href" && v)
+                        v = uricom_dec(v)[0]
+
+                    nodes[b]._sv = v;
+                }
+            }
+
+            var onodes = nodes.map((x) => x);
+            nodes.sort(function (n1, n2) {
+                var v1 = n1._sv,
+                    v2 = n2._sv;
+
+                if (v1 === undefined) {
+                    if (v2 === undefined) {
+                        return onodes.indexOf(n1) - onodes.indexOf(n2);
+                    }
+                    return -1 * rev;
+                }
+                if (v2 === undefined) return 1 * rev;
+
+                var ret = rev * (typ == 'int' ? (v1 - v2) : (v1.localeCompare(v2)));
+                if (ret === 0)
+                    ret = onodes.indexOf(n1) - onodes.indexOf(n2);
+
+                return ret;
+            });
+        }
+        for (var b = 0, bb = nodes.length; b < bb; b++) {
+            delete nodes[b]._sv;
+            if (is_srch)
+                delete nodes[b].ext;
+        }
+    }
+    catch (ex) {
+        console.log("failed to apply sort config: " + ex);
+    }
+    return nodes;
+}
+
+
 function sortTable(table, col, cb) {
     var tb = table.tBodies[0],
         th = table.tHead.rows[0].cells,
@@ -186,7 +255,6 @@ function makeSortable(table, cb) {
 }
 
 
-
 (function () {
     var ops = document.querySelectorAll('#ops>a');
     for (var a = 0; a < ops.length; a++) {

docs/notes.sh

@@ -83,6 +83,9 @@ sqlite3 up2k.db 'select mt1.w, mt1.k, mt1.v, mt2.v from mt mt1 inner join mt mt2
 time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.w where mt1.k = +mt2.k and mt1.rowid != mt2.rowid'  > warks
 cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
 
+# dump all dbs
+find -iname up2k.db | while IFS= read -r x; do sqlite3 "$x" 'select substr(w,1,12), rd, fn from up' | sed -r 's/\|/ \| /g' | while IFS= read -r y; do printf '%s | %s\n' "$x" "$y"; done; done
+
 
 ##
 ## media
@@ -126,6 +129,15 @@ pip install virtualenv
 # readme toc
 cat README.md | awk '!/^#/{next} {lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab)} {printf "%" ((lv-1)*4+1) "s [%s](#%s)\n", "*",$0,bab}'
 
+# fix firefox phantom breakpoints,
+# suggestions from bugtracker, doesnt work (debugger is not attachable)
+devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging
+burger > developer >> browser toolbox  (ctrl-alt-shift-i)
+iframe btn topright >> chrome://devtools/content/debugger/index.html
+dbg.asyncStore.pendingBreakpoints = {}
+
+# fix firefox phantom breakpoints
+about:config >> devtools.debugger.prefs-schema-version = -1
 
 ##
 ## http 206

scripts/make-sfx.sh

@@ -161,7 +161,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.*
+rm -f copyparty/web/deps/*.full.* copyparty/web/{Makefile,splash.js}
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |

tests/test_vfs.py

@@ -18,7 +18,9 @@ from copyparty import util
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr mte".split()}
+        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex["mtp"] = []
+        ex["mte"] = "a"
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)