v0.11.44

* extract codec and format info
* add FFprobe as fallback when mutagen fails
* add option to blacklist FFprobe for tags

README.md

@@ -70,7 +70,7 @@ running the sfx without arguments (for example doubleclicking it on Windows) wil
 
 some recommended options:
 * `-e2dsa` enables general file indexing, see [search configuration](#search-configuration)
-* `-e2ts` enables audio metadata indexing (needs either FFprobe or mutagen), see [optional dependencies](#optional-dependencies)
+* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
 * `-v /mnt/music:/music:r:afoo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, with user `foo` as `a`dmin (read/write), password `bar`
   * the syntax is `-v src:dst:perm:perm:...` so local-path, url-path, and one or more permissions to set
   * replace `:r:afoo` with `:rfoo` to only make the folder readable by `foo` and nobody else
@@ -155,7 +155,7 @@ small collection of user feedback
 
 # bugs
 
-* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
+* Windows: python 3.7 and older cannot read tags with FFprobe, so use Mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
 * `--th-ff-jpg` may fix video thumbnails on some FFmpeg versions
@@ -176,6 +176,9 @@ small collection of user feedback
 * Windows: msys2-python 3.8.6 occasionally throws `RuntimeError: release unlocked lock` when leaving a scoped mutex in up2k
   * this is an msys2 bug, the regular windows edition of python is fine
 
+* VirtualBox: sqlite throws `Disk I/O Error` when running in a VM and the up2k database is in a vboxsf
+  * use `--hist` or the `hist` volflag (`-v [...]:chist=/tmp/foo`) to place the db inside the vm instead
+
 
 # the browser
 
@@ -406,12 +409,12 @@ tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric val
 
 see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
 
-`--no-mutagen` disables mutagen and uses ffprobe instead, which...
-* is about 20x slower than mutagen
-* catches a few tags that mutagen doesn't
+`--no-mutagen` disables Mutagen and uses FFprobe instead, which...
+* is about 20x slower than Mutagen
+* catches a few tags that Mutagen doesn't
   * melodic key, video resolution, framerate, pixfmt
 * avoids pulling any GPL code into copyparty
-* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
+* more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve
 
 
 ## file parser plugins
@@ -545,7 +548,7 @@ below are some tweaks roughly ordered by usefulness:
 
 enable music tags:
 * either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
-* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
+* or `ffprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
 
 enable thumbnails of images:
 * `Pillow` (requires py2.7 or py3.5+)

contrib/systemd/copyparty.service

@@ -10,8 +10,8 @@
 #
 # with `Type=notify`, copyparty will signal systemd when it is ready to
 #   accept connections; correctly delaying units depending on copyparty.
-#   But note that journalctl does not show messages in the correct order
-#   (or with correct timestamps even), so you get confusing stuff like
+#   But note that journalctl will get the timestamps wrong due to
+#   python disabling line-buffering, so messages are out-of-order:
 #   https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
 
 [Unit]
@@ -19,6 +19,7 @@ Description=copyparty file server
 
 [Service]
 Type=notify
+SyslogIdentifier=copyparty
 ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 

copyparty/__main__.py

@@ -319,8 +319,9 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume state")
     ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
-    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
     ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
+    ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
     ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,ac,vc,res,.fps")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 41)
+VERSION = (0, 11, 44)
 CODENAME = "the grid"
-BUILD_DT = (2021, 7, 17)
+BUILD_DT = (2021, 7, 20)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/mtag.py

@@ -227,37 +227,47 @@ def parse_ffprobe(txt):
 class MTag(object):
     def __init__(self, log_func, args):
         self.log_func = log_func
+        self.args = args
         self.usable = True
-        self.prefer_mt = False
-        mappings = args.mtm
+        self.prefer_mt = not args.no_mtag_ff
         self.backend = "ffprobe" if args.no_mutagen else "mutagen"
-        or_ffprobe = " or ffprobe"
+        self.can_ffprobe = (
+            HAVE_FFPROBE
+            and not args.no_mtag_ff
+            and (not WINDOWS or sys.version_info >= (3, 8))
+        )
+        mappings = args.mtm
+        or_ffprobe = " or FFprobe"
 
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
                 import mutagen
             except:
-                self.log("could not load mutagen, trying ffprobe instead", c=3)
+                self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
         if self.backend == "ffprobe":
+            self.usable = self.can_ffprobe
             self.get = self.get_ffprobe
             self.prefer_mt = True
-            # about 20x slower
-            self.usable = HAVE_FFPROBE
 
-            if self.usable and WINDOWS and sys.version_info < (3, 8):
-                self.usable = False
+            if not HAVE_FFPROBE:
+                pass
+
+            elif args.no_mtag_ff:
+                msg = "found FFprobe but it was disabled by --no-mtag-ff"
+                self.log(msg, c=3)
+
+            elif WINDOWS and sys.version_info < (3, 8):
                 or_ffprobe = " or python >= 3.8"
-                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                msg = "found FFprobe but your python is too old; need 3.8 or newer"
                 self.log(msg, c=1)
 
         if not self.usable:
-            msg = "need mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
-            self.log(
-                msg.format(or_ffprobe, " " * 37, os.path.basename(sys.executable)), c=1
-            )
+            msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
+            pybin = os.path.basename(sys.executable)
+            self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
             return
 
         # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -387,7 +397,7 @@ class MTag(object):
             v2 = r2.get(k)
             if v1 == v2:
                 print("  ", k, v1)
-            elif v1 != "0000":  # ffprobe date=0
+            elif v1 != "0000":  # FFprobe date=0
                 diffs.append(k)
                 print(" 1", k, v1)
                 print(" 2", k, v2)
@@ -408,20 +418,33 @@ class MTag(object):
             md = mutagen.File(fsenc(abspath), easy=True)
             x = md.info.length
         except Exception as ex:
-            return {}
+            return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
-        ret = {}
-        try:
-            dur = int(md.info.length)
+        sz = os.path.getsize(fsenc(abspath))
+        ret = {".q": [0, int((sz / md.info.length) / 128)]}
+
+        for attr, k, norm in [
+            ["codec", "ac", unicode],
+            ["channels", "chs", int],
+            ["sample_rate", ".hz", int],
+            ["bitrate", ".aq", int],
+            ["length", ".dur", int],
+        ]:
             try:
-                q = int(md.info.bitrate / 1024)
+                v = getattr(md.info, attr)
             except:
-                q = int((os.path.getsize(fsenc(abspath)) / dur) / 128)
+                continue
 
-            ret[".dur"] = [0, dur]
-            ret[".q"] = [0, q]
-        except:
-            pass
+            if not v:
+                continue
+
+            if k == ".aq":
+                v /= 1000
+
+            if k == "ac" and v.startswith("mp4a.40."):
+                v = "aac"
+
+            ret[k] = [0, norm(v)]
 
         return self.normalize_tags(ret, md)
 

copyparty/th_srv.py

@@ -121,10 +121,10 @@ class ThumbSrv(object):
         if not self.args.no_vthumb and (not HAVE_FFMPEG or not HAVE_FFPROBE):
             missing = []
             if not HAVE_FFMPEG:
-                missing.append("ffmpeg")
+                missing.append("FFmpeg")
 
             if not HAVE_FFPROBE:
-                missing.append("ffprobe")
+                missing.append("FFprobe")
 
             msg = "cannot create video thumbnails because some of the required programs are not available: "
             msg += ", ".join(missing)

copyparty/u2idx.py

@@ -7,6 +7,7 @@ import time
 import threading
 from datetime import datetime
 
+from .__init__ import unicode
 from .util import s3dec, Pebkac, min_ex
 from .up2k import up2k_wark_from_hashlist
 
@@ -90,6 +91,8 @@ class U2idx(object):
         mt_ctr = 0
         mt_keycmp = "substr(up.w,1,16)"
         mt_keycmp2 = None
+        ptn_lc = re.compile(r" (mt[0-9]+\.v) ([=<!>]+) \? $")
+        ptn_lcv = re.compile(r"[a-zA-Z]")
 
         while True:
             uq = uq.strip()
@@ -182,6 +185,21 @@ class U2idx(object):
             va.append(v)
             is_key = True
 
+            # lowercase tag searches
+            m = ptn_lc.search(q)
+            if not m or not ptn_lcv.search(unicode(v)):
+                continue
+
+            va.pop()
+            va.append(v.lower())
+            q = q[: m.start()]
+
+            field, oper = m.groups()
+            if oper in ["=", "=="]:
+                q += " {} like ? ".format(field)
+            else:
+                q += " lower({}) {} ? ".format(field, oper)
+
         try:
             return self.run_query(vols, joins + "where " + q, va)
         except Exception as ex:

copyparty/up2k.py

@@ -304,7 +304,7 @@ class Up2k(object):
         self.log(msg.format(len(vols), time.time() - t0))
 
         if needed_mutagen:
-            msg = "could not read tags because no backends are available (mutagen or ffprobe)"
+            msg = "could not read tags because no backends are available (Mutagen or FFprobe)"
             self.log(msg, c=1)
 
         thr = None
@@ -596,7 +596,7 @@ class Up2k(object):
             c2 = conn.cursor()
             c3 = conn.cursor()
             n_left = cur.execute("select count(w) from up").fetchone()[0]
-            for w, rd, fn in cur.execute("select w, rd, fn from up"):
+            for w, rd, fn in cur.execute("select w, rd, fn from up order by rd, fn"):
                 n_left -= 1
                 q = "select w from mt where w = ?"
                 if c2.execute(q, (w[:16],)).fetchone():
@@ -1512,7 +1512,7 @@ def up2k_chunksize(filesize):
 
 
 def up2k_wark_from_hashlist(salt, filesize, hashes):
-    """ server-reproducible file identifier, independent of name or location """
+    """server-reproducible file identifier, independent of name or location"""
     ident = [salt, str(filesize)]
     ident.extend(hashes)
     ident = "\n".join(ident)

copyparty/web/browser.js

@@ -177,8 +177,10 @@ function opclick(e) {
 	goto(dest);
 
 	var input = QS('.opview.act input:not([type="hidden"])')
-	if (input && !is_touch)
+	if (input && !is_touch) {
+		tt.skip = true;
 		input.focus();
+	}
 }
 
 
@@ -1144,7 +1146,7 @@ var audio_eq = (function () {
 				v = parseFloat(vs);
 
 			if (isNaN(v) || v + '' != vs)
-				throw 42;
+				throw new Error('inval band');
 
 			if (isNaN(band))
 				r.amp = Math.round((v + step * 0.2) * 100) / 100;
@@ -1711,6 +1713,26 @@ var thegrid = (function () {
 })();
 
 
+function tree_scrollto() {
+	var act = QS('#treeul a.hl'),
+		ul = act ? act.offsetParent : null;
+
+	if (!ul)
+		return;
+
+	var ctr = ebi('tree'),
+		em = parseFloat(getComputedStyle(act).fontSize),
+		top = act.offsetTop + ul.offsetTop,
+		min = top - 11 * em,
+		max = top - (ctr.offsetHeight - 10 * em);
+
+	if (ctr.scrollTop > min)
+		ctr.scrollTop = Math.floor(min);
+	else if (ctr.scrollTop < max)
+		ctr.scrollTop = Math.floor(max);
+}
+
+
 function tree_neigh(n) {
 	var links = QSA('#treeul li>a+a');
 	if (!links.length) {
@@ -1736,6 +1758,7 @@ function tree_neigh(n) {
 	if (act >= links.length)
 		act = 0;
 
+	treectl.dir_cb = tree_scrollto;
 	links[act].click();
 }
 
@@ -1940,7 +1963,7 @@ document.onkeydown = function (e) {
 			for (var b = 1; b < sconf[a].length; b++) {
 				var k = sconf[a][b][0],
 					chk = 'srch_' + k + 'c',
-					tvs = ebi('srch_' + k + 'v').value.split(/ /g);
+					tvs = ebi('srch_' + k + 'v').value.split(/ +/g);
 
 				if (!ebi(chk).checked)
 					continue;
@@ -1953,7 +1976,7 @@ document.onkeydown = function (e) {
 					q += ' and ';
 
 					if (k == 'adv') {
-						q += tv.replace(/ /g, " and ").replace(/([=!><]=?)/, " $1 ");
+						q += tv.replace(/ +/g, " and ").replace(/([=!><]=?)/, " $1 ");
 						continue;
 					}
 
@@ -2284,7 +2307,12 @@ var treectl = (function () {
 		var fun = treectl.dir_cb;
 		if (fun) {
 			treectl.dir_cb = null;
-			fun();
+			try {
+				fun();
+			}
+			catch (ex) {
+				console.log("dir_cb failed", ex);
+			}
 		}
 	}
 
@@ -2576,7 +2604,7 @@ function find_file_col(txt) {
 	for (var a = 0; a < tds.length; a++) {
 		var spans = tds[a].getElementsByTagName('span');
 		if (spans.length && spans[0].textContent == txt) {
-			min = tds[a].getAttribute('class').indexOf('min') !== -1;
+			min = (tds[a].getAttribute('class') || '').indexOf('min') !== -1;
 			i = a;
 			break;
 		}
@@ -2706,6 +2734,10 @@ var filecols = (function () {
 			for (var b = 0, bb = tds.length; b < bb; b++)
 				tds[b].setAttribute('class', cls);
 		}
+		if (window['tt']) {
+			tt.att(ebi('hcols'));
+			tt.att(QS('#files thead'));
+		}
 	};
 	set_style();
 
@@ -2928,7 +2960,7 @@ var arcfmt = (function () {
 
 			var ofs = href.lastIndexOf('?');
 			if (ofs < 0)
-				throw 'missing arg in url';
+				throw new Error('missing arg in url');
 
 			o.setAttribute("href", href.slice(0, ofs + 1) + arg);
 			o.textContent = fmt.split('_')[0];
@@ -3090,7 +3122,7 @@ function reload_browser(not_mp) {
 
 	var oo = QSA('#files>tbody>tr>td:nth-child(3)');
 	for (var a = 0, aa = oo.length; a < aa; a++) {
-		var sz = oo[a].textContent.replace(/ /g, ""),
+		var sz = oo[a].textContent.replace(/ +/g, ""),
 			hsz = sz.replace(/\B(?=(\d{3})+(?!\d))/g, " ");
 
 		oo[a].textContent = hsz;

copyparty/web/md.js

@@ -176,7 +176,7 @@ function md_plug_err(ex, js) {
         var lns = js.split('\n');
         if (ln < lns.length) {
             o = mknod('span');
-            o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
+            o.style.cssText = "color:#ac2;font-size:.9em;font-family:'scp',monospace,monospace;display:block";
             o.textContent = lns[ln - 1];
         }
     }

copyparty/web/up2k.js

@@ -290,8 +290,7 @@ function U2pvis(act, btns) {
                 if (this.is_act(this.tab[a].in))
                     console.log("tab %d/%d = sz %s", a, aa, this.tab[a].bt);
 
-            console.log("a");
-            throw 42;
+            throw new Error('see console');
         }
 
         obj.innerHTML = fo.hp;
@@ -304,15 +303,8 @@ function U2pvis(act, btns) {
             oldcat = fo.in,
             bz_act = this.act == "bz";
 
-        if (oldcat == newcat) {
-            throw 42;
-        }
-
-        //console.log("oldcat %s %d, newcat %s %d, head=%d, tail=%d, file=%d, act.old=%s, act.new=%s, bz_act=%s",
-        //    oldcat, this.ctr[oldcat],
-        //    newcat, this.ctr[newcat],
-        //    this.head, this.tail, nfile,
-        //    this.is_act(oldcat), this.is_act(newcat), bz_act);
+        if (oldcat == newcat)
+            return;
 
         fo.in = newcat;
         this.ctr[oldcat]--;
@@ -468,6 +460,12 @@ function U2pvis(act, btns) {
 }
 
 
+function fsearch_explain(e) {
+    ev(e);
+    alert('you are currently in file-search mode\n\nswitch to upload-mode by clicking the green magnifying glass (next to the big yellow search button), and then refresh\n\nsorry');
+}
+
+
 function up2k_init(subtle) {
     // show modal message
     function showmodal(msg) {
@@ -1298,8 +1296,10 @@ function up2k_init(subtle) {
                         smsg = '';
 
                     if (!response || !response.hits || !response.hits.length) {
-                        msg = 'not found on server';
                         smsg = '404';
+                        msg = 'not found on server';
+                        if (has(perms, 'write'))
+                            msg += ' <a href="#" onclick="fsearch_explain()" class="fsearch_explain">(explain)</a>';
                     }
                     else {
                         smsg = 'found';

copyparty/web/upload.css

@@ -257,6 +257,11 @@ html.light #u2foot .warn span {
 	float: right;
 	margin-bottom: -.3em;
 }
+.fsearch_explain {
+	padding-left: .7em;
+	font-size: 1.1em;
+	line-height: 0;
+}
 
 
 

copyparty/web/util.js

@@ -503,13 +503,19 @@ var tt = (function () {
     var r = {
         "tt": mknod("div"),
         "en": true,
-        "el": null
+        "el": null,
+        "skip": false
     };
 
     r.tt.setAttribute('id', 'tt');
     document.body.appendChild(r.tt);
 
     r.show = function () {
+        if (r.skip) {
+            r.skip = false;
+            return;
+        }
+
         var cfg = sread('tooltips');
         if (cfg !== null && cfg != '1')
             return;

docs/notes.sh

@@ -166,7 +166,10 @@ dbg.asyncStore.pendingBreakpoints = {}
 about:config >> devtools.debugger.prefs-schema-version = -1
 
 # determine server version
-git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser,up2k}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser,up2k}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+
+# download all sfx versions
+curl https://api.github.com/repos/9001/copyparty/releases?per_page=100 | jq -r '.[] | .tag_name + " " + .name' | while read v t; do fn="copyparty $v $t.py"; [ -e $fn ] || curl https://github.com/9001/copyparty/releases/download/$v/copyparty-sfx.py -Lo "$fn"; done
 
 
 ##

scripts/make-sfx.sh

@@ -20,6 +20,11 @@ echo
 #
 # `no-cm` saves ~90k by removing easymde/codemirror
 #   (the fancy markdown editor)
+#
+# `no-fnt` saves ~9k by removing the source-code-pro font
+#   (mainly used my the markdown viewer/editor)
+#
+# `no-dd` saves ~2k by removing the mouse cursor
 
 
 # port install gnutar findutils gsed coreutils
@@ -57,14 +62,18 @@ use_gz=
 do_sh=1
 do_py=1
 while [ ! -z "$1" ]; do
-	[ "$1" = clean  ] && clean=1  && shift && continue
-	[ "$1" = re     ] && repack=1 && shift && continue
-	[ "$1" = gz     ] && use_gz=1 && shift && continue
-	[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
-	[ "$1" = no-cm  ] && no_cm=1  && shift && continue
-	[ "$1" = no-sh  ] && do_sh=   && shift && continue
-	[ "$1" = no-py  ] && do_py=   && shift && continue
-	break
+	case $1 in
+		clean)  clean=1  ; ;;
+		re)     repack=1 ; ;;
+		gz)     use_gz=1 ; ;;
+		no-ogv) no_ogv=1 ; ;;
+		no-fnt) no_fnt=1 ; ;;
+		no-dd)  no_dd=1  ; ;;
+		no-cm)  no_cm=1  ; ;;
+		no-sh)  do_sh=   ; ;;
+		no-py)  do_py=   ; ;;
+	esac
+	shift
 done
 
 tmv() {
@@ -190,6 +199,18 @@ done
 	sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
 }
 
+[ $no_fnt ] && {
+	rm -f copyparty/web/deps/scp.woff2
+	f=copyparty/web/md.css
+	sed -r '/scp\.woff2/d' <$f >t && tmv "$f"
+}
+
+[ $no_dd ] && {
+	rm -rf copyparty/web/dd
+	f=copyparty/web/browser.css
+	sed -r 's/(cursor: )url\([^)]+\), (pointer)/\1\2/; /[0-9]+% \{cursor:/d; /animation: cursor/d' <$f >t && tmv "$f"
+}
+
 [ $repack ] ||
 find | grep -E '\.py$' |
   grep -vE '__version__' |