paulmataruso/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-11-03 13:33:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

sqlite durability profiles

Browse Source
This commit is contained in:
ed
2022-12-10 10:01:33 +00:00
parent 999b7ae919
commit 7704b9c8a2
5 changed files with 63 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
copyparty/__main__.py
View File

@@ -540,10 +540,7 @@ def run_argparse(
\033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
\033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
\033[36mnoforget$\033[35m don't forget files when deleted from disk
\033[36mnowal\033[35m guarantee zero dataloss on powerloss by disabling wal
\033[36mwal\033[35m enable wal (default; overrides --no-wal)
\033[36mnosync\033[35m switch to unsafe mode for extremely fast uploads
\033[36msync\033[35m normal corruption-protected db config
\033[36mdbd=[acid|swal|wal|yolo]\033[35m database speed-durability tradeoff
\033[36mxlink$\033[35m cross-volume dupe detection / linking
\033[36mxdev\033[35m do not descend into other filesystems
\033[36mxvol\033[35m skip symlinks leaving the volume root
@@ -604,6 +601,25 @@ def run_argparse(
"""
),
],
[
"dbd",
"database durability profiles",
dedent(
"""
mainly affects uploads of many small files on slow HDDs; speeds measured uploading 520 files on a WD20SPZX (SMR 2.5" 5400rpm 4kb)
\033[32macid\033[0m = extremely safe but slow; the old default. Should never lose any data no matter what
\033[32mswal\033[0m = 2.4x faster uploads yet 99.9%% as safe -- theoretical chance of losing metadata for the ~200 most recently uploaded files if there's a power-loss or your OS crashes
\033[32mwal\033[0m = another 21x faster on HDDs yet 90%% as safe; same pitfall as \033[33mswal\033[0m except more likely
\033[32myolo\033[0m = another 1.5x faster, and removes the occasional sudden upload-pause while the disk syncs, but now you're at risk of losing the entire database in a powerloss / OS-crash
profiles can be set globally (--dbd=yolo), or per-volume with volflags: -v ~/Music:music:r:c,dbd=acid
"""
),
],
]
# fmt: off
@@ -673,15 +689,15 @@ def run_argparse(
ap2 = ap.add_argument_group("Zeroconf options")
ap2.add_argument("-z", action="store_true", help="enable all zeroconf backends (mdns, ssdp)")
ap2.add_argument("--z-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes\n └─example: \033[32meth0, wlo1, virhost0, 192.168.123.0/24, fd00:fda::/96\033[0m")
ap2.add_argument("--z-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--z-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes\n └─example: \033[32meth0, wlo1, virhost0, 192.168.123.0/24, fd00:fda::/96\033[0m")
ap2.add_argument("--z-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("-zv", action="store_true", help="verbose all zeroconf backends")
ap2.add_argument("--mc-hop", metavar="SEC", type=int, default=0, help="rejoin multicast groups every SEC seconds (workaround for some switches/routers which cause mDNS to suddenly stop working after some time); try [\033[32m300\033[0m] or [\033[32m180\033[0m]")
ap2 = ap.add_argument_group("Zeroconf-mDNS options:")
ap2.add_argument("--zm", action="store_true", help="announce the enabled protocols over mDNS (multicast DNS-SD) -- compatible with KDE, gnome, macOS, ...")
ap2.add_argument("--zm-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zm-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zm-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zm-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zm4", action="store_true", help="IPv4 only -- try this if some clients can't connect")
ap2.add_argument("--zm6", action="store_true", help="IPv6 only")
ap2.add_argument("--zmv", action="store_true", help="verbose mdns")
@@ -697,8 +713,8 @@ def run_argparse(
ap2 = ap.add_argument_group("Zeroconf-SSDP options:")
ap2.add_argument("--zs", action="store_true", help="announce the enabled protocols over SSDP -- compatible with Windows")
ap2.add_argument("--zs-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zs-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zs-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zs-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zsv", action="store_true", help="verbose SSDP")
ap2.add_argument("--zsl", metavar="PATH", type=u, default="/?hc", help="location to include in the url (or a complete external URL), for example [\033[32mpriv/?pw=hunter2\033[0m] (goes directly to /priv/ with password hunter2) or [\033[32m?hc=priv&pw=hunter2\033[0m] (shows mounting options for /priv/ with password)")
ap2.add_argument("--zsid", metavar="UUID", type=u, default=uuid.uuid4().urn[4:], help="USN (device identifier) to announce")
@@ -819,8 +835,7 @@ def run_argparse(
ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans (volflag=noidx)")
ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice (volflag=noforget)")
ap2.add_argument("--no-wal", action="store_true", help="1%% faster searches, more reliable upload performance, and slightly more resistant to dataloss, but makes uploads up to 2x slower (volflag=nowal)")
ap2.add_argument("--no-sync", action="store_true", help="make uploads extremely fast, but removes corruption protection -- if the OS crashes, you might lose the db (volflag=nosync)")
ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see --help-dbd (volflag=dbd)")
ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...) (volflag=xdev)")
ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root (volflag=xvol)")

11
copyparty/authsrv.py
View File

@@ -15,7 +15,6 @@ from datetime import datetime
from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS
from .bos import bos
from .util import (
DROPLICATIONS,
IMPLICATIONS,
META_NOBOTS,
SQLITE_VER,
@@ -1123,8 +1122,6 @@ class AuthSrv(object):
for ga, vf in (
("no_forget", "noforget"),
("no_dupe", "nodupe"),
("no_wal", "nowal"),
("no_sync", "nosync"),
("magic", "magic"),
("xlink", "xlink"),
):
@@ -1139,9 +1136,11 @@ class AuthSrv(object):
if k1 in vol.flags:
vol.flags[k2] = False
for k1, k2 in DROPLICATIONS:
if k1 in vol.flags:
vol.flags.pop(k2)
dbds = "acid|swal|wal|yolo"
vol.flags["dbd"] = dbd = vol.flags.get("dbd") or self.args.dbd
if dbd not in dbds.split("|"):
t = "invalid dbd [{}]; must be one of [{}]"
raise Exception(t.format(dbd, dbds))
# default tag cfgs if unset
if "mte" not in vol.flags:

2
copyparty/u2idx.py
View File

@@ -97,7 +97,7 @@ class U2idx(object):
return None
cur = None
if ANYWIN and self.args.no_wal:
if ANYWIN and bos.path.exists(db_path + "-wal"):
uri = ""
try:
uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())

38
copyparty/up2k.py
View File

@@ -558,7 +558,7 @@ class Up2k(object):
return False
for vol in all_vols.values():
if "nowal" in vol.flags:
if vol.flags["dbd"] == "acid":
continue
reg = self.register_vpath(vol.realpath, vol.flags)
@@ -696,15 +696,37 @@ class Up2k(object):
cur = self._open_db(db_path)
self.cur[ptop] = cur
# speeds measured uploading 520 small files on a WD20SPZX (SMR 2.5" 5400rpm 4kb)
dbd = flags["dbd"]
if dbd == "acid":
# 217.5s; python-defaults
zs = "delete"
sync = "full"
elif dbd == "swal":
# 88.0s; still 99.9% safe (can lose a bit of on OS crash)
zs = "wal"
sync = "full"
elif dbd == "yolo":
# 2.7s; may lose entire db on OS crash
zs = "wal"
sync = "off"
else:
# 4.1s; corruption-safe but more likely to lose wal
zs = "wal"
sync = "normal"
try:
zs = "delete" if "nowal" in flags else "wal"
cur.execute("pragma journal_mode=" + zs)
except:
pass
if "nosync" in flags:
cur.execute("pragma synchronous=0")
amode = cur.execute("pragma journal_mode=" + zs).fetchone()[0]
if amode.lower() != zs.lower():
t = "sqlite failed to set journal_mode {}; got {}"
raise Exception(t.format(zs, amode))
except Exception as ex:
if sync != "off":
sync = "full"
t = "reverting to sync={} because {}"
self.log(t.format(sync, ex))
cur.execute("pragma synchronous=" + sync)
cur.connection.commit()
return cur, db_path
except:

3
copyparty/util.py
View File

@@ -202,9 +202,6 @@ if ANYWIN:
UNPLICATIONS = [["no_dav", "daw"]]
DROPLICATIONS = [["wal", "nowal"], ["sync", "nosync"]]
MIMES = {
"opus": "audio/ogg; codecs=opus",
}