mirror of
https://github.com/9001/copyparty.git
synced 2025-11-03 21:43:12 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
ff8313d0fb |
40
.github/ISSUE_TEMPLATE/bug_report.md
vendored
40
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -1,40 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: bug
|
||||
assignees: '9001'
|
||||
|
||||
---
|
||||
|
||||
NOTE:
|
||||
all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md
|
||||
|
||||
|
||||
**Describe the bug**
|
||||
a description of what the bug is
|
||||
|
||||
**To Reproduce**
|
||||
List of steps to reproduce the issue, or, if it's hard to reproduce, then at least a detailed explanation of what you did to run into it
|
||||
|
||||
**Expected behavior**
|
||||
a description of what you expected to happen
|
||||
|
||||
**Screenshots**
|
||||
if applicable, add screenshots to help explain your problem, such as the kickass crashpage :^)
|
||||
|
||||
**Server details**
|
||||
if the issue is possibly on the server-side, then mention some of the following:
|
||||
* server OS / version:
|
||||
* python version:
|
||||
* copyparty arguments:
|
||||
* filesystem (`lsblk -f` on linux):
|
||||
|
||||
**Client details**
|
||||
if the issue is possibly on the client-side, then mention some of the following:
|
||||
* the device type and model:
|
||||
* OS version:
|
||||
* browser version:
|
||||
|
||||
**Additional context**
|
||||
any other context about the problem here
|
||||
22
.github/ISSUE_TEMPLATE/feature_request.md
vendored
22
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -1,22 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: '9001'
|
||||
|
||||
---
|
||||
|
||||
all of the below are optional, consider them as inspiration, delete and rewrite at will
|
||||
|
||||
**is your feature request related to a problem? Please describe.**
|
||||
a description of what the problem is, for example, `I'm always frustrated when [...]` or `Why is it not possible to [...]`
|
||||
|
||||
**Describe the idea / solution you'd like**
|
||||
a description of what you want to happen
|
||||
|
||||
**Describe any alternatives you've considered**
|
||||
a description of any alternative solutions or features you've considered
|
||||
|
||||
**Additional context**
|
||||
add any other context or screenshots about the feature request here
|
||||
10
.github/ISSUE_TEMPLATE/something-else.md
vendored
10
.github/ISSUE_TEMPLATE/something-else.md
vendored
@@ -1,10 +0,0 @@
|
||||
---
|
||||
name: Something else
|
||||
about: "┐(ﾟ∀ﾟ)┌"
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
|
||||
7
.github/branch-rename.md
vendored
7
.github/branch-rename.md
vendored
@@ -1,7 +0,0 @@
|
||||
modernize your local checkout of the repo like so,
|
||||
```sh
|
||||
git branch -m master hovudstraum
|
||||
git fetch origin
|
||||
git branch -u origin/hovudstraum hovudstraum
|
||||
git remote set-head origin -a
|
||||
```
|
||||
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
@@ -1,2 +0,0 @@
|
||||
Please include the following text somewhere in this PR description:
|
||||
This PR complies with the DCO; https://developercertificate.org/
|
||||
29
.gitignore
vendored
29
.gitignore
vendored
@@ -5,39 +5,18 @@ __pycache__/
|
||||
MANIFEST.in
|
||||
MANIFEST
|
||||
copyparty.egg-info/
|
||||
buildenv/
|
||||
build/
|
||||
dist/
|
||||
sfx/
|
||||
.venv/
|
||||
|
||||
/buildenv/
|
||||
/build/
|
||||
/dist/
|
||||
/py2/
|
||||
/sfx*
|
||||
/unt/
|
||||
/log/
|
||||
|
||||
# ide
|
||||
*.sublime-workspace
|
||||
|
||||
# winmerge
|
||||
*.bak
|
||||
|
||||
# apple pls
|
||||
.DS_Store
|
||||
|
||||
# derived
|
||||
copyparty/res/COPYING.txt
|
||||
copyparty/web/deps/
|
||||
srv/
|
||||
scripts/docker/i/
|
||||
contrib/package/arch/pkg/
|
||||
contrib/package/arch/src/
|
||||
|
||||
# state/logs
|
||||
up.*.txt
|
||||
.hist/
|
||||
scripts/docker/*.out
|
||||
scripts/docker/*.err
|
||||
/perf.*
|
||||
|
||||
# nix build output link
|
||||
result
|
||||
|
||||
3
.vscode/launch.json
vendored
3
.vscode/launch.json
vendored
@@ -8,7 +8,6 @@
|
||||
"module": "copyparty",
|
||||
"console": "integratedTerminal",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"justMyCode": false,
|
||||
"args": [
|
||||
//"-nw",
|
||||
"-ed",
|
||||
@@ -18,7 +17,7 @@
|
||||
"-mtp",
|
||||
".bpm=f,bin/mtag/audio-bpm.py",
|
||||
"-aed:wark",
|
||||
"-vsrv::r:rw,ed:c,dupe",
|
||||
"-vsrv::r:aed:cnodupe",
|
||||
"-vdist:dist:r"
|
||||
]
|
||||
},
|
||||
|
||||
8
.vscode/launch.py
vendored
Executable file → Normal file
8
.vscode/launch.py
vendored
Executable file → Normal file
@@ -1,5 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# takes arguments from launch.json
|
||||
# is used by no_dbg in tasks.json
|
||||
# launches 10x faster than mspython debugpy
|
||||
@@ -11,15 +9,15 @@ import sys
|
||||
|
||||
print(sys.executable)
|
||||
|
||||
import json5
|
||||
import shlex
|
||||
import jstyleson
|
||||
import subprocess as sp
|
||||
|
||||
|
||||
with open(".vscode/launch.json", "r", encoding="utf-8") as f:
|
||||
tj = f.read()
|
||||
|
||||
oj = json5.loads(tj)
|
||||
oj = jstyleson.loads(tj)
|
||||
argv = oj["configurations"][0]["args"]
|
||||
|
||||
try:
|
||||
@@ -30,8 +28,6 @@ except:
|
||||
|
||||
argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
|
||||
|
||||
argv += sys.argv[1:]
|
||||
|
||||
if re.search(" -j ?[0-9]", " ".join(argv)):
|
||||
argv = [sys.executable, "-m", "copyparty"] + argv
|
||||
sp.check_call(argv)
|
||||
|
||||
29
.vscode/settings.json
vendored
29
.vscode/settings.json
vendored
@@ -23,6 +23,7 @@
|
||||
"terminal.ansiBrightWhite": "#ffffff",
|
||||
},
|
||||
"python.testing.pytestEnabled": false,
|
||||
"python.testing.nosetestsEnabled": false,
|
||||
"python.testing.unittestEnabled": true,
|
||||
"python.testing.unittestArgs": [
|
||||
"-v",
|
||||
@@ -34,42 +35,17 @@
|
||||
"python.linting.pylintEnabled": true,
|
||||
"python.linting.flake8Enabled": true,
|
||||
"python.linting.banditEnabled": true,
|
||||
"python.linting.mypyEnabled": true,
|
||||
"python.linting.mypyArgs": [
|
||||
"--ignore-missing-imports",
|
||||
"--follow-imports=silent",
|
||||
"--show-column-numbers",
|
||||
"--strict"
|
||||
],
|
||||
"python.linting.flake8Args": [
|
||||
"--max-line-length=120",
|
||||
"--ignore=E722,F405,E203,W503,W293,E402,E501,E128",
|
||||
"--ignore=E722,F405,E203,W503,W293,E402",
|
||||
],
|
||||
"python.linting.banditArgs": [
|
||||
"--ignore=B104"
|
||||
],
|
||||
"python.linting.pylintArgs": [
|
||||
"--disable=missing-module-docstring",
|
||||
"--disable=missing-class-docstring",
|
||||
"--disable=missing-function-docstring",
|
||||
"--disable=import-outside-toplevel",
|
||||
"--disable=wrong-import-position",
|
||||
"--disable=raise-missing-from",
|
||||
"--disable=bare-except",
|
||||
"--disable=broad-except",
|
||||
"--disable=invalid-name",
|
||||
"--disable=line-too-long",
|
||||
"--disable=consider-using-f-string"
|
||||
],
|
||||
// python3 -m isort --py=27 --profile=black copyparty/
|
||||
"python.formatting.provider": "black",
|
||||
"editor.formatOnSave": true,
|
||||
"[html]": {
|
||||
"editor.formatOnSave": false,
|
||||
"editor.autoIndent": "keep",
|
||||
},
|
||||
"[css]": {
|
||||
"editor.formatOnSave": false,
|
||||
},
|
||||
"files.associations": {
|
||||
"*.makefile": "makefile"
|
||||
@@ -79,5 +55,4 @@
|
||||
"py27"
|
||||
],
|
||||
"python.linting.enabled": true,
|
||||
"python.pythonPath": "/usr/bin/python3"
|
||||
}
|
||||
5
.vscode/tasks.json
vendored
5
.vscode/tasks.json
vendored
@@ -9,10 +9,7 @@
|
||||
{
|
||||
"label": "no_dbg",
|
||||
"type": "shell",
|
||||
"command": "${config:python.pythonPath}",
|
||||
"args": [
|
||||
".vscode/launch.py"
|
||||
]
|
||||
"command": "${config:python.pythonPath} .vscode/launch.py"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
in the words of Abraham Lincoln:
|
||||
|
||||
> Be excellent to each other... and... PARTY ON, DUDES!
|
||||
|
||||
more specifically I'll paraphrase some examples from a german automotive corporation as they cover all the bases without being too wordy
|
||||
|
||||
## Examples of unacceptable behavior
|
||||
* intimidation, harassment, trolling
|
||||
* insulting, derogatory, harmful or prejudicial comments
|
||||
* posting private information without permission
|
||||
* political or personal attacks
|
||||
|
||||
## Examples of expected behavior
|
||||
* being nice, friendly, welcoming, inclusive, mindful and empathetic
|
||||
* acting considerate, modest, respectful
|
||||
* using polite and inclusive language
|
||||
* criticize constructively and accept constructive criticism
|
||||
* respect different points of view
|
||||
|
||||
## finally and even more specifically,
|
||||
* parse opinions and feedback objectively without prejudice
|
||||
* it's the message that matters, not who said it
|
||||
|
||||
aaand that's how you say `be nice` in a way that fills half a floppy w
|
||||
@@ -1,3 +0,0 @@
|
||||
* do something cool
|
||||
|
||||
really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight
|
||||
@@ -1,9 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
if you hit something extra juicy pls let me know on either of the following
|
||||
* email -- `copyparty@ocv.ze` except `ze` should be `me`
|
||||
* [mastodon dm](https://layer8.space/@tripflag) -- `@tripflag@layer8.space`
|
||||
* [github private vulnerability report](https://github.com/9001/copyparty/security/advisories/new), wow that form is complicated
|
||||
* [twitter dm](https://twitter.com/tripflag) (if im somehow not banned yet)
|
||||
|
||||
no bug bounties sorry! all i can offer is greetz in the release notes
|
||||
@@ -1,18 +1,4 @@
|
||||
# [`up2k.py`](up2k.py)
|
||||
* command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
|
||||
* file uploads, file-search, autoresume of aborted/broken uploads
|
||||
* sync local folder to server
|
||||
* generally faster than browsers
|
||||
* if something breaks just restart it
|
||||
|
||||
|
||||
# [`partyjournal.py`](partyjournal.py)
|
||||
produces a chronological list of all uploads by collecting info from up2k databases and the filesystem
|
||||
* outputs a standalone html file
|
||||
* optional mapping from IP-addresses to nicknames
|
||||
|
||||
|
||||
# [`partyfuse.py`](partyfuse.py)
|
||||
# [`copyparty-fuse.py`](copyparty-fuse.py)
|
||||
* mount a copyparty server as a local filesystem (read-only)
|
||||
* **supports Windows!** -- expect `194 MiB/s` sequential read
|
||||
* **supports Linux** -- expect `117 MiB/s` sequential read
|
||||
@@ -31,19 +17,19 @@ also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x perfor
|
||||
* install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)
|
||||
* [x] add python 3.x to PATH (it asks during install)
|
||||
* `python -m pip install --user fusepy`
|
||||
* `python ./partyfuse.py n: http://192.168.1.69:3923/`
|
||||
* `python ./copyparty-fuse.py n: http://192.168.1.69:3923/`
|
||||
|
||||
10% faster in [msys2](https://www.msys2.org/), 700% faster if debug prints are enabled:
|
||||
* `pacman -S mingw64/mingw-w64-x86_64-python{,-pip}`
|
||||
* `/mingw64/bin/python3 -m pip install --user fusepy`
|
||||
* `/mingw64/bin/python3 ./partyfuse.py [...]`
|
||||
* `/mingw64/bin/python3 ./copyparty-fuse.py [...]`
|
||||
|
||||
you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releases/latest), let me know if you [figure out how](https://github.com/dokan-dev/dokany/wiki/FUSE)
|
||||
(winfsp's sshfs leaks, doesn't look like winfsp itself does, should be fine)
|
||||
|
||||
|
||||
|
||||
# [`partyfuse2.py`](partyfuse2.py)
|
||||
# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
|
||||
* mount a copyparty server as a local filesystem (read-only)
|
||||
* does the same thing except more correct, `samba` approves
|
||||
* **supports Linux** -- expect `18 MiB/s` (wait what)
|
||||
@@ -51,7 +37,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
|
||||
|
||||
|
||||
|
||||
# [`partyfuse-streaming.py`](partyfuse-streaming.py)
|
||||
# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
|
||||
* pretend this doesn't exist
|
||||
|
||||
|
||||
@@ -61,7 +47,6 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
|
||||
* copyparty can Popen programs like these during file indexing to collect additional metadata
|
||||
|
||||
|
||||
|
||||
# [`dbtool.py`](dbtool.py)
|
||||
upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
|
||||
|
||||
@@ -76,9 +61,3 @@ cd /mnt/nas/music/.hist
|
||||
~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy key
|
||||
~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
|
||||
```
|
||||
|
||||
|
||||
|
||||
# [`prisonparty.sh`](prisonparty.sh)
|
||||
* run copyparty in a chroot, preventing any accidental file access
|
||||
* creates bindmounts for /bin, /lib, and so on, see `sysdirs=`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
"""partyfuse-streaming: remote copyparty as a local filesystem"""
|
||||
"""copyparty-fuse-streaming: remote copyparty as a local filesystem"""
|
||||
__author__ = "ed <copyparty@ocv.me>"
|
||||
__copyright__ = 2020
|
||||
__license__ = "MIT"
|
||||
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
|
||||
mount a copyparty server (local or remote) as a filesystem
|
||||
|
||||
usage:
|
||||
python partyfuse-streaming.py http://192.168.1.69:3923/ ./music
|
||||
python copyparty-fuse-streaming.py http://192.168.1.69:3923/ ./music
|
||||
|
||||
dependencies:
|
||||
python3 -m pip install --user fusepy
|
||||
@@ -21,7 +21,7 @@ dependencies:
|
||||
+ on Windows: https://github.com/billziss-gh/winfsp/releases/latest
|
||||
|
||||
this was a mistake:
|
||||
fork of partyfuse.py with a streaming cache rather than readahead,
|
||||
fork of copyparty-fuse.py with a streaming cache rather than readahead,
|
||||
thought this was gonna be way faster (and it kind of is)
|
||||
except the overhead of reopening connections on trunc totally kills it
|
||||
"""
|
||||
@@ -42,7 +42,6 @@ import threading
|
||||
import traceback
|
||||
import http.client # py2: httplib
|
||||
import urllib.parse
|
||||
import calendar
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
@@ -62,12 +61,12 @@ except:
|
||||
else:
|
||||
libfuse = "apt install libfuse\n modprobe fuse"
|
||||
|
||||
m = """\033[33m
|
||||
could not import fuse; these may help:
|
||||
{} -m pip install --user fusepy
|
||||
{}
|
||||
\033[0m"""
|
||||
print(m.format(sys.executable, libfuse))
|
||||
print(
|
||||
"\n could not import fuse; these may help:"
|
||||
+ "\n python3 -m pip install --user fusepy\n "
|
||||
+ libfuse
|
||||
+ "\n"
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@@ -154,7 +153,7 @@ def dewin(txt):
|
||||
class RecentLog(object):
|
||||
def __init__(self):
|
||||
self.mtx = threading.Lock()
|
||||
self.f = None # open("partyfuse.log", "wb")
|
||||
self.f = None # open("copyparty-fuse.log", "wb")
|
||||
self.q = []
|
||||
|
||||
thr = threading.Thread(target=self.printer)
|
||||
@@ -185,9 +184,9 @@ class RecentLog(object):
|
||||
print("".join(q), end="")
|
||||
|
||||
|
||||
# [windows/cmd/cpy3] python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/cmd/cpy3] python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
|
||||
#
|
||||
# [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
|
||||
# [alpine] ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done
|
||||
@@ -346,7 +345,7 @@ class Gateway(object):
|
||||
except:
|
||||
pass
|
||||
|
||||
def sendreq(self, meth, path, headers, **kwargs):
|
||||
def sendreq(self, *args, headers={}, **kwargs):
|
||||
if self.password:
|
||||
headers["Cookie"] = "=".join(["cppwd", self.password])
|
||||
|
||||
@@ -355,21 +354,21 @@ class Gateway(object):
|
||||
if c.rx_path:
|
||||
raise Exception()
|
||||
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
c.rx = c.getresponse()
|
||||
return c
|
||||
except:
|
||||
tid = threading.current_thread().ident
|
||||
dbg(
|
||||
"\033[1;37;44mbad conn {:x}\n {} {}\n {}\033[0m".format(
|
||||
tid, meth, path, c.rx_path if c else "(null)"
|
||||
"\033[1;37;44mbad conn {:x}\n {}\n {}\033[0m".format(
|
||||
tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
|
||||
)
|
||||
)
|
||||
|
||||
self.closeconn(c)
|
||||
c = self.getconn()
|
||||
try:
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
c.rx = c.getresponse()
|
||||
return c
|
||||
except:
|
||||
@@ -387,7 +386,7 @@ class Gateway(object):
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
|
||||
c = self.sendreq("GET", web_path, {})
|
||||
c = self.sendreq("GET", web_path)
|
||||
if c.rx.status != 200:
|
||||
self.closeconn(c)
|
||||
log(
|
||||
@@ -441,7 +440,7 @@ class Gateway(object):
|
||||
)
|
||||
)
|
||||
|
||||
c = self.sendreq("GET", web_path, {"Range": hdr_range})
|
||||
c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
|
||||
if c.rx.status != http.client.PARTIAL_CONTENT:
|
||||
self.closeconn(c)
|
||||
raise Exception(
|
||||
@@ -496,7 +495,7 @@ class Gateway(object):
|
||||
ts = 60 * 60 * 24 * 2
|
||||
try:
|
||||
sz = int(fsize)
|
||||
ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
|
||||
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
except:
|
||||
info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
|
||||
# python cannot strptime(1959-01-01) on windows
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
"""partyfuse: remote copyparty as a local filesystem"""
|
||||
"""copyparty-fuse: remote copyparty as a local filesystem"""
|
||||
__author__ = "ed <copyparty@ocv.me>"
|
||||
__copyright__ = 2019
|
||||
__license__ = "MIT"
|
||||
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
|
||||
mount a copyparty server (local or remote) as a filesystem
|
||||
|
||||
usage:
|
||||
python partyfuse.py http://192.168.1.69:3923/ ./music
|
||||
python copyparty-fuse.py http://192.168.1.69:3923/ ./music
|
||||
|
||||
dependencies:
|
||||
python3 -m pip install --user fusepy
|
||||
@@ -22,7 +22,7 @@ dependencies:
|
||||
|
||||
note:
|
||||
you probably want to run this on windows clients:
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/contrib/explorer-nothumbs-nofoldertypes.reg
|
||||
https://github.com/9001/copyparty/blob/master/contrib/explorer-nothumbs-nofoldertypes.reg
|
||||
|
||||
get server cert:
|
||||
awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
|
||||
@@ -45,7 +45,6 @@ import threading
|
||||
import traceback
|
||||
import http.client # py2: httplib
|
||||
import urllib.parse
|
||||
import calendar
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
@@ -55,13 +54,10 @@ MACOS = platform.system() == "Darwin"
|
||||
info = log = dbg = None
|
||||
|
||||
|
||||
print(
|
||||
"{} v{} @ {}".format(
|
||||
platform.python_implementation(),
|
||||
".".join([str(x) for x in sys.version_info]),
|
||||
sys.executable,
|
||||
)
|
||||
)
|
||||
print("{} v{} @ {}".format(
|
||||
platform.python_implementation(),
|
||||
".".join([str(x) for x in sys.version_info]),
|
||||
sys.executable))
|
||||
|
||||
|
||||
try:
|
||||
@@ -72,14 +68,14 @@ except:
|
||||
elif MACOS:
|
||||
libfuse = "install https://osxfuse.github.io/"
|
||||
else:
|
||||
libfuse = "apt install libfuse3-3\n modprobe fuse"
|
||||
libfuse = "apt install libfuse\n modprobe fuse"
|
||||
|
||||
m = """\033[33m
|
||||
could not import fuse; these may help:
|
||||
{} -m pip install --user fusepy
|
||||
{}
|
||||
\033[0m"""
|
||||
print(m.format(sys.executable, libfuse))
|
||||
print(
|
||||
"\n could not import fuse; these may help:"
|
||||
+ "\n python3 -m pip install --user fusepy\n "
|
||||
+ libfuse
|
||||
+ "\n"
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@@ -166,7 +162,7 @@ def dewin(txt):
|
||||
class RecentLog(object):
|
||||
def __init__(self):
|
||||
self.mtx = threading.Lock()
|
||||
self.f = None # open("partyfuse.log", "wb")
|
||||
self.f = None # open("copyparty-fuse.log", "wb")
|
||||
self.q = []
|
||||
|
||||
thr = threading.Thread(target=self.printer)
|
||||
@@ -197,9 +193,9 @@ class RecentLog(object):
|
||||
print("".join(q), end="")
|
||||
|
||||
|
||||
# [windows/cmd/cpy3] python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/cmd/cpy3] python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
|
||||
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
|
||||
#
|
||||
# [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
|
||||
# [alpine] ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done
|
||||
@@ -303,14 +299,14 @@ class Gateway(object):
|
||||
except:
|
||||
pass
|
||||
|
||||
def sendreq(self, meth, path, headers, **kwargs):
|
||||
def sendreq(self, *args, headers={}, **kwargs):
|
||||
tid = get_tid()
|
||||
if self.password:
|
||||
headers["Cookie"] = "=".join(["cppwd", self.password])
|
||||
|
||||
try:
|
||||
c = self.getconn(tid)
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
return c.getresponse()
|
||||
except:
|
||||
dbg("bad conn")
|
||||
@@ -318,7 +314,7 @@ class Gateway(object):
|
||||
self.closeconn(tid)
|
||||
try:
|
||||
c = self.getconn(tid)
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
return c.getresponse()
|
||||
except:
|
||||
info("http connection failed:\n" + traceback.format_exc())
|
||||
@@ -335,7 +331,7 @@ class Gateway(object):
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
|
||||
r = self.sendreq("GET", web_path, {})
|
||||
r = self.sendreq("GET", web_path)
|
||||
if r.status != 200:
|
||||
self.closeconn()
|
||||
log(
|
||||
@@ -372,7 +368,7 @@ class Gateway(object):
|
||||
)
|
||||
)
|
||||
|
||||
r = self.sendreq("GET", web_path, {"Range": hdr_range})
|
||||
r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
|
||||
if r.status != http.client.PARTIAL_CONTENT:
|
||||
self.closeconn()
|
||||
raise Exception(
|
||||
@@ -394,16 +390,15 @@ class Gateway(object):
|
||||
|
||||
rsp = json.loads(rsp.decode("utf-8"))
|
||||
ret = []
|
||||
for statfun, nodes in [
|
||||
[self.stat_dir, rsp["dirs"]],
|
||||
[self.stat_file, rsp["files"]],
|
||||
]:
|
||||
for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
|
||||
for n in nodes:
|
||||
fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
|
||||
fname = unquote(n["href"]).rstrip(b"/")
|
||||
fname = fname.decode("wtf-8")
|
||||
if bad_good:
|
||||
fname = enwin(fname)
|
||||
|
||||
ret.append([fname, statfun(n["ts"], n["sz"]), 0])
|
||||
fun = self.stat_dir if is_dir else self.stat_file
|
||||
ret.append([fname, fun(n["ts"], n["sz"]), 0])
|
||||
|
||||
return ret
|
||||
|
||||
@@ -444,7 +439,7 @@ class Gateway(object):
|
||||
ts = 60 * 60 * 24 * 2
|
||||
try:
|
||||
sz = int(fsize)
|
||||
ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
|
||||
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
except:
|
||||
info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
|
||||
# python cannot strptime(1959-01-01) on windows
|
||||
@@ -997,7 +992,7 @@ def main():
|
||||
ap.add_argument(
|
||||
"-cf", metavar="NUM_BLOCKS", type=int, default=nf, help="file cache"
|
||||
)
|
||||
ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
|
||||
ap.add_argument("-a", metavar="PASSWORD", help="password")
|
||||
ap.add_argument("-d", action="store_true", help="enable debug")
|
||||
ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
|
||||
ap.add_argument("-td", action="store_true", help="disable certificate check")
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
"""partyfuse2: remote copyparty as a local filesystem"""
|
||||
"""copyparty-fuseb: remote copyparty as a local filesystem"""
|
||||
__author__ = "ed <copyparty@ocv.me>"
|
||||
__copyright__ = 2020
|
||||
__license__ = "MIT"
|
||||
@@ -11,18 +11,14 @@ import re
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import stat
|
||||
import errno
|
||||
import struct
|
||||
import codecs
|
||||
import platform
|
||||
import threading
|
||||
import http.client # py2: httplib
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
try:
|
||||
import fuse
|
||||
@@ -32,19 +28,9 @@ try:
|
||||
if not hasattr(fuse, "__version__"):
|
||||
raise Exception("your fuse-python is way old")
|
||||
except:
|
||||
if WINDOWS:
|
||||
libfuse = "install https://github.com/billziss-gh/winfsp/releases/latest"
|
||||
elif MACOS:
|
||||
libfuse = "install https://osxfuse.github.io/"
|
||||
else:
|
||||
libfuse = "apt install libfuse\n modprobe fuse"
|
||||
|
||||
m = """\033[33m
|
||||
could not import fuse; these may help:
|
||||
{} -m pip install --user fuse-python
|
||||
{}
|
||||
\033[0m"""
|
||||
print(m.format(sys.executable, libfuse))
|
||||
print(
|
||||
"\n could not import fuse; these may help:\n python3 -m pip install --user fuse-python\n apt install libfuse\n modprobe fuse\n"
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@@ -52,22 +38,18 @@ except:
|
||||
mount a copyparty server (local or remote) as a filesystem
|
||||
|
||||
usage:
|
||||
python ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
|
||||
python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas
|
||||
|
||||
dependencies:
|
||||
sudo apk add fuse-dev python3-dev
|
||||
python3 -m pip install --user fuse-python
|
||||
|
||||
fork of partyfuse.py based on fuse-python which
|
||||
fork of copyparty-fuse.py based on fuse-python which
|
||||
appears to be more compliant than fusepy? since this works with samba
|
||||
(probably just my garbage code tbh)
|
||||
"""
|
||||
|
||||
|
||||
WINDOWS = sys.platform == "win32"
|
||||
MACOS = platform.system() == "Darwin"
|
||||
|
||||
|
||||
def threadless_log(msg):
|
||||
print(msg + "\n", end="")
|
||||
|
||||
@@ -111,41 +93,6 @@ def html_dec(txt):
|
||||
)
|
||||
|
||||
|
||||
def register_wtf8():
|
||||
def wtf8_enc(text):
|
||||
return str(text).encode("utf-8", "surrogateescape"), len(text)
|
||||
|
||||
def wtf8_dec(binary):
|
||||
return bytes(binary).decode("utf-8", "surrogateescape"), len(binary)
|
||||
|
||||
def wtf8_search(encoding_name):
|
||||
return codecs.CodecInfo(wtf8_enc, wtf8_dec, name="wtf-8")
|
||||
|
||||
codecs.register(wtf8_search)
|
||||
|
||||
|
||||
bad_good = {}
|
||||
good_bad = {}
|
||||
|
||||
|
||||
def enwin(txt):
|
||||
return "".join([bad_good.get(x, x) for x in txt])
|
||||
|
||||
for bad, good in bad_good.items():
|
||||
txt = txt.replace(bad, good)
|
||||
|
||||
return txt
|
||||
|
||||
|
||||
def dewin(txt):
|
||||
return "".join([good_bad.get(x, x) for x in txt])
|
||||
|
||||
for bad, good in bad_good.items():
|
||||
txt = txt.replace(good, bad)
|
||||
|
||||
return txt
|
||||
|
||||
|
||||
class CacheNode(object):
|
||||
def __init__(self, tag, data):
|
||||
self.tag = tag
|
||||
@@ -168,9 +115,8 @@ class Stat(fuse.Stat):
|
||||
|
||||
|
||||
class Gateway(object):
|
||||
def __init__(self, base_url, pw):
|
||||
def __init__(self, base_url):
|
||||
self.base_url = base_url
|
||||
self.pw = pw
|
||||
|
||||
ui = urllib.parse.urlparse(base_url)
|
||||
self.web_root = ui.path.strip("/")
|
||||
@@ -189,7 +135,8 @@ class Gateway(object):
|
||||
self.conns = {}
|
||||
|
||||
def quotep(self, path):
|
||||
path = path.encode("wtf-8")
|
||||
# TODO: mojibake support
|
||||
path = path.encode("utf-8", "ignore")
|
||||
return quote(path, safe="/")
|
||||
|
||||
def getconn(self, tid=None):
|
||||
@@ -212,29 +159,20 @@ class Gateway(object):
|
||||
except:
|
||||
pass
|
||||
|
||||
def sendreq(self, *args, **ka):
|
||||
def sendreq(self, *args, **kwargs):
|
||||
tid = get_tid()
|
||||
if self.pw:
|
||||
ck = "cppwd=" + self.pw
|
||||
try:
|
||||
ka["headers"]["Cookie"] = ck
|
||||
except:
|
||||
ka["headers"] = {"Cookie": ck}
|
||||
try:
|
||||
c = self.getconn(tid)
|
||||
c.request(*list(args), **ka)
|
||||
c.request(*list(args), **kwargs)
|
||||
return c.getresponse()
|
||||
except:
|
||||
self.closeconn(tid)
|
||||
c = self.getconn(tid)
|
||||
c.request(*list(args), **ka)
|
||||
c.request(*list(args), **kwargs)
|
||||
return c.getresponse()
|
||||
|
||||
def listdir(self, path):
|
||||
if bad_good:
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
|
||||
r = self.sendreq("GET", web_path)
|
||||
if r.status != 200:
|
||||
self.closeconn()
|
||||
@@ -244,12 +182,9 @@ class Gateway(object):
|
||||
)
|
||||
)
|
||||
|
||||
return self.parse_jls(r)
|
||||
return self.parse_html(r)
|
||||
|
||||
def download_file_range(self, path, ofs1, ofs2):
|
||||
if bad_good:
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
|
||||
hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
|
||||
log("downloading {}".format(hdr_range))
|
||||
@@ -265,27 +200,40 @@ class Gateway(object):
|
||||
|
||||
return r.read()
|
||||
|
||||
def parse_jls(self, datasrc):
|
||||
rsp = b""
|
||||
def parse_html(self, datasrc):
|
||||
ret = []
|
||||
remainder = b""
|
||||
ptn = re.compile(
|
||||
r"^<tr><td>(-|DIR)</td><td><a [^>]+>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$"
|
||||
)
|
||||
|
||||
while True:
|
||||
buf = datasrc.read(1024 * 32)
|
||||
buf = remainder + datasrc.read(4096)
|
||||
# print('[{}]'.format(buf.decode('utf-8')))
|
||||
if not buf:
|
||||
break
|
||||
|
||||
rsp += buf
|
||||
remainder = b""
|
||||
endpos = buf.rfind(b"\n")
|
||||
if endpos >= 0:
|
||||
remainder = buf[endpos + 1 :]
|
||||
buf = buf[:endpos]
|
||||
|
||||
rsp = json.loads(rsp.decode("utf-8"))
|
||||
ret = []
|
||||
for statfun, nodes in [
|
||||
[self.stat_dir, rsp["dirs"]],
|
||||
[self.stat_file, rsp["files"]],
|
||||
]:
|
||||
for n in nodes:
|
||||
fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
|
||||
if bad_good:
|
||||
fname = enwin(fname)
|
||||
lines = buf.decode("utf-8").split("\n")
|
||||
for line in lines:
|
||||
m = ptn.match(line)
|
||||
if not m:
|
||||
# print(line)
|
||||
continue
|
||||
|
||||
ret.append([fname, statfun(n["ts"], n["sz"]), 0])
|
||||
ftype, fname, fsize, fdate = m.groups()
|
||||
fname = html_dec(fname)
|
||||
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
sz = int(fsize)
|
||||
if ftype == "-":
|
||||
ret.append([fname, self.stat_file(ts, sz), 0])
|
||||
else:
|
||||
ret.append([fname, self.stat_dir(ts, sz), 0])
|
||||
|
||||
return ret
|
||||
|
||||
@@ -314,7 +262,6 @@ class CPPF(Fuse):
|
||||
Fuse.__init__(self, *args, **kwargs)
|
||||
|
||||
self.url = None
|
||||
self.pw = None
|
||||
|
||||
self.dircache = []
|
||||
self.dircache_mtx = threading.Lock()
|
||||
@@ -324,7 +271,7 @@ class CPPF(Fuse):
|
||||
|
||||
def init2(self):
|
||||
# TODO figure out how python-fuse wanted this to go
|
||||
self.gw = Gateway(self.url, self.pw) # .decode('utf-8'))
|
||||
self.gw = Gateway(self.url) # .decode('utf-8'))
|
||||
info("up")
|
||||
|
||||
def clean_dircache(self):
|
||||
@@ -589,8 +536,6 @@ class CPPF(Fuse):
|
||||
|
||||
def getattr(self, path):
|
||||
log("getattr [{}]".format(path))
|
||||
if WINDOWS:
|
||||
path = enwin(path) # windows occasionally decodes f0xx to xx
|
||||
|
||||
path = path.strip("/")
|
||||
try:
|
||||
@@ -623,25 +568,9 @@ class CPPF(Fuse):
|
||||
|
||||
def main():
|
||||
time.strptime("19970815", "%Y%m%d") # python#7980
|
||||
register_wtf8()
|
||||
if WINDOWS:
|
||||
os.system("rem")
|
||||
|
||||
for ch in '<>:"\\|?*':
|
||||
# microsoft maps illegal characters to f0xx
|
||||
# (e000 to f8ff is basic-plane private-use)
|
||||
bad_good[ch] = chr(ord(ch) + 0xF000)
|
||||
|
||||
for n in range(0, 0x100):
|
||||
# map surrogateescape to another private-use area
|
||||
bad_good[chr(n + 0xDC00)] = chr(n + 0xF100)
|
||||
|
||||
for k, v in bad_good.items():
|
||||
good_bad[v] = k
|
||||
|
||||
server = CPPF()
|
||||
server.parser.add_option(mountopt="url", metavar="BASE_URL", default=None)
|
||||
server.parser.add_option(mountopt="pw", metavar="PASSWORD", default=None)
|
||||
server.parse(values=server, errex=1)
|
||||
if not server.url or not str(server.url).startswith("http"):
|
||||
print("\nerror:")
|
||||
@@ -649,7 +578,7 @@ def main():
|
||||
print(" need argument: mount-path")
|
||||
print("example:")
|
||||
print(
|
||||
" ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
|
||||
" ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
109
bin/dbtool.py
109
bin/dbtool.py
@@ -8,10 +8,7 @@ import sqlite3
|
||||
import argparse
|
||||
|
||||
DB_VER1 = 3
|
||||
DB_VER2 = 5
|
||||
|
||||
BY_PATH = None
|
||||
NC = None
|
||||
DB_VER2 = 4
|
||||
|
||||
|
||||
def die(msg):
|
||||
@@ -60,13 +57,8 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
if BY_PATH:
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
hit = d2.execute(q, (rd, fn)).fetchone()
|
||||
else:
|
||||
q = "select w from up where substr(w,1,16) = ? and +w = ?"
|
||||
hit = d2.execute(q, (w1[:16], w1)).fetchone()
|
||||
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
hit = d2.execute(q, (rd, fn)).fetchone()
|
||||
if not hit:
|
||||
miss += 1
|
||||
if verbose:
|
||||
@@ -78,32 +70,27 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
n = 0
|
||||
miss = {}
|
||||
nmiss = 0
|
||||
for w1s, k, v in d1.execute("select * from mt"):
|
||||
for w1, k, v in d1.execute("select * from mt"):
|
||||
|
||||
n += 1
|
||||
if n % 100_000 == 0:
|
||||
m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
|
||||
print(m)
|
||||
|
||||
q = "select w, rd, fn from up where substr(w,1,16) = ?"
|
||||
w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w1,)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
if BY_PATH:
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
else:
|
||||
q = "select w from up where substr(w,1,16) = ? and +w = ?"
|
||||
w2 = d2.execute(q, (w1s, w1)).fetchone()
|
||||
|
||||
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
if w2:
|
||||
w2 = w2[0]
|
||||
|
||||
v2 = None
|
||||
if w2:
|
||||
v2 = d2.execute(
|
||||
"select v from mt where w = ? and +k = ?", (w2[:16], k)
|
||||
"select v from mt where w = ? and +k = ?", (w2, k)
|
||||
).fetchone()
|
||||
if v2:
|
||||
v2 = v2[0]
|
||||
@@ -137,7 +124,7 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
|
||||
for k, v in sorted(miss.items()):
|
||||
if v:
|
||||
print(f"{n1} has {v:7} more {k:<7} tags than {n2}")
|
||||
print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
|
||||
|
||||
print(f"in total, {nmiss} missing tags in {n2}\n")
|
||||
|
||||
@@ -145,75 +132,47 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
def copy_mtp(d1, d2, tag, rm):
|
||||
nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
|
||||
n = 0
|
||||
ncopy = 0
|
||||
nskip = 0
|
||||
for w1s, k, v in d1.execute("select * from mt where k = ?", (tag,)):
|
||||
ndone = 0
|
||||
for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
|
||||
n += 1
|
||||
if n % 25_000 == 0:
|
||||
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ncopy} copied, {nskip} skipped\033[0m"
|
||||
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
|
||||
print(m)
|
||||
|
||||
q = "select w, rd, fn from up where substr(w,1,16) = ?"
|
||||
w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w1,)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
if BY_PATH:
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
else:
|
||||
q = "select w from up where substr(w,1,16) = ? and +w = ?"
|
||||
w2 = d2.execute(q, (w1s, w1)).fetchone()
|
||||
|
||||
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
if not w2:
|
||||
continue
|
||||
|
||||
w2s = w2[0][:16]
|
||||
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2s, k)).fetchone()
|
||||
w2 = w2[0]
|
||||
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
|
||||
if hit:
|
||||
hit = hit[0]
|
||||
|
||||
if hit != v:
|
||||
if NC and hit is not None:
|
||||
nskip += 1
|
||||
continue
|
||||
|
||||
ncopy += 1
|
||||
ndone += 1
|
||||
if hit is not None:
|
||||
d2.execute("delete from mt where w = ? and +k = ?", (w2s, k))
|
||||
d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
|
||||
|
||||
d2.execute("insert into mt values (?,?,?)", (w2s, k, v))
|
||||
d2.execute("insert into mt values (?,?,?)", (w2, k, v))
|
||||
if rm:
|
||||
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2s,))
|
||||
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
|
||||
|
||||
d2.commit()
|
||||
print(f"copied {ncopy} {tag} tags over, skipped {nskip}")
|
||||
|
||||
|
||||
def examples():
|
||||
print(
|
||||
"""
|
||||
# clearing the journal
|
||||
./dbtool.py up2k.db
|
||||
|
||||
# copy tags ".bpm" and "key" from old.db to up2k.db, and remove the mtp flag from matching files (so copyparty won't run any mtps on it)
|
||||
./dbtool.py -ls up2k.db
|
||||
./dbtool.py -src old.db up2k.db -cmp
|
||||
./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy key
|
||||
./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
|
||||
|
||||
"""
|
||||
)
|
||||
print(f"copied {ndone} {tag} tags over")
|
||||
|
||||
|
||||
def main():
|
||||
global NC, BY_PATH
|
||||
os.system("")
|
||||
print()
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("db", help="database to work on")
|
||||
ap.add_argument("-h2", action="store_true", help="show examples")
|
||||
ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
|
||||
|
||||
ap2 = ap.add_argument_group("informational / read-only stuff")
|
||||
@@ -226,29 +185,11 @@ def main():
|
||||
ap2.add_argument(
|
||||
"-rm-mtp-flag",
|
||||
action="store_true",
|
||||
help="when an mtp tag is copied over, also mark that file as done, so copyparty won't run any mtps on those files",
|
||||
help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
|
||||
)
|
||||
ap2.add_argument("-vac", action="store_true", help="optimize DB")
|
||||
|
||||
ap2 = ap.add_argument_group("behavior modifiers")
|
||||
ap2.add_argument(
|
||||
"-nc",
|
||||
action="store_true",
|
||||
help="no-clobber; don't replace/overwrite existing tags",
|
||||
)
|
||||
ap2.add_argument(
|
||||
"-by-path",
|
||||
action="store_true",
|
||||
help="match files based on location rather than warks (content-hash), use this if the databases have different wark salts",
|
||||
)
|
||||
|
||||
ar = ap.parse_args()
|
||||
if ar.h2:
|
||||
examples()
|
||||
return
|
||||
|
||||
NC = ar.nc
|
||||
BY_PATH = ar.by_path
|
||||
|
||||
for v in [ar.db, ar.src]:
|
||||
if v and not os.path.exists(v):
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
standalone programs which are executed by copyparty when an event happens (upload, file rename, delete, ...)
|
||||
|
||||
these programs either take zero arguments, or a filepath (the affected file), or a json message with filepath + additional info
|
||||
|
||||
run copyparty with `--help-hooks` for usage details / hook type explanations (xbu/xau/xiu/xbr/xar/xbd/xad)
|
||||
|
||||
> **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
|
||||
|
||||
|
||||
# after upload
|
||||
* [notify.py](notify.py) shows a desktop notification ([example](https://user-images.githubusercontent.com/241032/215335767-9c91ed24-d36e-4b6b-9766-fb95d12d163f.png))
|
||||
* [notify2.py](notify2.py) uses the json API to show more context
|
||||
* [image-noexif.py](image-noexif.py) removes image exif by overwriting / directly editing the uploaded file
|
||||
* [discord-announce.py](discord-announce.py) announces new uploads on discord using webhooks ([example](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png))
|
||||
* [reject-mimetype.py](reject-mimetype.py) rejects uploads unless the mimetype is acceptable
|
||||
|
||||
|
||||
# upload batches
|
||||
these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every single file), `xiu` hooks are given a list of recent uploads on STDIN after the server has gone idle for N seconds, reducing server load + providing more context
|
||||
* [xiu.py](xiu.py) is a "minimal" example showing a list of filenames + total filesize
|
||||
* [xiu-sha.py](xiu-sha.py) produces a sha512 checksum list in the volume root
|
||||
|
||||
|
||||
# before upload
|
||||
* [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
|
||||
|
||||
|
||||
# on message
|
||||
* [wget.py](wget.py) lets you download files by POSTing URLs to copyparty
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import json
|
||||
import requests
|
||||
from copyparty.util import humansize, quotep
|
||||
|
||||
|
||||
_ = r"""
|
||||
announces a new upload on discord
|
||||
|
||||
example usage as global config:
|
||||
--xau f,t5,j,bin/hooks/discord-announce.py
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xau=f,t5,j,bin/hooks/discord-announce.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all uploads with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xbu = execute after upload
|
||||
f = fork; don't wait for it to finish
|
||||
t5 = timeout if it's still running after 5 sec
|
||||
j = provide upload information as json; not just the filename
|
||||
|
||||
replace "xau" with "xbu" to announce Before upload starts instead of After completion
|
||||
|
||||
# how to discord:
|
||||
first create the webhook url; https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks
|
||||
then use this to design your message: https://discohook.org/
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
WEBHOOK = "https://discord.com/api/webhooks/1234/base64"
|
||||
WEBHOOK = "https://discord.com/api/webhooks/1066830390280597718/M1TDD110hQA-meRLMRhdurych8iyG35LDoI1YhzbrjGP--BXNZodZFczNVwK4Ce7Yme5"
|
||||
|
||||
# read info from copyparty
|
||||
inf = json.loads(sys.argv[1])
|
||||
vpath = inf["vp"]
|
||||
filename = vpath.split("/")[-1]
|
||||
url = f"https://{inf['host']}/{quotep(vpath)}"
|
||||
|
||||
# compose the message to discord
|
||||
j = {
|
||||
"title": filename,
|
||||
"url": url,
|
||||
"description": url.rsplit("/", 1)[0],
|
||||
"color": 0x449900,
|
||||
"fields": [
|
||||
{"name": "Size", "value": humansize(inf["sz"])},
|
||||
{"name": "User", "value": inf["user"]},
|
||||
{"name": "IP", "value": inf["ip"]},
|
||||
],
|
||||
}
|
||||
|
||||
for v in j["fields"]:
|
||||
v["inline"] = True
|
||||
|
||||
r = requests.post(WEBHOOK, json={"embeds": [j]})
|
||||
print(f"discord: {r}\n", end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
|
||||
|
||||
_ = r"""
|
||||
remove exif tags from uploaded images; the eventhook edition of
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/image-noexif.py
|
||||
|
||||
dependencies:
|
||||
exiftool / perl-Image-ExifTool
|
||||
|
||||
being an upload hook, this will take effect after upload completion
|
||||
but before copyparty has hashed/indexed the file, which means that
|
||||
copyparty will never index the original file, so deduplication will
|
||||
not work as expected... which is mostly OK but ehhh
|
||||
|
||||
note: modifies the file in-place, so don't set the `f` (fork) flag
|
||||
|
||||
example usages; either as global config (all volumes) or as volflag:
|
||||
--xau bin/hooks/image-noexif.py
|
||||
-v srv/inc:inc:r:rw,ed:c,xau=bin/hooks/image-noexif.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
explained:
|
||||
share fs-path srv/inc at /inc (readable by all, read-write for user ed)
|
||||
running this xau (execute-after-upload) plugin for all uploaded files
|
||||
"""
|
||||
|
||||
|
||||
# filetypes to process; ignores everything else
|
||||
EXTS = ("jpg", "jpeg", "avif", "heif", "heic")
|
||||
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
fp = sys.argv[1]
|
||||
ext = fp.lower().split(".")[-1]
|
||||
if ext not in EXTS:
|
||||
return
|
||||
|
||||
cwd, fn = os.path.split(fp)
|
||||
os.chdir(cwd)
|
||||
f1 = fsenc(fn)
|
||||
cmd = [
|
||||
b"exiftool",
|
||||
b"-exif:all=",
|
||||
b"-iptc:all=",
|
||||
b"-xmp:all=",
|
||||
b"-P",
|
||||
b"-overwrite_original",
|
||||
b"--",
|
||||
f1,
|
||||
]
|
||||
sp.check_output(cmd)
|
||||
print("image-noexif: stripped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except:
|
||||
pass
|
||||
@@ -1,66 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from plyer import notification
|
||||
|
||||
|
||||
_ = r"""
|
||||
show os notification on upload; works on windows, linux, macos, android
|
||||
|
||||
depdencies:
|
||||
windows: python3 -m pip install --user -U plyer
|
||||
linux: python3 -m pip install --user -U plyer
|
||||
macos: python3 -m pip install --user -U plyer pyobjus
|
||||
android: just termux and termux-api
|
||||
|
||||
example usages; either as global config (all volumes) or as volflag:
|
||||
--xau f,bin/hooks/notify.py
|
||||
-v srv/inc:inc:r:rw,ed:c,xau=f,bin/hooks/notify.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all uploads with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xau = execute after upload
|
||||
f = fork so it doesn't block uploads
|
||||
"""
|
||||
|
||||
|
||||
try:
|
||||
from copyparty.util import humansize
|
||||
except:
|
||||
|
||||
def humansize(n):
|
||||
return n
|
||||
|
||||
|
||||
def main():
|
||||
fp = sys.argv[1]
|
||||
dp, fn = os.path.split(fp)
|
||||
try:
|
||||
sz = humansize(os.path.getsize(fp))
|
||||
except:
|
||||
sz = "?"
|
||||
|
||||
msg = "{} ({})\n📁 {}".format(fn, sz, dp)
|
||||
title = "File received"
|
||||
|
||||
if "com.termux" in sys.executable:
|
||||
sp.run(["termux-notification", "-t", title, "-c", msg])
|
||||
return
|
||||
|
||||
icon = "emblem-documents-symbolic" if sys.platform == "linux" else ""
|
||||
notification.notify(
|
||||
title=title,
|
||||
message=msg,
|
||||
app_icon=icon,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from datetime import datetime
|
||||
from plyer import notification
|
||||
|
||||
|
||||
_ = r"""
|
||||
same as notify.py but with additional info (uploader, ...)
|
||||
and also supports --xm (notify on 📟 message)
|
||||
|
||||
example usages; either as global config (all volumes) or as volflag:
|
||||
--xm f,j,bin/hooks/notify2.py
|
||||
--xau f,j,bin/hooks/notify2.py
|
||||
-v srv/inc:inc:r:rw,ed:c,xm=f,j,bin/hooks/notify2.py
|
||||
-v srv/inc:inc:r:rw,ed:c,xau=f,j,bin/hooks/notify2.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all uploads / msgs with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xau = execute after upload
|
||||
f = fork so it doesn't block uploads
|
||||
j = provide json instead of filepath list
|
||||
"""
|
||||
|
||||
|
||||
try:
|
||||
from copyparty.util import humansize
|
||||
except:
|
||||
|
||||
def humansize(n):
|
||||
return n
|
||||
|
||||
|
||||
def main():
|
||||
inf = json.loads(sys.argv[1])
|
||||
fp = inf["ap"]
|
||||
sz = humansize(inf["sz"])
|
||||
dp, fn = os.path.split(fp)
|
||||
mt = datetime.utcfromtimestamp(inf["mt"]).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
msg = f"{fn} ({sz})\n📁 {dp}"
|
||||
title = "File received"
|
||||
icon = "emblem-documents-symbolic" if sys.platform == "linux" else ""
|
||||
|
||||
if inf.get("txt"):
|
||||
msg = inf["txt"]
|
||||
title = "Message received"
|
||||
icon = "mail-unread-symbolic" if sys.platform == "linux" else ""
|
||||
|
||||
msg += f"\n👤 {inf['user']} ({inf['ip']})\n🕒 {mt}"
|
||||
|
||||
if "com.termux" in sys.executable:
|
||||
sp.run(["termux-notification", "-t", title, "-c", msg])
|
||||
return
|
||||
|
||||
notification.notify(
|
||||
title=title,
|
||||
message=msg,
|
||||
app_icon=icon,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
_ = r"""
|
||||
reject file uploads by file extension
|
||||
|
||||
example usage as global config:
|
||||
--xbu c,bin/hooks/reject-extension.py
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xbu=c,bin/hooks/reject-extension.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all uploads with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xbu = execute before upload
|
||||
c = check result, reject upload if error
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
bad = "exe scr com pif bat ps1 jar msi"
|
||||
|
||||
ext = sys.argv[1].split(".")[-1]
|
||||
|
||||
sys.exit(1 if ext in bad.split() else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import magic
|
||||
|
||||
|
||||
_ = r"""
|
||||
reject file uploads by mimetype
|
||||
|
||||
dependencies (linux, macos):
|
||||
python3 -m pip install --user -U python-magic
|
||||
|
||||
dependencies (windows):
|
||||
python3 -m pip install --user -U python-magic-bin
|
||||
|
||||
example usage as global config:
|
||||
--xau c,bin/hooks/reject-mimetype.py
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xau=c,bin/hooks/reject-mimetype.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all uploads with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xau = execute after upload
|
||||
c = check result, reject upload if error
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
ok = ["image/jpeg", "image/png"]
|
||||
|
||||
mt = magic.from_file(sys.argv[1], mime=True)
|
||||
|
||||
print(mt)
|
||||
|
||||
sys.exit(1 if mt not in ok else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,60 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import subprocess as sp
|
||||
|
||||
|
||||
_ = r"""
|
||||
use copyparty as a file downloader by POSTing URLs as
|
||||
application/x-www-form-urlencoded (for example using the
|
||||
message/pager function on the website)
|
||||
|
||||
example usage as global config:
|
||||
--xm f,j,t3600,bin/hooks/wget.py
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xm=f,j,t3600,bin/hooks/wget.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all messages with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xm = execute on message-to-server-log
|
||||
f = fork so it doesn't block uploads
|
||||
j = provide message information as json; not just the text
|
||||
c3 = mute all output
|
||||
t3600 = timeout and kill download after 1 hour
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
inf = json.loads(sys.argv[1])
|
||||
url = inf["txt"]
|
||||
if "://" not in url:
|
||||
url = "https://" + url
|
||||
|
||||
os.chdir(inf["ap"])
|
||||
|
||||
name = url.split("?")[0].split("/")[-1]
|
||||
tfn = "-- DOWNLOADING " + name
|
||||
print(f"{tfn}\n", end="")
|
||||
open(tfn, "wb").close()
|
||||
|
||||
cmd = ["wget", "--trust-server-names", "-nv", "--", url]
|
||||
|
||||
try:
|
||||
sp.check_call(cmd)
|
||||
except:
|
||||
t = "-- FAILED TO DONWLOAD " + name
|
||||
print(f"{t}\n", end="")
|
||||
open(t, "wb").close()
|
||||
|
||||
os.unlink(tfn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,108 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
_ = r"""
|
||||
this hook will produce a single sha512 file which
|
||||
covers all recent uploads (plus metadata comments)
|
||||
|
||||
use this with --xiu, which makes copyparty buffer
|
||||
uploads until server is idle, providing file infos
|
||||
on stdin (filepaths or json)
|
||||
|
||||
example usage as global config:
|
||||
--xiu i5,j,bin/hooks/xiu-sha.py
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xiu=i5,j,bin/hooks/xiu-sha.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on batches of uploads with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xiu = execute after uploads...
|
||||
i5 = ...after volume has been idle for 5sec
|
||||
j = provide json instead of filepath list
|
||||
|
||||
note the "f" (fork) flag is not set, so this xiu
|
||||
will block other xiu hooks while it's running
|
||||
"""
|
||||
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p
|
||||
|
||||
|
||||
def humantime(ts):
|
||||
return datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def find_files_root(inf):
|
||||
di = 9000
|
||||
for f1, f2 in zip(inf, inf[1:]):
|
||||
p1 = f1["ap"].replace("\\", "/").rsplit("/", 1)[0]
|
||||
p2 = f2["ap"].replace("\\", "/").rsplit("/", 1)[0]
|
||||
di = min(len(p1), len(p2), di)
|
||||
di = next((i for i in range(di) if p1[i] != p2[i]), di)
|
||||
|
||||
return di + 1
|
||||
|
||||
|
||||
def find_vol_root(inf):
|
||||
return len(inf[0]["ap"][: -len(inf[0]["vp"])])
|
||||
|
||||
|
||||
def main():
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
inf = json.loads(zs)
|
||||
|
||||
# root directory (where to put the sha512 file);
|
||||
# di = find_files_root(inf) # next to the file closest to volume root
|
||||
di = find_vol_root(inf) # top of the entire volume
|
||||
|
||||
ret = []
|
||||
total_sz = 0
|
||||
for md in inf:
|
||||
ap = md["ap"]
|
||||
rp = ap[di:]
|
||||
total_sz += md["sz"]
|
||||
fsize = "{:,}".format(md["sz"])
|
||||
mtime = humantime(md["mt"])
|
||||
up_ts = humantime(md["at"])
|
||||
|
||||
h = hashlib.sha512()
|
||||
with open(fsenc(md["ap"]), "rb", 512 * 1024) as f:
|
||||
while True:
|
||||
buf = f.read(512 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
|
||||
h.update(buf)
|
||||
|
||||
cksum = h.hexdigest()
|
||||
meta = " | ".join([md["wark"], up_ts, mtime, fsize, md["ip"]])
|
||||
ret.append("# {}\n{} *{}".format(meta, cksum, rp))
|
||||
|
||||
ret.append("# {} files, {} bytes total".format(len(inf), total_sz))
|
||||
ret.append("")
|
||||
ftime = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")
|
||||
fp = "{}xfer-{}.sha512".format(inf[0]["ap"][:di], ftime)
|
||||
with open(fsenc(fp), "wb") as f:
|
||||
f.write("\n".join(ret).encode("utf-8", "replace"))
|
||||
|
||||
print("wrote checksums to {}".format(fp))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
_ = r"""
|
||||
this hook prints absolute filepaths + total size
|
||||
|
||||
use this with --xiu, which makes copyparty buffer
|
||||
uploads until server is idle, providing file infos
|
||||
on stdin (filepaths or json)
|
||||
|
||||
example usage as global config:
|
||||
--xiu i1,j,bin/hooks/xiu.py
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xiu=i1,j,bin/hooks/xiu.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on batches of uploads with the params listed below)
|
||||
|
||||
parameters explained,
|
||||
xiu = execute after uploads...
|
||||
i1 = ...after volume has been idle for 1sec
|
||||
j = provide json instead of filepath list
|
||||
|
||||
note the "f" (fork) flag is not set, so this xiu
|
||||
will block other xiu hooks while it's running
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
inf = json.loads(zs)
|
||||
|
||||
total_sz = 0
|
||||
for upload in inf:
|
||||
sz = upload["sz"]
|
||||
total_sz += sz
|
||||
print("{:9} {}".format(sz, upload["ap"]))
|
||||
|
||||
print("{} files, {} bytes total".format(len(inf), total_sz))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,28 +1,10 @@
|
||||
standalone programs which take an audio file as argument
|
||||
|
||||
you may want to forget about all this fancy complicated stuff and just use [event hooks](../hooks/) instead (which doesn't need `-e2ts` or ffmpeg)
|
||||
|
||||
----
|
||||
|
||||
**NOTE:** these all require `-e2ts` to be functional, meaning you need to do at least one of these: `apt install ffmpeg` or `pip3 install mutagen`
|
||||
|
||||
some of these rely on libraries which are not MIT-compatible
|
||||
|
||||
* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
|
||||
* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
|
||||
|
||||
these invoke standalone programs which are GPL or similar, so is legally fine for most purposes:
|
||||
|
||||
* [media-hash.py](./media-hash.py) generates checksums for audio and video streams; uses FFmpeg (LGPL or GPL)
|
||||
* [image-noexif.py](./image-noexif.py) removes exif tags from images; uses exiftool (GPLv1 or artistic-license)
|
||||
|
||||
these do not have any problematic dependencies at all:
|
||||
|
||||
* [cksum.py](./cksum.py) computes various checksums
|
||||
* [exe.py](./exe.py) grabs metadata from .exe and .dll files (example for retrieving multiple tags with one parser)
|
||||
* [wget.py](./wget.py) lets you download files by POSTing URLs to copyparty
|
||||
* also available as an [event hook](../hooks/wget.py)
|
||||
|
||||
|
||||
# dependencies
|
||||
|
||||
@@ -31,15 +13,12 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
|
||||
*alternatively* (or preferably) use packages from your distro instead, then you'll need at least these:
|
||||
|
||||
* from distro: `numpy vamp-plugin-sdk beatroot-vamp mixxx-keyfinder ffmpeg`
|
||||
* from pip: `keyfinder vamp`
|
||||
* from pypy: `keyfinder vamp`
|
||||
|
||||
|
||||
# usage from copyparty
|
||||
|
||||
`copyparty -e2dsa -e2ts` followed by any combination of these:
|
||||
* `-mtp key=f,audio-key.py`
|
||||
* `-mtp .bpm=f,audio-bpm.py`
|
||||
* `-mtp ahash,vhash=f,media-hash.py`
|
||||
`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
|
||||
|
||||
* `f,` makes the detected value replace any existing values
|
||||
* the `.` in `.bpm` indicates numeric value
|
||||
@@ -47,12 +26,9 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
|
||||
* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
|
||||
|
||||
|
||||
## usage with volflags
|
||||
## usage with volume-flags
|
||||
|
||||
instead of affecting all volumes, you can set the options for just one volume like so:
|
||||
|
||||
`copyparty -v /mnt/nas/music:/music:r:c,e2dsa:c,e2ts` immediately followed by any combination of these:
|
||||
|
||||
* `:c,mtp=key=f,audio-key.py`
|
||||
* `:c,mtp=.bpm=f,audio-bpm.py`
|
||||
* `:c,mtp=ahash,vhash=f,media-hash.py`
|
||||
```
|
||||
copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
|
||||
```
|
||||
|
||||
@@ -16,24 +16,20 @@ dep: ffmpeg
|
||||
"""
|
||||
|
||||
|
||||
# save beat timestamps to ".beats/filename.txt"
|
||||
SAVE = False
|
||||
|
||||
|
||||
def det(tf):
|
||||
# fmt: off
|
||||
sp.check_call([
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-hide_banner",
|
||||
b"-v", b"fatal",
|
||||
b"-y", b"-i", fsenc(sys.argv[1]),
|
||||
b"-map", b"0:a:0",
|
||||
b"-ac", b"1",
|
||||
b"-ar", b"22050",
|
||||
b"-t", b"360",
|
||||
b"-f", b"f32le",
|
||||
fsenc(tf)
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-v", "fatal",
|
||||
"-ss", "13",
|
||||
"-y", "-i", fsenc(sys.argv[1]),
|
||||
"-ac", "1",
|
||||
"-ar", "22050",
|
||||
"-t", "300",
|
||||
"-f", "f32le",
|
||||
tf
|
||||
])
|
||||
# fmt: on
|
||||
|
||||
@@ -50,29 +46,10 @@ def det(tf):
|
||||
print(c["list"][0]["label"].split(" ")[0])
|
||||
return
|
||||
|
||||
# throws if detection failed:
|
||||
beats = [float(x["timestamp"]) for x in cl]
|
||||
bds = [b - a for a, b in zip(beats, beats[1:])]
|
||||
bds.sort()
|
||||
n0 = int(len(bds) * 0.2)
|
||||
n1 = int(len(bds) * 0.75) + 1
|
||||
bds = bds[n0:n1]
|
||||
bpm = sum(bds)
|
||||
bpm = round(60 * (len(bds) / bpm), 2)
|
||||
print(f"{bpm:.2f}")
|
||||
|
||||
if SAVE:
|
||||
fdir, fname = os.path.split(sys.argv[1])
|
||||
bdir = os.path.join(fdir, ".beats")
|
||||
try:
|
||||
os.mkdir(fsenc(bdir))
|
||||
except:
|
||||
pass
|
||||
|
||||
fp = os.path.join(bdir, fname) + ".txt"
|
||||
with open(fsenc(fp), "wb") as f:
|
||||
txt = "\n".join([f"{x:.2f}" for x in beats])
|
||||
f.write(txt.encode("utf-8"))
|
||||
# throws if detection failed:
|
||||
bpm = float(cl[-1]["timestamp"] - cl[1]["timestamp"])
|
||||
bpm = round(60 * ((len(cl) - 1) / bpm), 2)
|
||||
print(f"{bpm:.2f}")
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -23,15 +23,14 @@ dep: ffmpeg
|
||||
def det(tf):
|
||||
# fmt: off
|
||||
sp.check_call([
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-hide_banner",
|
||||
b"-v", b"fatal",
|
||||
b"-y", b"-i", fsenc(sys.argv[1]),
|
||||
b"-map", b"0:a:0",
|
||||
b"-t", b"300",
|
||||
b"-sample_fmt", b"s16",
|
||||
fsenc(tf)
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-v", "fatal",
|
||||
"-y", "-i", fsenc(sys.argv[1]),
|
||||
"-t", "300",
|
||||
"-sample_fmt", "s16",
|
||||
tf
|
||||
])
|
||||
# fmt: on
|
||||
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import json
|
||||
import zlib
|
||||
import struct
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p
|
||||
|
||||
|
||||
"""
|
||||
calculates various checksums for uploads,
|
||||
usage: -mtp crc32,md5,sha1,sha256b=ad,bin/mtag/cksum.py
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
config = "crc32 md5 md5b sha1 sha1b sha256 sha256b sha512/240 sha512b/240"
|
||||
# b suffix = base64 encoded
|
||||
# slash = truncate to n bits
|
||||
|
||||
known = {
|
||||
"md5": hashlib.md5,
|
||||
"sha1": hashlib.sha1,
|
||||
"sha256": hashlib.sha256,
|
||||
"sha512": hashlib.sha512,
|
||||
}
|
||||
config = config.split()
|
||||
hashers = {
|
||||
k: v()
|
||||
for k, v in known.items()
|
||||
if k in [x.split("/")[0].rstrip("b") for x in known]
|
||||
}
|
||||
crc32 = 0 if "crc32" in config else None
|
||||
|
||||
with open(fsenc(sys.argv[1]), "rb", 512 * 1024) as f:
|
||||
while True:
|
||||
buf = f.read(64 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
|
||||
for x in hashers.values():
|
||||
x.update(buf)
|
||||
|
||||
if crc32 is not None:
|
||||
crc32 = zlib.crc32(buf, crc32)
|
||||
|
||||
ret = {}
|
||||
for s in config:
|
||||
alg = s.split("/")[0]
|
||||
b64 = alg.endswith("b")
|
||||
alg = alg.rstrip("b")
|
||||
if alg in hashers:
|
||||
v = hashers[alg].digest()
|
||||
elif alg == "crc32":
|
||||
v = crc32
|
||||
if v < 0:
|
||||
v &= 2 ** 32 - 1
|
||||
v = struct.pack(">L", v)
|
||||
else:
|
||||
raise Exception("what is {}".format(s))
|
||||
|
||||
if "/" in s:
|
||||
v = v[: int(int(s.split("/")[1]) / 8)]
|
||||
|
||||
if b64:
|
||||
v = base64.b64encode(v).decode("ascii").rstrip("=")
|
||||
else:
|
||||
try:
|
||||
v = v.hex()
|
||||
except:
|
||||
import binascii
|
||||
|
||||
v = binascii.hexlify(v)
|
||||
|
||||
ret[s] = v
|
||||
|
||||
print(json.dumps(ret, indent=4))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
fetch latest msg from guestbook and return as tag
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=guestbook=t10,ad,p,bin/mtag/guestbook-read.py:mte=+guestbook
|
||||
|
||||
explained:
|
||||
for realpath srv/hello (served at /hello), write-only for eveyrone,
|
||||
enable file analysis on upload (e2ts),
|
||||
use mtp plugin "bin/mtag/guestbook-read.py" to provide metadata tag "guestbook",
|
||||
do this on all uploads regardless of extension,
|
||||
t10 = 10 seconds timeout for each dwonload,
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
p = request upload info as json on stdin (need ip)
|
||||
mte=+guestbook enabled indexing of that tag for this volume
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
|
||||
# set 0 to allow infinite msgs from one IP,
|
||||
# other values delete older messages to make space,
|
||||
# so 1 only keeps latest msg
|
||||
NUM_MSGS_TO_KEEP = 1
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
fdir = os.path.dirname(fp)
|
||||
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
ip = md["up_ip"]
|
||||
|
||||
# can put the database inside `fdir` if you'd like,
|
||||
# by default it saves to PWD:
|
||||
# os.chdir(fdir)
|
||||
|
||||
db = sqlite3.connect("guestbook.db3")
|
||||
with db:
|
||||
t = "select msg from gb where ip = ? order by ts desc"
|
||||
r = db.execute(t, (ip,)).fetchone()
|
||||
if r:
|
||||
print(r[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,111 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
store messages from users in an sqlite database
|
||||
which can be read from another mtp for example
|
||||
|
||||
takes input from application/x-www-form-urlencoded POSTs,
|
||||
for example using the message/pager function on the website
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=xgb=ebin,t10,ad,p,bin/mtag/guestbook.py:mte=+xgb
|
||||
|
||||
explained:
|
||||
for realpath srv/hello (served at /hello),write-only for eveyrone,
|
||||
enable file analysis on upload (e2ts),
|
||||
use mtp plugin "bin/mtag/guestbook.py" to provide metadata tag "xgb",
|
||||
do this on all uploads with the file extension "bin",
|
||||
t300 = 300 seconds timeout for each dwonload,
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
p = request upload info as json on stdin
|
||||
mte=+xgb enabled indexing of that tag for this volume
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
# set 0 to allow infinite msgs from one IP,
|
||||
# other values delete older messages to make space,
|
||||
# so 1 only keeps latest msg
|
||||
NUM_MSGS_TO_KEEP = 1
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
fdir = os.path.dirname(fp)
|
||||
fname = os.path.basename(fp)
|
||||
if not fname.startswith("put-") or not fname.endswith(".bin"):
|
||||
raise Exception("not a post file")
|
||||
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
buf = b""
|
||||
with open(fp, "rb") as f:
|
||||
while True:
|
||||
b = f.read(4096)
|
||||
buf += b
|
||||
if len(buf) > 4096:
|
||||
raise Exception("too big")
|
||||
|
||||
if not b:
|
||||
break
|
||||
|
||||
if not buf:
|
||||
raise Exception("file is empty")
|
||||
|
||||
buf = unquote(buf.replace(b"+", b" "))
|
||||
txt = buf.decode("utf-8")
|
||||
|
||||
if not txt.startswith("msg="):
|
||||
raise Exception("does not start with msg=")
|
||||
|
||||
ip = md["up_ip"]
|
||||
ts = md["up_at"]
|
||||
txt = txt[4:]
|
||||
|
||||
# can put the database inside `fdir` if you'd like,
|
||||
# by default it saves to PWD:
|
||||
# os.chdir(fdir)
|
||||
|
||||
db = sqlite3.connect("guestbook.db3")
|
||||
try:
|
||||
db.execute("select 1 from gb").fetchone()
|
||||
except:
|
||||
with db:
|
||||
db.execute("create table gb (ip text, ts real, msg text)")
|
||||
db.execute("create index gb_ip on gb(ip)")
|
||||
|
||||
with db:
|
||||
if NUM_MSGS_TO_KEEP == 1:
|
||||
t = "delete from gb where ip = ?"
|
||||
db.execute(t, (ip,))
|
||||
|
||||
t = "insert into gb values (?,?,?)"
|
||||
db.execute(t, (ip, ts, txt))
|
||||
|
||||
if NUM_MSGS_TO_KEEP > 1:
|
||||
t = "select ts from gb where ip = ? order by ts desc"
|
||||
hits = db.execute(t, (ip,)).fetchall()
|
||||
|
||||
if len(hits) > NUM_MSGS_TO_KEEP:
|
||||
lim = hits[NUM_MSGS_TO_KEEP][0]
|
||||
t = "delete from gb where ip = ? and ts <= ?"
|
||||
db.execute(t, (ip, lim))
|
||||
|
||||
print(txt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
remove exif tags from uploaded images
|
||||
|
||||
dependencies:
|
||||
exiftool
|
||||
|
||||
about:
|
||||
creates a "noexif" subfolder and puts exif-stripped copies of each image there,
|
||||
the reason for the subfolder is to avoid issues with the up2k.db / deduplication:
|
||||
|
||||
if the original image is modified in-place, then copyparty will keep the original
|
||||
hash in up2k.db for a while (until the next volume rescan), so if the image is
|
||||
reuploaded after a rescan then the upload will be renamed and kept as a dupe
|
||||
|
||||
alternatively you could switch the logic around, making a copy of the original
|
||||
image into a subfolder named "exif" and modify the original in-place, but then
|
||||
up2k.db will be out of sync until the next rescan, so any additional uploads
|
||||
of the same image will get symlinked (deduplicated) to the modified copy
|
||||
instead of the original in "exif"
|
||||
|
||||
or maybe delete the original image after processing, that would kinda work too
|
||||
|
||||
example copyparty config to use this:
|
||||
-v/mnt/nas/pics:pics:rwmd,ed:c,e2ts,mte=+noexif:c,mtp=noexif=ejpg,ejpeg,ad,bin/mtag/image-noexif.py
|
||||
|
||||
explained:
|
||||
for realpath /mnt/nas/pics (served at /pics) with read-write-modify-delete for ed,
|
||||
enable file analysis on upload (e2ts),
|
||||
append "noexif" to the list of known tags (mtp),
|
||||
and use mtp plugin "bin/mtag/image-noexif.py" to provide that tag,
|
||||
do this on all uploads with the file extension "jpg" or "jpeg",
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
and your python must have sqlite3 support compiled in
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import filecmp
|
||||
import subprocess as sp
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
cwd, fn = os.path.split(sys.argv[1])
|
||||
if os.path.basename(cwd) == "noexif":
|
||||
return
|
||||
|
||||
os.chdir(cwd)
|
||||
f1 = fsenc(fn)
|
||||
f2 = fsenc(os.path.join(b"noexif", fn))
|
||||
cmd = [
|
||||
b"exiftool",
|
||||
b"-exif:all=",
|
||||
b"-iptc:all=",
|
||||
b"-xmp:all=",
|
||||
b"-P",
|
||||
b"-o",
|
||||
b"noexif/",
|
||||
b"--",
|
||||
f1,
|
||||
]
|
||||
sp.check_output(cmd)
|
||||
if not os.path.exists(f2):
|
||||
print("failed")
|
||||
return
|
||||
|
||||
if filecmp.cmp(f1, f2, shallow=False):
|
||||
print("clean")
|
||||
else:
|
||||
print("exif")
|
||||
|
||||
# lastmod = os.path.getmtime(f1)
|
||||
# times = (int(time.time()), int(lastmod))
|
||||
# os.utime(f2, times)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except:
|
||||
pass
|
||||
@@ -4,9 +4,7 @@ set -e
|
||||
|
||||
# install dependencies for audio-*.py
|
||||
#
|
||||
# linux/alpine: requires gcc g++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-dev py3-{wheel,pip} py3-numpy{,-dev}
|
||||
# linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3,libsndfile1}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
|
||||
# linux/fedora: requires gcc gcc-c++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-devel python3-numpy vamp-plugin-sdk qm-vamp-plugins
|
||||
# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
|
||||
# win64: requires msys2-mingw64 environment
|
||||
# macos: requires macports
|
||||
#
|
||||
@@ -57,7 +55,6 @@ hash -r
|
||||
command -v python3 && pybin=python3 || pybin=python
|
||||
}
|
||||
|
||||
$pybin -c 'import numpy' ||
|
||||
$pybin -m pip install --user numpy
|
||||
|
||||
|
||||
@@ -103,11 +100,8 @@ export -f dl_files
|
||||
|
||||
|
||||
github_tarball() {
|
||||
rm -rf g
|
||||
mkdir g
|
||||
cd g
|
||||
dl_text "$1" |
|
||||
tee ../json |
|
||||
tee json |
|
||||
(
|
||||
# prefer jq if available
|
||||
jq -r '.tarball_url' ||
|
||||
@@ -116,11 +110,8 @@ github_tarball() {
|
||||
awk -F\" '/"tarball_url": "/ {print$4}'
|
||||
) |
|
||||
tee /dev/stderr |
|
||||
head -n 1 |
|
||||
tr -d '\r' | tr '\n' '\0' |
|
||||
xargs -0 bash -c 'dl_files "$@"' _
|
||||
mv * ../tgz
|
||||
cd ..
|
||||
}
|
||||
|
||||
|
||||
@@ -135,7 +126,6 @@ gitlab_tarball() {
|
||||
tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
|
||||
) |
|
||||
tee /dev/stderr |
|
||||
head -n 1 |
|
||||
tr -d '\r' | tr '\n' '\0' |
|
||||
tee links |
|
||||
xargs -0 bash -c 'dl_files "$@"' _
|
||||
@@ -147,27 +137,20 @@ install_keyfinder() {
|
||||
# use msys2 in mingw-w64 mode
|
||||
# pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
|
||||
|
||||
[ -e $HOME/pe/keyfinder ] && {
|
||||
echo found a keyfinder build in ~/pe, skipping
|
||||
return
|
||||
}
|
||||
|
||||
cd "$td"
|
||||
github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
|
||||
ls -al
|
||||
|
||||
tar -xf tgz
|
||||
rm tgz
|
||||
tar -xf mixxxdj-libkeyfinder-*
|
||||
rm -- *.tar.gz
|
||||
cd mixxxdj-libkeyfinder*
|
||||
|
||||
h="$HOME"
|
||||
so="lib/libkeyfinder.so"
|
||||
memes=(-DBUILD_TESTING=OFF)
|
||||
memes=()
|
||||
|
||||
[ $win ] &&
|
||||
so="bin/libkeyfinder.dll" &&
|
||||
h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
|
||||
memes+=(-G "MinGW Makefiles")
|
||||
memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
|
||||
|
||||
[ $mac ] &&
|
||||
so="lib/libkeyfinder.dylib"
|
||||
@@ -187,7 +170,7 @@ install_keyfinder() {
|
||||
}
|
||||
|
||||
# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
|
||||
CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
|
||||
CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
|
||||
LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
|
||||
PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
|
||||
$pybin -m pip install --user keyfinder
|
||||
@@ -224,22 +207,6 @@ install_vamp() {
|
||||
|
||||
$pybin -m pip install --user vamp
|
||||
|
||||
cd "$td"
|
||||
echo '#include <vamp-sdk/Plugin.h>' | g++ -x c++ -c -o /dev/null - || [ -e ~/pe/vamp-sdk ] || {
|
||||
printf '\033[33mcould not find the vamp-sdk, building from source\033[0m\n'
|
||||
(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/2588/vamp-plugin-sdk-2.9.0.tar.gz)
|
||||
sha512sum -c <(
|
||||
echo "7ef7f837d19a08048b059e0da408373a7964ced452b290fae40b85d6d70ca9000bcfb3302cd0b4dc76cf2a848528456f78c1ce1ee0c402228d812bd347b6983b -"
|
||||
) <vamp-plugin-sdk-2.9.0.tar.gz
|
||||
tar -xf vamp-plugin-sdk-2.9.0.tar.gz
|
||||
rm -- *.tar.gz
|
||||
ls -al
|
||||
cd vamp-plugin-sdk-*
|
||||
./configure --prefix=$HOME/pe/vamp-sdk
|
||||
make -j1 install
|
||||
}
|
||||
|
||||
cd "$td"
|
||||
have_beatroot || {
|
||||
printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
|
||||
(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
|
||||
@@ -247,11 +214,8 @@ install_vamp() {
|
||||
echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874 -"
|
||||
) <beatroot-vamp-v1.0.tar.gz
|
||||
tar -xf beatroot-vamp-v1.0.tar.gz
|
||||
rm -- *.tar.gz
|
||||
cd beatroot-vamp-v1.0
|
||||
[ -e ~/pe/vamp-sdk ] &&
|
||||
sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux
|
||||
make -f Makefile.linux -j4 LDFLAGS=-L$HOME/pe/vamp-sdk/lib
|
||||
make -f Makefile.linux -j4
|
||||
# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
|
||||
mkdir ~/vamp
|
||||
cp -pv beatroot-vamp.* ~/vamp/
|
||||
@@ -265,7 +229,6 @@ install_vamp() {
|
||||
|
||||
# not in use because it kinda segfaults, also no windows support
|
||||
install_soundtouch() {
|
||||
cd "$td"
|
||||
gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
|
||||
|
||||
tar -xvf soundtouch-*
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import base64
|
||||
import hashlib
|
||||
import subprocess as sp
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
"""
|
||||
dep: ffmpeg
|
||||
"""
|
||||
|
||||
|
||||
def det():
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-hide_banner",
|
||||
b"-v", b"fatal",
|
||||
b"-i", fsenc(sys.argv[1]),
|
||||
b"-f", b"framemd5",
|
||||
b"-"
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE)
|
||||
# ps = io.TextIOWrapper(p.stdout, encoding="utf-8")
|
||||
ps = p.stdout
|
||||
|
||||
chans = {}
|
||||
for ln in ps:
|
||||
if ln.startswith(b"#stream#"):
|
||||
break
|
||||
|
||||
m = re.match(r"^#media_type ([0-9]): ([a-zA-Z])", ln.decode("utf-8"))
|
||||
if m:
|
||||
chans[m.group(1)] = m.group(2)
|
||||
|
||||
hashers = [hashlib.sha512(), hashlib.sha512()]
|
||||
for ln in ps:
|
||||
n = int(ln[:1])
|
||||
v = ln.rsplit(b",", 1)[-1].strip()
|
||||
hashers[n].update(v)
|
||||
|
||||
r = {}
|
||||
for k, v in chans.items():
|
||||
dg = hashers[int(k)].digest()[:12]
|
||||
dg = base64.urlsafe_b64encode(dg).decode("ascii")
|
||||
r[v[0].lower() + "hash"] = dg
|
||||
|
||||
print(json.dumps(r, indent=4))
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
det()
|
||||
except:
|
||||
pass # mute
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
|
||||
|
||||
"""
|
||||
mtp test -- opens a texteditor
|
||||
|
||||
usage:
|
||||
-vsrv/v1:v1:r:c,mte=+x1:c,mtp=x1=ad,p,bin/mtag/mousepad.py
|
||||
|
||||
explained:
|
||||
c,mte: list of tags to index in this volume
|
||||
c,mtp: add new tag provider
|
||||
x1: dummy tag to provide
|
||||
ad: dontcare if audio or not
|
||||
p: priority 1 (run after initial tag-scan with ffprobe or mutagen)
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
env = os.environ.copy()
|
||||
env["DISPLAY"] = ":0.0"
|
||||
|
||||
if False:
|
||||
# open the uploaded file
|
||||
fp = sys.argv[-1]
|
||||
else:
|
||||
# display stdin contents (`oth_tags`)
|
||||
fp = "/dev/stdin"
|
||||
|
||||
p = sp.Popen(["/usr/bin/mousepad", fp])
|
||||
p.communicate()
|
||||
|
||||
|
||||
main()
|
||||
@@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess as sp
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
_ = r"""
|
||||
first checks the tag "vidchk" which must be "ok" to continue,
|
||||
then uploads all files to some cloud storage (RCLONE_REMOTE)
|
||||
and DELETES THE ORIGINAL FILES if rclone returns 0 ("success")
|
||||
|
||||
deps:
|
||||
rclone
|
||||
|
||||
usage:
|
||||
-mtp x2=t43200,ay,p2,bin/mtag/rclone-upload.py
|
||||
|
||||
explained:
|
||||
t43200: timeout 12h
|
||||
ay: only process files which contain audio (including video with audio)
|
||||
p2: set priority 2 (after vidchk's suggested priority of 1),
|
||||
so the output of vidchk will be passed in here
|
||||
|
||||
complete usage example as vflags along with vidchk:
|
||||
-vsrv/vidchk:vidchk:r:rw,ed:c,e2dsa,e2ts,mtp=vidchk=t600,p,bin/mtag/vidchk.py:c,mtp=rupload=t43200,ay,p2,bin/mtag/rclone-upload.py:c,mte=+vidchk,rupload
|
||||
|
||||
setup: see https://rclone.org/drive/
|
||||
|
||||
if you wanna use this script standalone / separately from copyparty,
|
||||
either set CONDITIONAL_UPLOAD False or provide the following stdin:
|
||||
{"vidchk":"ok"}
|
||||
"""
|
||||
|
||||
|
||||
RCLONE_REMOTE = "notmybox"
|
||||
CONDITIONAL_UPLOAD = True
|
||||
|
||||
|
||||
def main():
|
||||
fp = sys.argv[1]
|
||||
if CONDITIONAL_UPLOAD:
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
chk = md.get("vidchk", None)
|
||||
if chk != "ok":
|
||||
print(f"vidchk={chk}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
dst = f"{RCLONE_REMOTE}:".encode("utf-8")
|
||||
cmd = [b"rclone", b"copy", b"--", fsenc(fp), dst]
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
sp.check_call(cmd)
|
||||
except:
|
||||
print("rclone failed", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"{time.time() - t0:.1f} sec")
|
||||
os.unlink(fsenc(fp))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,21 +0,0 @@
|
||||
// ==UserScript==
|
||||
// @name twitter-unmute
|
||||
// @namespace http://ocv.me/
|
||||
// @version 0.1
|
||||
// @description memes
|
||||
// @author ed <irc.rizon.net>
|
||||
// @match https://twitter.com/*
|
||||
// @icon https://www.google.com/s2/favicons?domain=twitter.com
|
||||
// @grant GM_addStyle
|
||||
// ==/UserScript==
|
||||
|
||||
function grunnur() {
|
||||
setInterval(function () {
|
||||
//document.querySelector('div[aria-label="Unmute"]').click();
|
||||
document.querySelector('video').muted = false;
|
||||
}, 200);
|
||||
}
|
||||
|
||||
var scr = document.createElement('script');
|
||||
scr.textContent = '(' + grunnur.toString() + ')();';
|
||||
(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
|
||||
@@ -1,39 +0,0 @@
|
||||
# example config file to use copyparty as a youtube manifest collector,
|
||||
# use with copyparty like: python copyparty.py -c yt-ipr.conf
|
||||
#
|
||||
# see docs/example.conf for a better explanation of the syntax, but
|
||||
# newlines are block separators, so adding blank lines inside a volume definition is bad
|
||||
# (use comments as separators instead)
|
||||
|
||||
|
||||
# create user ed, password wark
|
||||
u ed:wark
|
||||
|
||||
|
||||
# create a volume at /ytm which stores files at ./srv/ytm
|
||||
./srv/ytm
|
||||
/ytm
|
||||
# write-only, but read-write for user ed
|
||||
w
|
||||
rw ed
|
||||
# rescan the volume on startup
|
||||
c e2dsa
|
||||
# collect tags from all new files since last scan
|
||||
c e2ts
|
||||
# optionally enable compression to make the files 50% smaller
|
||||
c pk
|
||||
# only allow uploads which are between 16k and 1m large
|
||||
c sz=16k-1m
|
||||
# allow up to 10 uploads over 5 minutes from each ip
|
||||
c maxn=10,300
|
||||
# move uploads into subfolders: YEAR-MONTH / DAY-HOUR / <upload>
|
||||
c rotf=%Y-%m/%d-%H
|
||||
# delete uploads when they are 24 hours old
|
||||
c lifetime=86400
|
||||
# add the parser and tell copyparty what tags it can expect from it
|
||||
c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
|
||||
# decide which tags we want to index and in what order
|
||||
c mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
|
||||
|
||||
|
||||
# create any other volumes you'd like down here, or merge this with an existing config file
|
||||
@@ -1,47 +0,0 @@
|
||||
// ==UserScript==
|
||||
// @name youtube-playerdata-hub
|
||||
// @match https://youtube.com/*
|
||||
// @match https://*.youtube.com/*
|
||||
// @version 1.0
|
||||
// @grant GM_addStyle
|
||||
// ==/UserScript==
|
||||
|
||||
function main() {
|
||||
var server = 'https://127.0.0.1:3923/ytm?pw=wark',
|
||||
interval = 60; // sec
|
||||
|
||||
var sent = {};
|
||||
function send(txt, mf_url, desc) {
|
||||
if (sent[mf_url])
|
||||
return;
|
||||
|
||||
fetch(server + '&_=' + Date.now(), { method: "PUT", body: txt });
|
||||
console.log('[yt-pdh] yeet %d bytes, %s', txt.length, desc);
|
||||
sent[mf_url] = 1;
|
||||
}
|
||||
|
||||
function collect() {
|
||||
try {
|
||||
var pd = document.querySelector('ytd-watch-flexy');
|
||||
if (!pd)
|
||||
return console.log('[yt-pdh] no video found');
|
||||
|
||||
pd = pd.playerData;
|
||||
var mu = pd.streamingData.dashManifestUrl || pd.streamingData.hlsManifestUrl;
|
||||
if (!mu || !mu.length)
|
||||
return console.log('[yt-pdh] no manifest found');
|
||||
|
||||
var desc = pd.videoDetails.videoId + ', ' + pd.videoDetails.title;
|
||||
send(JSON.stringify(pd), mu, desc);
|
||||
}
|
||||
catch (ex) {
|
||||
console.log("[yt-pdh]", ex);
|
||||
}
|
||||
}
|
||||
setInterval(collect, interval * 1000);
|
||||
}
|
||||
|
||||
var scr = document.createElement('script');
|
||||
scr.textContent = '(' + main.toString() + ')();';
|
||||
(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
|
||||
console.log('[yt-pdh] a');
|
||||
@@ -1,139 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
use copyparty as a chromecast replacement:
|
||||
* post a URL and it will open in the default browser
|
||||
* upload a file and it will open in the default application
|
||||
* the `key` command simulates keyboard input
|
||||
* the `x` command executes other xdotool commands
|
||||
* the `c` command executes arbitrary unix commands
|
||||
|
||||
the android app makes it a breeze to post pics and links:
|
||||
https://github.com/9001/party-up/releases
|
||||
(iOS devices have to rely on the web-UI)
|
||||
|
||||
goes without saying, but this is HELLA DANGEROUS,
|
||||
GIVES RCE TO ANYONE WHO HAVE UPLOAD PERMISSIONS
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -v.::w:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,c0,bin/mtag/very-bad-idea.py
|
||||
|
||||
recommended deps:
|
||||
apt install xdotool libnotify-bin
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/meadup.js
|
||||
|
||||
and you probably want `twitter-unmute.user.js` from the res folder
|
||||
|
||||
|
||||
-----------------------------------------------------------------------
|
||||
-- startup script:
|
||||
-----------------------------------------------------------------------
|
||||
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# create qr code
|
||||
ip=$(ip r | awk '/^default/{print$(NF-2)}'); echo http://$ip:3923/ | qrencode -o - -s 4 >/dev/shm/cpp-qr.png
|
||||
/usr/bin/feh -x /dev/shm/cpp-qr.png &
|
||||
|
||||
# reposition and make topmost (with janky raspbian support)
|
||||
( sleep 0.5
|
||||
xdotool search --name cpp-qr.png windowactivate --sync windowmove 1780 0
|
||||
wmctrl -r :ACTIVE: -b toggle,above || true
|
||||
|
||||
ps aux | grep -E 'sleep[ ]7\.27' ||
|
||||
while true; do
|
||||
w=$(xdotool getactivewindow)
|
||||
xdotool search --name cpp-qr.png windowactivate windowraise windowfocus
|
||||
xdotool windowactivate $w
|
||||
xdotool windowfocus $w
|
||||
sleep 7.27 || break
|
||||
done &
|
||||
xeyes # distraction window to prevent ^w from closing the qr-code
|
||||
) &
|
||||
|
||||
# bail if copyparty is already running
|
||||
ps aux | grep -E '[3] copy[p]arty' && exit 0
|
||||
|
||||
# dumb chrome wrapper to allow autoplay
|
||||
cat >/usr/local/bin/chromium-browser <<'EOF'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
/usr/bin/chromium-browser --autoplay-policy=no-user-gesture-required "$@"
|
||||
EOF
|
||||
chmod 755 /usr/local/bin/chromium-browser
|
||||
|
||||
# start the server (note: replace `-v.::rw:` with `-v.::w:` to disallow retrieving uploaded stuff)
|
||||
cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,very-bad-idea.py
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess as sp
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
with open(fp, "rb") as f:
|
||||
txt = f.read(4096)
|
||||
|
||||
if txt.startswith(b"msg="):
|
||||
open_post(txt)
|
||||
else:
|
||||
open_url(fp)
|
||||
|
||||
|
||||
def open_post(txt):
|
||||
txt = unquote(txt.replace(b"+", b" ")).decode("utf-8")[4:]
|
||||
try:
|
||||
k, v = txt.split(" ", 1)
|
||||
except:
|
||||
open_url(txt)
|
||||
|
||||
if k == "key":
|
||||
sp.call(["xdotool", "key"] + v.split(" "))
|
||||
elif k == "x":
|
||||
sp.call(["xdotool"] + v.split(" "))
|
||||
elif k == "c":
|
||||
env = os.environ.copy()
|
||||
while " " in v:
|
||||
v1, v2 = v.split(" ", 1)
|
||||
if "=" not in v1:
|
||||
break
|
||||
|
||||
ek, ev = v1.split("=", 1)
|
||||
env[ek] = ev
|
||||
v = v2
|
||||
|
||||
sp.call(v.split(" "), env=env)
|
||||
else:
|
||||
open_url(txt)
|
||||
|
||||
|
||||
def open_url(txt):
|
||||
ext = txt.rsplit(".")[-1].lower()
|
||||
sp.call(["notify-send", "--", txt])
|
||||
if ext not in ["jpg", "jpeg", "png", "gif", "webp"]:
|
||||
# sp.call(["wmctrl", "-c", ":ACTIVE:"]) # closes the active window correctly
|
||||
sp.call(["killall", "vlc"])
|
||||
sp.call(["killall", "mpv"])
|
||||
sp.call(["killall", "feh"])
|
||||
time.sleep(0.5)
|
||||
for _ in range(20):
|
||||
sp.call(["xdotool", "key", "ctrl+w"]) # closes the open tab correctly
|
||||
# else:
|
||||
# sp.call(["xdotool", "getactivewindow", "windowminimize"]) # minimizes the focused windo
|
||||
|
||||
# close any error messages:
|
||||
sp.call(["xdotool", "search", "--name", "Error", "windowclose"])
|
||||
# sp.call(["xdotool", "key", "ctrl+alt+d"]) # doesnt work at all
|
||||
# sp.call(["xdotool", "keydown", "--delay", "100", "ctrl+alt+d"])
|
||||
# sp.call(["xdotool", "keyup", "ctrl+alt+d"])
|
||||
sp.call(["xdg-open", txt])
|
||||
|
||||
|
||||
main()
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
_ = r"""
|
||||
inspects video files for errors and such
|
||||
plus stores a bunch of metadata to filename.ff.json
|
||||
|
||||
usage:
|
||||
-mtp vidchk=t600,ay,p,bin/mtag/vidchk.py
|
||||
|
||||
explained:
|
||||
t600: timeout 10min
|
||||
ay: only process files which contain audio (including video with audio)
|
||||
p: set priority 1 (lowest priority after initial ffprobe/mutagen for base tags),
|
||||
makes copyparty feed base tags into this script as json
|
||||
|
||||
if you wanna use this script standalone / separately from copyparty,
|
||||
provide the video resolution on stdin as json: {"res":"1920x1080"}
|
||||
"""
|
||||
|
||||
|
||||
FAST = True # parse entire file at container level
|
||||
# FAST = False # fully decode audio and video streams
|
||||
|
||||
|
||||
# warnings to ignore
|
||||
harmless = re.compile(
|
||||
r"Unsupported codec with id |Could not find codec parameters.*Attachment:|analyzeduration"
|
||||
+ r"|timescale not set"
|
||||
)
|
||||
|
||||
|
||||
def wfilter(lines):
|
||||
return [x for x in lines if x.strip() and not harmless.search(x)]
|
||||
|
||||
|
||||
def errchk(so, se, rc, dbg):
|
||||
if dbg:
|
||||
with open(dbg, "wb") as f:
|
||||
f.write(b"so:\n" + so + b"\nse:\n" + se + b"\n")
|
||||
|
||||
if rc:
|
||||
err = (so + se).decode("utf-8", "replace").split("\n", 1)
|
||||
err = wfilter(err) or err
|
||||
return f"ERROR {rc}: {err[0]}"
|
||||
|
||||
if se:
|
||||
err = se.decode("utf-8", "replace").split("\n", 1)
|
||||
err = wfilter(err)
|
||||
if err:
|
||||
return f"Warning: {err[0]}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
fp = sys.argv[1]
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
fdir = os.path.dirname(os.path.realpath(fp))
|
||||
flag = os.path.join(fdir, ".processed")
|
||||
if os.path.exists(flag):
|
||||
return "already processed"
|
||||
|
||||
try:
|
||||
w, h = [int(x) for x in md["res"].split("x")]
|
||||
if not w + h:
|
||||
raise Exception()
|
||||
except:
|
||||
return "could not determine resolution"
|
||||
|
||||
# grab streams/format metadata + 2 seconds of frames at the start and end
|
||||
zs = "ffprobe -hide_banner -v warning -of json -show_streams -show_format -show_packets -show_data_hash crc32 -read_intervals %+2,999999%+2"
|
||||
cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
|
||||
so, se = p.communicate()
|
||||
|
||||
# spaces to tabs, drops filesize from 69k to 48k
|
||||
so = b"\n".join(
|
||||
[
|
||||
b"\t" * int((len(x) - len(x.lstrip())) / 4) + x.lstrip()
|
||||
for x in (so or b"").split(b"\n")
|
||||
]
|
||||
)
|
||||
with open(fsenc(f"{fp}.ff.json"), "wb") as f:
|
||||
f.write(so)
|
||||
|
||||
err = errchk(so, se, p.returncode, f"{fp}.vidchk")
|
||||
if err:
|
||||
return err
|
||||
|
||||
if max(w, h) < 1280 and min(w, h) < 720:
|
||||
return "resolution too small"
|
||||
|
||||
zs = (
|
||||
"ffmpeg -y -hide_banner -nostdin -v warning"
|
||||
+ " -err_detect +crccheck+bitstream+buffer+careful+compliant+aggressive+explode"
|
||||
+ " -xerror -i"
|
||||
)
|
||||
|
||||
cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
|
||||
|
||||
if FAST:
|
||||
zs = "-c copy -f null -"
|
||||
else:
|
||||
zs = "-vcodec rawvideo -acodec pcm_s16le -f null -"
|
||||
|
||||
cmd += zs.encode("ascii").split(b" ")
|
||||
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
|
||||
so, se = p.communicate()
|
||||
return errchk(so, se, p.returncode, f"{fp}.vidchk")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(main() or "ok")
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
DEPRECATED -- replaced by event hooks;
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/wget.py
|
||||
|
||||
---
|
||||
|
||||
use copyparty as a file downloader by POSTing URLs as
|
||||
application/x-www-form-urlencoded (for example using the
|
||||
message/pager function on the website)
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -vsrv/wget:wget:rwmd,ed:c,e2ts,mtp=title=ebin,t300,ad,bin/mtag/wget.py
|
||||
|
||||
explained:
|
||||
for realpath srv/wget (served at /wget) with read-write-modify-delete for ed,
|
||||
enable file analysis on upload (e2ts),
|
||||
use mtp plugin "bin/mtag/wget.py" to provide metadata tag "title",
|
||||
do this on all uploads with the file extension "bin",
|
||||
t300 = 300 seconds timeout for each dwonload,
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
fdir = os.path.dirname(fp)
|
||||
fname = os.path.basename(fp)
|
||||
if not fname.startswith("put-") or not fname.endswith(".bin"):
|
||||
raise Exception("not a post file")
|
||||
|
||||
buf = b""
|
||||
with open(fp, "rb") as f:
|
||||
while True:
|
||||
b = f.read(4096)
|
||||
buf += b
|
||||
if len(buf) > 4096:
|
||||
raise Exception("too big")
|
||||
|
||||
if not b:
|
||||
break
|
||||
|
||||
if not buf:
|
||||
raise Exception("file is empty")
|
||||
|
||||
buf = unquote(buf.replace(b"+", b" "))
|
||||
url = buf.decode("utf-8")
|
||||
|
||||
if not url.startswith("msg="):
|
||||
raise Exception("does not start with msg=")
|
||||
|
||||
url = url[4:]
|
||||
if "://" not in url:
|
||||
url = "https://" + url
|
||||
|
||||
os.chdir(fdir)
|
||||
|
||||
name = url.split("?")[0].split("/")[-1]
|
||||
tfn = "-- DOWNLOADING " + name
|
||||
open(tfn, "wb").close()
|
||||
|
||||
cmd = ["wget", "--trust-server-names", "--", url]
|
||||
|
||||
try:
|
||||
sp.check_call(cmd)
|
||||
|
||||
# OPTIONAL:
|
||||
# on success, delete the .bin file which contains the URL
|
||||
os.unlink(fp)
|
||||
except:
|
||||
open("-- FAILED TO DONWLOAD " + name, "wb").close()
|
||||
|
||||
os.unlink(tfn)
|
||||
print(url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,198 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import json
|
||||
import base64
|
||||
import string
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
"""
|
||||
youtube initial player response
|
||||
|
||||
it's probably best to use this through a config file; see res/yt-ipr.conf
|
||||
|
||||
but if you want to use plain arguments instead then:
|
||||
-v srv/ytm:ytm:w:rw,ed
|
||||
:c,e2ts,e2dsa
|
||||
:c,sz=16k-1m:c,maxn=10,300:c,rotf=%Y-%m/%d-%H
|
||||
:c,mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
|
||||
:c,mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
|
||||
|
||||
see res/yt-ipr.user.js for the example userscript to go with this
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
with gzip.open(sys.argv[1], "rt", encoding="utf-8", errors="replace") as f:
|
||||
txt = f.read()
|
||||
except:
|
||||
with open(sys.argv[1], "r", encoding="utf-8", errors="replace") as f:
|
||||
txt = f.read()
|
||||
|
||||
txt = "{" + txt.split("{", 1)[1]
|
||||
|
||||
try:
|
||||
pd = json.loads(txt)
|
||||
except json.decoder.JSONDecodeError as ex:
|
||||
pd = json.loads(txt[: ex.pos])
|
||||
|
||||
# print(json.dumps(pd, indent=2))
|
||||
|
||||
if "videoDetails" in pd:
|
||||
parse_youtube(pd)
|
||||
else:
|
||||
parse_freg(pd)
|
||||
|
||||
|
||||
def get_expiration(url):
|
||||
et = re.search(r"[?&]expire=([0-9]+)", url).group(1)
|
||||
et = datetime.utcfromtimestamp(int(et))
|
||||
return et.strftime("%Y-%m-%d, %H:%M")
|
||||
|
||||
|
||||
def parse_youtube(pd):
|
||||
vd = pd["videoDetails"]
|
||||
sd = pd["streamingData"]
|
||||
|
||||
et = sd["adaptiveFormats"][0]["url"]
|
||||
et = get_expiration(et)
|
||||
|
||||
mf = []
|
||||
if "dashManifestUrl" in sd:
|
||||
mf.append("dash")
|
||||
if "hlsManifestUrl" in sd:
|
||||
mf.append("hls")
|
||||
|
||||
r = {
|
||||
"yt-id": vd["videoId"],
|
||||
"yt-title": vd["title"],
|
||||
"yt-author": vd["author"],
|
||||
"yt-channel": vd["channelId"],
|
||||
"yt-views": vd["viewCount"],
|
||||
"yt-private": vd["isPrivate"],
|
||||
# "yt-expires": sd["expiresInSeconds"],
|
||||
"yt-manifest": ",".join(mf),
|
||||
"yt-expires": et,
|
||||
}
|
||||
print(json.dumps(r))
|
||||
|
||||
freg_conv(pd)
|
||||
|
||||
|
||||
def parse_freg(pd):
|
||||
md = pd["metadata"]
|
||||
r = {
|
||||
"yt-id": md["id"],
|
||||
"yt-title": md["title"],
|
||||
"yt-author": md["channelName"],
|
||||
"yt-channel": md["channelURL"].strip("/").split("/")[-1],
|
||||
"yt-expires": get_expiration(list(pd["video"].values())[0]),
|
||||
}
|
||||
print(json.dumps(r))
|
||||
|
||||
|
||||
def freg_conv(pd):
|
||||
# based on getURLs.js v1.5 (2021-08-07)
|
||||
# fmt: off
|
||||
priority = {
|
||||
"video": [
|
||||
337, 315, 266, 138, # 2160p60
|
||||
313, 336, # 2160p
|
||||
308, # 1440p60
|
||||
271, 264, # 1440p
|
||||
335, 303, 299, # 1080p60
|
||||
248, 169, 137, # 1080p
|
||||
334, 302, 298, # 720p60
|
||||
247, 136 # 720p
|
||||
],
|
||||
"audio": [
|
||||
251, 141, 171, 140, 250, 249, 139
|
||||
]
|
||||
}
|
||||
|
||||
vid_id = pd["videoDetails"]["videoId"]
|
||||
chan_id = pd["videoDetails"]["channelId"]
|
||||
|
||||
try:
|
||||
thumb_url = pd["microformat"]["playerMicroformatRenderer"]["thumbnail"]["thumbnails"][0]["url"]
|
||||
start_ts = pd["microformat"]["playerMicroformatRenderer"]["liveBroadcastDetails"]["startTimestamp"]
|
||||
except:
|
||||
thumb_url = f"https://img.youtube.com/vi/{vid_id}/maxresdefault.jpg"
|
||||
start_ts = ""
|
||||
|
||||
# fmt: on
|
||||
|
||||
metadata = {
|
||||
"title": pd["videoDetails"]["title"],
|
||||
"id": vid_id,
|
||||
"channelName": pd["videoDetails"]["author"],
|
||||
"channelURL": "https://www.youtube.com/channel/" + chan_id,
|
||||
"description": pd["videoDetails"]["shortDescription"],
|
||||
"thumbnailUrl": thumb_url,
|
||||
"startTimestamp": start_ts,
|
||||
}
|
||||
|
||||
if [x for x in vid_id if x not in string.ascii_letters + string.digits + "_-"]:
|
||||
print(f"malicious json", file=sys.stderr)
|
||||
return
|
||||
|
||||
basepath = os.path.dirname(sys.argv[1])
|
||||
|
||||
thumb_fn = f"{basepath}/{vid_id}.jpg"
|
||||
tmp_fn = f"{thumb_fn}.{os.getpid()}"
|
||||
if not os.path.exists(thumb_fn) and (
|
||||
thumb_url.startswith("https://img.youtube.com/vi/")
|
||||
or thumb_url.startswith("https://i.ytimg.com/vi/")
|
||||
):
|
||||
try:
|
||||
with urllib.request.urlopen(thumb_url) as fi:
|
||||
with open(tmp_fn, "wb") as fo:
|
||||
fo.write(fi.read())
|
||||
|
||||
os.rename(tmp_fn, thumb_fn)
|
||||
except:
|
||||
if os.path.exists(tmp_fn):
|
||||
os.unlink(tmp_fn)
|
||||
|
||||
try:
|
||||
with open(thumb_fn, "rb") as f:
|
||||
thumb = base64.b64encode(f.read()).decode("ascii")
|
||||
except:
|
||||
thumb = "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/yQALCAABAAEBAREA/8wABgAQEAX/2gAIAQEAAD8A0s8g/9k="
|
||||
|
||||
metadata["thumbnail"] = "data:image/jpeg;base64," + thumb
|
||||
|
||||
ret = {
|
||||
"metadata": metadata,
|
||||
"version": "1.5",
|
||||
"createTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
}
|
||||
|
||||
for stream, itags in priority.items():
|
||||
for itag in itags:
|
||||
url = None
|
||||
for afmt in pd["streamingData"]["adaptiveFormats"]:
|
||||
if itag == afmt["itag"]:
|
||||
url = afmt["url"]
|
||||
break
|
||||
|
||||
if url:
|
||||
ret[stream] = {itag: url}
|
||||
break
|
||||
|
||||
fn = f"{basepath}/{vid_id}.urls.json"
|
||||
with open(fn, "w", encoding="utf-8", errors="replace") as f:
|
||||
f.write(json.dumps(ret, indent=4))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except:
|
||||
# raise
|
||||
pass
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
partyjournal.py: chronological history of uploads
|
||||
2021-12-31, v0.1, ed <irc.rizon.net>, MIT-Licensed
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/partyjournal.py
|
||||
|
||||
produces a chronological list of all uploads,
|
||||
by collecting info from up2k databases and the filesystem
|
||||
|
||||
specify subnet `192.168.1.*` with argument `.=192.168.1.`,
|
||||
affecting all successive mappings
|
||||
|
||||
usage:
|
||||
./partyjournal.py > partyjournal.html .=192.168.1. cart=125 steen=114 steen=131 sleepy=121 fscarlet=144 ed=101 ed=123
|
||||
|
||||
"""
|
||||
|
||||
import sys
|
||||
import base64
|
||||
import sqlite3
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
FS_ENCODING = sys.getfilesystemencoding()
|
||||
|
||||
|
||||
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
|
||||
pass
|
||||
|
||||
|
||||
##
|
||||
## snibbed from copyparty
|
||||
|
||||
|
||||
def s3dec(v):
|
||||
if not v.startswith("//"):
|
||||
return v
|
||||
|
||||
v = base64.urlsafe_b64decode(v.encode("ascii")[2:])
|
||||
return v.decode(FS_ENCODING, "replace")
|
||||
|
||||
|
||||
def quotep(txt):
|
||||
btxt = txt.encode("utf-8", "replace")
|
||||
quot1 = quote(btxt, safe=b"/")
|
||||
quot1 = quot1.encode("ascii")
|
||||
quot2 = quot1.replace(b" ", b"+")
|
||||
return quot2.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def html_escape(s, quote=False, crlf=False):
|
||||
"""html.escape but also newlines"""
|
||||
s = s.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
if quote:
|
||||
s = s.replace('"', """).replace("'", "'")
|
||||
if crlf:
|
||||
s = s.replace("\r", " ").replace("\n", " ")
|
||||
|
||||
return s
|
||||
|
||||
|
||||
## end snibs
|
||||
##
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(formatter_class=APF)
|
||||
ap.add_argument("who", nargs="*")
|
||||
ar = ap.parse_args()
|
||||
|
||||
imap = {}
|
||||
subnet = ""
|
||||
for v in ar.who:
|
||||
if "=" not in v:
|
||||
raise Exception("bad who: " + v)
|
||||
|
||||
k, v = v.split("=")
|
||||
if k == ".":
|
||||
subnet = v
|
||||
continue
|
||||
|
||||
imap["{}{}".format(subnet, v)] = k
|
||||
|
||||
print(repr(imap), file=sys.stderr)
|
||||
|
||||
print(
|
||||
"""\
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><style>
|
||||
|
||||
html, body {
|
||||
color: #ccc;
|
||||
background: #222;
|
||||
font-family: sans-serif;
|
||||
}
|
||||
a {
|
||||
color: #fc5;
|
||||
}
|
||||
td, th {
|
||||
padding: .2em .5em;
|
||||
border: 1px solid #999;
|
||||
border-width: 0 1px 1px 0;
|
||||
white-space: nowrap;
|
||||
}
|
||||
td:nth-child(1),
|
||||
td:nth-child(2),
|
||||
td:nth-child(3) {
|
||||
font-family: monospace, monospace;
|
||||
text-align: right;
|
||||
}
|
||||
tr:first-child {
|
||||
position: sticky;
|
||||
top: -1px;
|
||||
}
|
||||
th {
|
||||
background: #222;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
</style></head><body><table><tr>
|
||||
<th>wark</th>
|
||||
<th>time</th>
|
||||
<th>size</th>
|
||||
<th>who</th>
|
||||
<th>link</th>
|
||||
</tr>"""
|
||||
)
|
||||
|
||||
db_path = ".hist/up2k.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
q = r"pragma table_info(up)"
|
||||
inf = conn.execute(q).fetchall()
|
||||
cols = [x[1] for x in inf]
|
||||
print("<!-- " + str(cols) + " -->")
|
||||
# ['w', 'mt', 'sz', 'rd', 'fn', 'ip', 'at']
|
||||
|
||||
q = r"select * from up order by case when at > 0 then at else mt end"
|
||||
for w, mt, sz, rd, fn, ip, at in conn.execute(q):
|
||||
link = "/".join([s3dec(x) for x in [rd, fn] if x])
|
||||
if fn.startswith("put-") and sz < 4096:
|
||||
try:
|
||||
with open(link, "rb") as f:
|
||||
txt = f.read().decode("utf-8", "replace")
|
||||
except:
|
||||
continue
|
||||
|
||||
if txt.startswith("msg="):
|
||||
txt = txt.encode("utf-8", "replace")
|
||||
txt = unquote(txt.replace(b"+", b" "))
|
||||
link = txt.decode("utf-8")[4:]
|
||||
|
||||
sz = "{:,}".format(sz)
|
||||
v = [
|
||||
w[:16],
|
||||
datetime.utcfromtimestamp(at if at > 0 else mt).strftime(
|
||||
"%Y-%m-%d %H:%M:%S"
|
||||
),
|
||||
sz,
|
||||
imap.get(ip, ip),
|
||||
]
|
||||
|
||||
row = "<tr>\n "
|
||||
row += "\n ".join(["<td>{}</th>".format(x) for x in v])
|
||||
row += '\n <td><a href="{}">{}</a></td>'.format(link, html_escape(link))
|
||||
row += "\n</tr>"
|
||||
print(row)
|
||||
|
||||
print("</table></body></html>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,141 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# runs copyparty (or any other program really) in a chroot
|
||||
#
|
||||
# assumption: these directories, and everything within, are owned by root
|
||||
sysdirs=(); for v in /bin /lib /lib32 /lib64 /sbin /usr /etc/alternatives ; do
|
||||
[ -e $v ] && sysdirs+=($v)
|
||||
done
|
||||
|
||||
# error-handler
|
||||
help() { cat <<'EOF'
|
||||
|
||||
usage:
|
||||
./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
|
||||
|
||||
example:
|
||||
./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
|
||||
|
||||
example for running straight from source (instead of using an sfx):
|
||||
PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
|
||||
|
||||
note that if you have python modules installed as --user (such as bpm/key detectors),
|
||||
you should add /home/foo/.local as a VOLDIR
|
||||
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
# read arguments
|
||||
trap help EXIT
|
||||
jail="$(realpath "$1")"; shift
|
||||
uid="$1"; shift
|
||||
gid="$1"; shift
|
||||
|
||||
vols=()
|
||||
while true; do
|
||||
v="$1"; shift
|
||||
[ "$v" = -- ] && break # end of volumes
|
||||
[ "$#" -eq 0 ] && break # invalid usage
|
||||
vols+=( "$(realpath "$v" || echo "$v")" )
|
||||
done
|
||||
pybin="$1"; shift
|
||||
pybin="$(command -v "$pybin")"
|
||||
pyarg=
|
||||
while true; do
|
||||
v="$1"
|
||||
[ "${v:0:1}" = - ] || break
|
||||
pyarg="$pyarg $v"
|
||||
shift
|
||||
done
|
||||
cpp="$1"; shift
|
||||
[ -d "$cpp" ] && cppdir="$PWD" || {
|
||||
# sfx, not module
|
||||
cpp="$(realpath "$cpp")"
|
||||
cppdir="$(dirname "$cpp")"
|
||||
}
|
||||
trap - EXIT
|
||||
|
||||
|
||||
# debug/vis
|
||||
echo
|
||||
echo "chroot-dir = $jail"
|
||||
echo "user:group = $uid:$gid"
|
||||
echo " copyparty = $cpp"
|
||||
echo
|
||||
printf '\033[33m%s\033[0m\n' "copyparty can access these folders and all their subdirectories:"
|
||||
for v in "${vols[@]}"; do
|
||||
printf '\033[36m ├─\033[0m %s \033[36m ── added by (You)\033[0m\n' "$v"
|
||||
done
|
||||
printf '\033[36m ├─\033[0m %s \033[36m ── where the copyparty binary is\033[0m\n' "$cppdir"
|
||||
printf '\033[36m ╰─\033[0m %s \033[36m ── the folder you are currently in\033[0m\n' "$PWD"
|
||||
vols+=("$cppdir" "$PWD")
|
||||
echo
|
||||
|
||||
|
||||
# remove any trailing slashes
|
||||
jail="${jail%/}"
|
||||
|
||||
|
||||
# bind-mount system directories and volumes
|
||||
printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | sed -r 's`/$``' | LC_ALL=C sort | uniq |
|
||||
while IFS= read -r v; do
|
||||
[ -e "$v" ] || {
|
||||
printf '\033[1;31mfolder does not exist:\033[0m %s\n' "$v"
|
||||
continue
|
||||
}
|
||||
i1=$(stat -c%D.%i "$v" 2>/dev/null || echo a)
|
||||
i2=$(stat -c%D.%i "$jail$v" 2>/dev/null || echo b)
|
||||
# echo "v [$v] i1 [$i1] i2 [$i2]"
|
||||
[ $i1 = $i2 ] && continue
|
||||
|
||||
mkdir -p "$jail$v"
|
||||
mount --bind "$v" "$jail$v"
|
||||
done
|
||||
|
||||
|
||||
cln() {
|
||||
rv=$?
|
||||
wait -f -p rv $p || true
|
||||
cd /
|
||||
echo "stopping chroot..."
|
||||
lsof "$jail" | grep -F "$jail" &&
|
||||
echo "chroot is in use; will not unmount" ||
|
||||
{
|
||||
mount | grep -F " on $jail" |
|
||||
awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
|
||||
LC_ALL=C sort -r | tee /dev/stderr | tr '\n' '\0' | xargs -r0 umount
|
||||
}
|
||||
exit $rv
|
||||
}
|
||||
trap cln EXIT
|
||||
|
||||
|
||||
# create a tmp
|
||||
mkdir -p "$jail/tmp"
|
||||
chmod 777 "$jail/tmp"
|
||||
|
||||
|
||||
# create a dev
|
||||
(cd $jail; mkdir -p dev; cd dev
|
||||
[ -e null ] || mknod -m 666 null c 1 3
|
||||
[ -e zero ] || mknod -m 666 zero c 1 5
|
||||
[ -e random ] || mknod -m 444 random c 1 8
|
||||
[ -e urandom ] || mknod -m 444 urandom c 1 9
|
||||
)
|
||||
|
||||
|
||||
# run copyparty
|
||||
export HOME=$(getent passwd $uid | cut -d: -f6)
|
||||
export USER=$(getent passwd $uid | cut -d: -f1)
|
||||
export LOGNAME="$USER"
|
||||
#echo "pybin [$pybin]"
|
||||
#echo "pyarg [$pyarg]"
|
||||
#echo "cpp [$cpp]"
|
||||
chroot --userspec=$uid:$gid "$jail" "$pybin" $pyarg "$cpp" "$@" &
|
||||
p=$!
|
||||
trap 'kill -USR1 $p' USR1
|
||||
trap 'kill $p' INT TERM
|
||||
wait
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
unforget.py: rebuild db from logfiles
|
||||
2022-09-07, v0.1, ed <irc.rizon.net>, MIT-Licensed
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/unforget.py
|
||||
|
||||
only makes sense if running copyparty with --no-forget
|
||||
(e.g. immediately shifting uploads to other storage)
|
||||
|
||||
usage:
|
||||
xz -d < log | ./unforget.py .hist/up2k.db
|
||||
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import sqlite3
|
||||
import argparse
|
||||
|
||||
|
||||
FS_ENCODING = sys.getfilesystemencoding()
|
||||
|
||||
|
||||
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
|
||||
pass
|
||||
|
||||
|
||||
mem_cur = sqlite3.connect(":memory:").cursor()
|
||||
mem_cur.execute(r"create table a (b text)")
|
||||
|
||||
|
||||
def s3enc(rd: str, fn: str) -> tuple[str, str]:
|
||||
ret: list[str] = []
|
||||
for v in [rd, fn]:
|
||||
try:
|
||||
mem_cur.execute("select * from a where b = ?", (v,))
|
||||
ret.append(v)
|
||||
except:
|
||||
wtf8 = v.encode(FS_ENCODING, "surrogateescape")
|
||||
ret.append("//" + base64.urlsafe_b64encode(wtf8).decode("ascii"))
|
||||
|
||||
return ret[0], ret[1]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("db")
|
||||
ar = ap.parse_args()
|
||||
|
||||
db = sqlite3.connect(ar.db).cursor()
|
||||
ptn_times = re.compile(r"no more chunks, setting times \(([0-9]+)")
|
||||
at = 0
|
||||
ctr = 0
|
||||
|
||||
for ln in [x.decode("utf-8", "replace").rstrip() for x in sys.stdin.buffer]:
|
||||
if "no more chunks, setting times (" in ln:
|
||||
m = ptn_times.search(ln)
|
||||
if m:
|
||||
at = int(m.group(1))
|
||||
|
||||
if '"hash": []' in ln:
|
||||
try:
|
||||
ofs = ln.find("{")
|
||||
j = json.loads(ln[ofs:])
|
||||
except:
|
||||
pass
|
||||
|
||||
w = j["wark"]
|
||||
if db.execute("select w from up where w = ?", (w,)).fetchone():
|
||||
continue
|
||||
|
||||
# PYTHONPATH=/home/ed/dev/copyparty/ python3 -m copyparty -e2dsa -v foo:foo:rwmd,ed -aed:wark --no-forget
|
||||
# 05:34:43.845 127.0.0.1 42496 no more chunks, setting times (1662528883, 1658001882)
|
||||
# 05:34:43.863 127.0.0.1 42496 {"name": "f\"2", "purl": "/foo/bar/baz/", "size": 1674, "lmod": 1658001882, "sprs": true, "hash": [], "wark": "LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg"}
|
||||
# | w | mt | sz | rd | fn | ip | at |
|
||||
# | LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg | 1658001882 | 1674 | bar/baz | f"2 | 127.0.0.1 | 1662528883 |
|
||||
|
||||
rd, fn = s3enc(j["purl"].strip("/"), j["name"])
|
||||
ip = ln.split(" ")[1].split("m")[-1]
|
||||
|
||||
q = "insert into up values (?,?,?,?,?,?,?)"
|
||||
v = (w, int(j["lmod"]), int(j["size"]), rd, fn, ip, at)
|
||||
db.execute(q, v)
|
||||
ctr += 1
|
||||
if ctr % 1024 == 1023:
|
||||
print(f"{ctr} commit...")
|
||||
db.connection.commit()
|
||||
|
||||
if ctr:
|
||||
db.connection.commit()
|
||||
|
||||
print(f"unforgot {ctr} files")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1178
bin/up2k.py
1178
bin/up2k.py
File diff suppressed because it is too large
Load Diff
24
bin/up2k.sh
Normal file → Executable file
24
bin/up2k.sh
Normal file → Executable file
@@ -8,7 +8,7 @@ set -e
|
||||
##
|
||||
## config
|
||||
|
||||
datalen=$((128*1024*1024))
|
||||
datalen=$((2*1024*1024*1024))
|
||||
target=127.0.0.1
|
||||
posturl=/inc
|
||||
passwd=wark
|
||||
@@ -37,10 +37,10 @@ gendata() {
|
||||
# pipe a chunk, get the base64 checksum
|
||||
gethash() {
|
||||
printf $(
|
||||
sha512sum | cut -c-66 |
|
||||
sha512sum | cut -c-64 |
|
||||
sed -r 's/ .*//;s/(..)/\\x\1/g'
|
||||
) |
|
||||
base64 -w0 | cut -c-44 |
|
||||
base64 -w0 | cut -c-43 |
|
||||
tr '+/' '-_'
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ printf '\033[36m'
|
||||
{
|
||||
{
|
||||
cat <<EOF
|
||||
POST $posturl/ HTTP/1.1
|
||||
POST $posturl/handshake.php HTTP/1.1
|
||||
Connection: Close
|
||||
Cookie: cppwd=$passwd
|
||||
Content-Type: text/plain;charset=UTF-8
|
||||
@@ -145,16 +145,14 @@ printf '\033[0m\nwark: %s\n' $wark
|
||||
##
|
||||
## wait for signal to continue
|
||||
|
||||
true || {
|
||||
w8=/dev/shm/$salt.w8
|
||||
touch $w8
|
||||
w8=/dev/shm/$salt.w8
|
||||
touch $w8
|
||||
|
||||
echo "ready; rm -f $w8"
|
||||
echo "ready; rm -f $w8"
|
||||
|
||||
while [ -e $w8 ]; do
|
||||
sleep 0.2
|
||||
done
|
||||
}
|
||||
while [ -e $w8 ]; do
|
||||
sleep 0.2
|
||||
done
|
||||
|
||||
|
||||
##
|
||||
@@ -177,7 +175,7 @@ while [ $remains -gt 0 ]; do
|
||||
|
||||
{
|
||||
cat <<EOF
|
||||
POST $posturl/ HTTP/1.1
|
||||
POST $posturl/chunkpit.php HTTP/1.1
|
||||
Connection: Keep-Alive
|
||||
Cookie: cppwd=$passwd
|
||||
Content-Type: application/octet-stream
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
### [`plugins/`](plugins/)
|
||||
* example extensions
|
||||
|
||||
### [`copyparty.bat`](copyparty.bat)
|
||||
* launches copyparty with no arguments (anon read+write within same folder)
|
||||
* intended for windows machines with no python.exe in PATH
|
||||
@@ -22,29 +19,17 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
|
||||
* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
|
||||
* `pw`: password (remove `Parameters` if anon-write)
|
||||
|
||||
### [`media-osd-bgone.ps1`](media-osd-bgone.ps1)
|
||||
* disables the [windows OSD popup](https://user-images.githubusercontent.com/241032/122821375-0e08df80-d2dd-11eb-9fd9-184e8aacf1d0.png) (the thing on the left) which appears every time you hit media hotkeys to adjust volume or change song while playing music with the copyparty web-ui, or most other audio players really
|
||||
|
||||
### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
|
||||
* disables thumbnails and folder-type detection in windows explorer
|
||||
* makes it way faster (especially for slow/networked locations (such as partyfuse))
|
||||
|
||||
### [`webdav-cfg.reg`](webdav-cfg.bat)
|
||||
* improves the native webdav support in windows;
|
||||
* removes the 47.6 MiB filesize limit when downloading from webdav
|
||||
* optionally enables webdav basic-auth over plaintext http
|
||||
* optionally helps disable wpad, removing the 10sec latency
|
||||
* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
|
||||
|
||||
### [`cfssl.sh`](cfssl.sh)
|
||||
* creates CA and server certificates using cfssl
|
||||
* give a 3rd argument to install it to your copyparty config
|
||||
* systemd service at [`systemd/cfssl.service`](systemd/cfssl.service)
|
||||
|
||||
# OS integration
|
||||
init-scripts to start copyparty as a service
|
||||
* [`systemd/copyparty.service`](systemd/copyparty.service) runs the sfx normally
|
||||
* [`rc/copyparty`](rc/copyparty) runs sfx normally on freebsd, create a `copyparty` user
|
||||
* [`systemd/prisonparty.service`](systemd/prisonparty.service) runs the sfx in a chroot
|
||||
* [`systemd/copyparty.service`](systemd/copyparty.service)
|
||||
* [`openrc/copyparty`](openrc/copyparty)
|
||||
|
||||
# Reverse-proxy
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
# when running copyparty behind a reverse proxy,
|
||||
# the following arguments are recommended:
|
||||
#
|
||||
# -i 127.0.0.1 only accept connections from nginx
|
||||
#
|
||||
# if you are doing location-based proxying (such as `/stuff` below)
|
||||
# you must run copyparty with --rp-loc=stuff
|
||||
#
|
||||
# on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
|
||||
|
||||
LoadModule proxy_module modules/mod_proxy.so
|
||||
ProxyPass "/stuff" "http://127.0.0.1:3923/stuff"
|
||||
# do not specify ProxyPassReverse
|
||||
RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}
|
||||
@@ -1,14 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# ca-name and server-fqdn
|
||||
# ca-name and server-name
|
||||
ca_name="$1"
|
||||
srv_fqdn="$2"
|
||||
srv_name="$2"
|
||||
|
||||
[ -z "$srv_fqdn" ] && {
|
||||
[ -z "$srv_name" ] && {
|
||||
echo "need arg 1: ca name"
|
||||
echo "need arg 2: server fqdn and/or IPs, comma-separated"
|
||||
echo "optional arg 3: if set, write cert into copyparty cfg"
|
||||
echo "need arg 2: server name"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -32,15 +31,15 @@ EOF
|
||||
gen_srv() {
|
||||
(tee /dev/stderr <<EOF
|
||||
{"key": {"algo":"rsa", "size":4096},
|
||||
"names": [{"O":"$ca_name - $srv_fqdn"}]}
|
||||
"names": [{"O":"$ca_name - $srv_name"}]}
|
||||
EOF
|
||||
)|
|
||||
cfssl gencert -ca ca.pem -ca-key ca.key \
|
||||
-profile=www -hostname="$srv_fqdn" - |
|
||||
cfssljson -bare "$srv_fqdn"
|
||||
-profile=www -hostname="$srv_name.$ca_name" - |
|
||||
cfssljson -bare "$srv_name"
|
||||
|
||||
mv "$srv_fqdn-key.pem" "$srv_fqdn.key"
|
||||
rm "$srv_fqdn.csr"
|
||||
mv "$srv_name-key.pem" "$srv_name.key"
|
||||
rm "$srv_name.csr"
|
||||
}
|
||||
|
||||
|
||||
@@ -58,13 +57,13 @@ show() {
|
||||
awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
|
||||
}
|
||||
show ca.pem
|
||||
show "$srv_fqdn.pem"
|
||||
show "$srv_name.pem"
|
||||
|
||||
|
||||
# write cert into copyparty config
|
||||
[ -z "$3" ] || {
|
||||
mkdir -p ~/.config/copyparty
|
||||
cat "$srv_fqdn".{key,pem} ca.pem >~/.config/copyparty/cert.pem
|
||||
cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>💾🎉 redirect</title>
|
||||
<title>⇆🎉 redirect</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<style>
|
||||
|
||||
|
||||
Binary file not shown.
@@ -1,104 +0,0 @@
|
||||
# media-osd-bgone.ps1: disable media-control OSD on win10do
|
||||
# v1.1, 2021-06-25, ed <irc.rizon.net>, MIT-licensed
|
||||
# https://github.com/9001/copyparty/blob/hovudstraum/contrib/media-osd-bgone.ps1
|
||||
#
|
||||
# locates the first window that looks like the media OSD and minimizes it;
|
||||
# doing this once after each reboot should do the trick
|
||||
# (adjust the width/height filter if it doesn't work)
|
||||
#
|
||||
# ---------------------------------------------------------------------
|
||||
#
|
||||
# tip: save the following as "media-osd-bgone.bat" next to this script:
|
||||
# start cmd /c "powershell -command ""set-executionpolicy -scope process bypass; .\media-osd-bgone.ps1"" & ping -n 2 127.1 >nul"
|
||||
#
|
||||
# then create a shortcut to that bat-file and move the shortcut here:
|
||||
# %appdata%\Microsoft\Windows\Start Menu\Programs\Startup
|
||||
#
|
||||
# and now this will autorun on bootup
|
||||
|
||||
|
||||
Add-Type -TypeDefinition @"
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Windows.Forms;
|
||||
|
||||
namespace A {
|
||||
public class B : Control {
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, int dwExtraInfo);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
static extern IntPtr FindWindowEx(IntPtr hwndParent, IntPtr hwndChildAfter, string lpszClass, string lpszWindow);
|
||||
|
||||
[DllImport("user32.dll", SetLastError=true)]
|
||||
static extern bool GetWindowRect(IntPtr hwnd, out RECT lpRect);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct RECT {
|
||||
public int x;
|
||||
public int y;
|
||||
public int x2;
|
||||
public int y2;
|
||||
}
|
||||
|
||||
bool fa() {
|
||||
RECT r;
|
||||
IntPtr it = IntPtr.Zero;
|
||||
while ((it = FindWindowEx(IntPtr.Zero, it, "NativeHWNDHost", "")) != IntPtr.Zero) {
|
||||
if (FindWindowEx(it, IntPtr.Zero, "DirectUIHWND", "") == IntPtr.Zero)
|
||||
continue;
|
||||
|
||||
if (!GetWindowRect(it, out r))
|
||||
continue;
|
||||
|
||||
int w = r.x2 - r.x + 1;
|
||||
int h = r.y2 - r.y + 1;
|
||||
|
||||
Console.WriteLine("[*] hwnd {0:x} @ {1}x{2} sz {3}x{4}", it, r.x, r.y, w, h);
|
||||
if (h != 141)
|
||||
continue;
|
||||
|
||||
ShowWindow(it, 6);
|
||||
Console.WriteLine("[+] poof");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void fb() {
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
|
||||
Thread.Sleep(500);
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
|
||||
|
||||
while (true) {
|
||||
if (fa()) {
|
||||
break;
|
||||
}
|
||||
Console.WriteLine("[!] not found");
|
||||
Thread.Sleep(1000);
|
||||
}
|
||||
this.Invoke((MethodInvoker)delegate {
|
||||
Application.Exit();
|
||||
});
|
||||
}
|
||||
|
||||
public void Run() {
|
||||
Console.WriteLine("[+] hi");
|
||||
new Thread(new ThreadStart(fb)).Start();
|
||||
Application.Run();
|
||||
Console.WriteLine("[+] bye");
|
||||
}
|
||||
}
|
||||
}
|
||||
"@ -ReferencedAssemblies System.Windows.Forms
|
||||
|
||||
(New-Object -TypeName A.B).Run()
|
||||
@@ -1,20 +1,11 @@
|
||||
# when running copyparty behind a reverse proxy,
|
||||
# the following arguments are recommended:
|
||||
#
|
||||
# -i 127.0.0.1 only accept connections from nginx
|
||||
#
|
||||
# -nc must match or exceed the webserver's max number of concurrent clients;
|
||||
# copyparty default is 1024 if OS permits it (see "max clients:" on startup),
|
||||
# nginx default is 512 (worker_processes 1, worker_connections 512)
|
||||
#
|
||||
# you may also consider adding -j0 for CPU-intensive configurations
|
||||
# (5'000 requests per second, or 20gbps upload/download in parallel)
|
||||
#
|
||||
# on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
|
||||
# when running copyparty behind a reverse-proxy,
|
||||
# make sure that copyparty allows at least as many clients as the proxy does,
|
||||
# so run copyparty with -nc 512 if your nginx has the default limits
|
||||
# (worker_processes 1, worker_connections 512)
|
||||
|
||||
upstream cpp {
|
||||
server 127.0.0.1:3923;
|
||||
keepalive 1;
|
||||
keepalive 120;
|
||||
}
|
||||
server {
|
||||
listen 443 ssl;
|
||||
@@ -38,9 +29,3 @@ server {
|
||||
proxy_set_header Connection "Keep-Alive";
|
||||
}
|
||||
}
|
||||
|
||||
# default client_max_body_size (1M) blocks uploads larger than 256 MiB
|
||||
client_max_body_size 1024M;
|
||||
client_header_timeout 610m;
|
||||
client_body_timeout 610m;
|
||||
send_timeout 610m;
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
mkKeyValue = key: value:
|
||||
if value == true then
|
||||
# sets with a true boolean value are coerced to just the key name
|
||||
key
|
||||
else if value == false then
|
||||
# or omitted completely when false
|
||||
""
|
||||
else
|
||||
(generators.mkKeyValueDefault { inherit mkValueString; } ": " key value);
|
||||
|
||||
mkAttrsString = value: (generators.toKeyValue { inherit mkKeyValue; } value);
|
||||
|
||||
mkValueString = value:
|
||||
if isList value then
|
||||
(concatStringsSep ", " (map mkValueString value))
|
||||
else if isAttrs value then
|
||||
"\n" + (mkAttrsString value)
|
||||
else
|
||||
(generators.mkValueStringDefault { } value);
|
||||
|
||||
mkSectionName = value: "[" + (escape [ "[" "]" ] value) + "]";
|
||||
|
||||
mkSection = name: attrs: ''
|
||||
${mkSectionName name}
|
||||
${mkAttrsString attrs}
|
||||
'';
|
||||
|
||||
mkVolume = name: attrs: ''
|
||||
${mkSectionName name}
|
||||
${attrs.path}
|
||||
${mkAttrsString {
|
||||
accs = attrs.access;
|
||||
flags = attrs.flags;
|
||||
}}
|
||||
'';
|
||||
|
||||
passwordPlaceholder = name: "{{password-${name}}}";
|
||||
|
||||
accountsWithPlaceholders = mapAttrs (name: attrs: passwordPlaceholder name);
|
||||
|
||||
configStr = ''
|
||||
${mkSection "global" cfg.settings}
|
||||
${mkSection "accounts" (accountsWithPlaceholders cfg.accounts)}
|
||||
${concatStringsSep "\n" (mapAttrsToList mkVolume cfg.volumes)}
|
||||
'';
|
||||
|
||||
name = "copyparty";
|
||||
cfg = config.services.copyparty;
|
||||
configFile = pkgs.writeText "${name}.conf" configStr;
|
||||
runtimeConfigPath = "/run/${name}/${name}.conf";
|
||||
home = "/var/lib/${name}";
|
||||
defaultShareDir = "${home}/data";
|
||||
in {
|
||||
options.services.copyparty = {
|
||||
enable = mkEnableOption "web-based file manager";
|
||||
|
||||
package = mkOption {
|
||||
type = types.package;
|
||||
default = pkgs.copyparty;
|
||||
defaultText = "pkgs.copyparty";
|
||||
description = ''
|
||||
Package of the application to run, exposed for overriding purposes.
|
||||
'';
|
||||
};
|
||||
|
||||
openFilesLimit = mkOption {
|
||||
default = 4096;
|
||||
type = types.either types.int types.str;
|
||||
description = "Number of files to allow copyparty to open.";
|
||||
};
|
||||
|
||||
settings = mkOption {
|
||||
type = types.attrs;
|
||||
description = ''
|
||||
Global settings to apply.
|
||||
Directly maps to values in the [global] section of the copyparty config.
|
||||
See `${getExe cfg.package} --help` for more details.
|
||||
'';
|
||||
default = {
|
||||
i = "127.0.0.1";
|
||||
no-reload = true;
|
||||
};
|
||||
example = literalExpression ''
|
||||
{
|
||||
i = "0.0.0.0";
|
||||
no-reload = true;
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
accounts = mkOption {
|
||||
type = types.attrsOf (types.submodule ({ ... }: {
|
||||
options = {
|
||||
passwordFile = mkOption {
|
||||
type = types.str;
|
||||
description = ''
|
||||
Runtime file path to a file containing the user password.
|
||||
Must be readable by the copyparty user.
|
||||
'';
|
||||
example = "/run/keys/copyparty/ed";
|
||||
};
|
||||
};
|
||||
}));
|
||||
description = ''
|
||||
A set of copyparty accounts to create.
|
||||
'';
|
||||
default = { };
|
||||
example = literalExpression ''
|
||||
{
|
||||
ed.passwordFile = "/run/keys/copyparty/ed";
|
||||
};
|
||||
'';
|
||||
};
|
||||
|
||||
volumes = mkOption {
|
||||
type = types.attrsOf (types.submodule ({ ... }: {
|
||||
options = {
|
||||
path = mkOption {
|
||||
type = types.str;
|
||||
description = ''
|
||||
Path of a directory to share.
|
||||
'';
|
||||
};
|
||||
access = mkOption {
|
||||
type = types.attrs;
|
||||
description = ''
|
||||
Attribute list of permissions and the users to apply them to.
|
||||
|
||||
The key must be a string containing any combination of allowed permission:
|
||||
"r" (read): list folder contents, download files
|
||||
"w" (write): upload files; need "r" to see the uploads
|
||||
"m" (move): move files and folders; need "w" at destination
|
||||
"d" (delete): permanently delete files and folders
|
||||
"g" (get): download files, but cannot see folder contents
|
||||
"G" (upget): "get", but can see filekeys of their own uploads
|
||||
|
||||
For example: "rwmd"
|
||||
|
||||
The value must be one of:
|
||||
an account name, defined in `accounts`
|
||||
a list of account names
|
||||
"*", which means "any account"
|
||||
'';
|
||||
example = literalExpression ''
|
||||
{
|
||||
# wG = write-upget = see your own uploads only
|
||||
wG = "*";
|
||||
# read-write-modify-delete for users "ed" and "k"
|
||||
rwmd = ["ed" "k"];
|
||||
};
|
||||
'';
|
||||
};
|
||||
flags = mkOption {
|
||||
type = types.attrs;
|
||||
description = ''
|
||||
Attribute list of volume flags to apply.
|
||||
See `${getExe cfg.package} --help-flags` for more details.
|
||||
'';
|
||||
example = literalExpression ''
|
||||
{
|
||||
# "fk" enables filekeys (necessary for upget permission) (4 chars long)
|
||||
fk = 4;
|
||||
# scan for new files every 60sec
|
||||
scan = 60;
|
||||
# volflag "e2d" enables the uploads database
|
||||
e2d = true;
|
||||
# "d2t" disables multimedia parsers (in case the uploads are malicious)
|
||||
d2t = true;
|
||||
# skips hashing file contents if path matches *.iso
|
||||
nohash = "\.iso$";
|
||||
};
|
||||
'';
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
}));
|
||||
description = "A set of copyparty volumes to create";
|
||||
default = {
|
||||
"/" = {
|
||||
path = defaultShareDir;
|
||||
access = { r = "*"; };
|
||||
};
|
||||
};
|
||||
example = literalExpression ''
|
||||
{
|
||||
"/" = {
|
||||
path = ${defaultShareDir};
|
||||
access = {
|
||||
# wG = write-upget = see your own uploads only
|
||||
wG = "*";
|
||||
# read-write-modify-delete for users "ed" and "k"
|
||||
rwmd = ["ed" "k"];
|
||||
};
|
||||
};
|
||||
};
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
systemd.services.copyparty = {
|
||||
description = "http file sharing hub";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
environment = {
|
||||
PYTHONUNBUFFERED = "true";
|
||||
XDG_CONFIG_HOME = "${home}/.config";
|
||||
};
|
||||
|
||||
preStart = let
|
||||
replaceSecretCommand = name: attrs:
|
||||
"${getExe pkgs.replace-secret} '${
|
||||
passwordPlaceholder name
|
||||
}' '${attrs.passwordFile}' ${runtimeConfigPath}";
|
||||
in ''
|
||||
set -euo pipefail
|
||||
install -m 600 ${configFile} ${runtimeConfigPath}
|
||||
${concatStringsSep "\n"
|
||||
(mapAttrsToList replaceSecretCommand cfg.accounts)}
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${getExe cfg.package} -c ${runtimeConfigPath}";
|
||||
|
||||
# Hardening options
|
||||
User = "copyparty";
|
||||
Group = "copyparty";
|
||||
RuntimeDirectory = name;
|
||||
RuntimeDirectoryMode = "0700";
|
||||
StateDirectory = [ name "${name}/data" "${name}/.config" ];
|
||||
StateDirectoryMode = "0700";
|
||||
WorkingDirectory = home;
|
||||
TemporaryFileSystem = "/:ro";
|
||||
BindReadOnlyPaths = [
|
||||
"/nix/store"
|
||||
"-/etc/resolv.conf"
|
||||
"-/etc/nsswitch.conf"
|
||||
"-/etc/hosts"
|
||||
"-/etc/localtime"
|
||||
] ++ (mapAttrsToList (k: v: "-${v.passwordFile}") cfg.accounts);
|
||||
BindPaths = [ home ] ++ (mapAttrsToList (k: v: v.path) cfg.volumes);
|
||||
# Would re-mount paths ignored by temporary root
|
||||
#ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictSUIDSGID = true;
|
||||
PrivateMounts = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectHostname = true;
|
||||
ProtectClock = true;
|
||||
ProtectProc = "invisible";
|
||||
ProcSubset = "pid";
|
||||
RestrictNamespaces = true;
|
||||
RemoveIPC = true;
|
||||
UMask = "0077";
|
||||
LimitNOFILE = cfg.openFilesLimit;
|
||||
NoNewPrivileges = true;
|
||||
LockPersonality = true;
|
||||
RestrictRealtime = true;
|
||||
};
|
||||
};
|
||||
|
||||
users.groups.copyparty = { };
|
||||
users.users.copyparty = {
|
||||
description = "Service user for copyparty";
|
||||
group = "copyparty";
|
||||
home = home;
|
||||
isSystemUser = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -8,11 +8,11 @@
|
||||
#
|
||||
# you may want to:
|
||||
# change '/usr/bin/python' to another interpreter
|
||||
# change '/mnt::rw' to another location or permission-set
|
||||
# change '/mnt::a' to another location or permission-set
|
||||
|
||||
name="$SVCNAME"
|
||||
command_background=true
|
||||
pidfile="/var/run/$SVCNAME.pid"
|
||||
|
||||
command="/usr/bin/python3 /usr/local/bin/copyparty-sfx.py"
|
||||
command_args="-q -v /mnt::rw"
|
||||
command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
|
||||
command_args="-q -v /mnt::a"
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# Maintainer: icxes <dev.null@need.moe>
|
||||
pkgname=copyparty
|
||||
pkgver="1.6.14"
|
||||
pkgrel=1
|
||||
pkgdesc="Portable file sharing hub"
|
||||
arch=("any")
|
||||
url="https://github.com/9001/${pkgname}"
|
||||
license=('MIT')
|
||||
depends=("python" "lsof")
|
||||
optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
|
||||
"python-jinja: faster html generator"
|
||||
"python-mutagen: music tags (alternative)"
|
||||
"python-pillow: thumbnails for images"
|
||||
"python-pyvips: thumbnails for images (higher quality, faster, uses more ram)"
|
||||
"libkeyfinder-git: detection of musical keys"
|
||||
"qm-vamp-plugins: BPM detection"
|
||||
"python-pyopenssl: ftps functionality"
|
||||
"python-impacket-git: smb support (bad idea)"
|
||||
)
|
||||
source=("${url}/releases/download/v${pkgver}/${pkgname}-sfx.py"
|
||||
"${pkgname}.conf"
|
||||
"${pkgname}.service"
|
||||
"prisonparty.service"
|
||||
"index.md"
|
||||
"https://raw.githubusercontent.com/9001/${pkgname}/v${pkgver}/bin/prisonparty.sh"
|
||||
"https://raw.githubusercontent.com/9001/${pkgname}/v${pkgver}/LICENSE"
|
||||
)
|
||||
backup=("etc/${pkgname}.d/init" )
|
||||
sha256sums=("f3294a22fdd086605fe8d14bfeff620c6cff45c9019fd7d4af1a0ddd9e0d3947"
|
||||
"b8565eba5e64dedba1cf6c7aac7e31c5a731ed7153d6810288a28f00a36c28b2"
|
||||
"f65c207e0670f9d78ad2e399bda18d5502ff30d2ac79e0e7fc48e7fbdc39afdc"
|
||||
"c4f396b083c9ec02ad50b52412c84d2a82be7f079b2d016e1c9fad22d68285ff"
|
||||
"dba701de9fd584405917e923ea1e59dbb249b96ef23bad479cf4e42740b774c8"
|
||||
"8e89d281483e22d11d111bed540652af35b66af6f14f49faae7b959f6cdc6475"
|
||||
"cb2ce3d6277bf2f5a82ecf336cc44963bc6490bcf496ffbd75fc9e21abaa75f3"
|
||||
)
|
||||
|
||||
package() {
|
||||
cd "${srcdir}/"
|
||||
|
||||
install -dm755 "${pkgdir}/etc/${pkgname}.d"
|
||||
install -Dm755 "${pkgname}-sfx.py" "${pkgdir}/usr/bin/${pkgname}"
|
||||
install -Dm755 "prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
|
||||
install -Dm644 "${pkgname}.conf" "${pkgdir}/etc/${pkgname}.d/init"
|
||||
install -Dm644 "${pkgname}.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}.service"
|
||||
install -Dm644 "prisonparty.service" "${pkgdir}/usr/lib/systemd/system/prisonparty.service"
|
||||
install -Dm644 "index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
|
||||
install -Dm644 "LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
|
||||
|
||||
find /etc/${pkgname}.d -iname '*.conf' 2>/dev/null | grep -qE . && return
|
||||
echo "┏━━━━━━━━━━━━━━━──-"
|
||||
echo "┃ Configure ${pkgname} by adding .conf files into /etc/${pkgname}.d/"
|
||||
echo "┃ and maybe copy+edit one of the following to /etc/systemd/system/:"
|
||||
echo "┣━♦ /usr/lib/systemd/system/${pkgname}.service (standard)"
|
||||
echo "┣━♦ /usr/lib/systemd/system/prisonparty.service (chroot)"
|
||||
echo "┗━━━━━━━━━━━━━━━──-"
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
## import all *.conf files from the current folder (/etc/copyparty.d)
|
||||
% ./
|
||||
|
||||
# add additional .conf files to this folder;
|
||||
# see example config files for reference:
|
||||
# https://github.com/9001/copyparty/blob/hovudstraum/docs/example.conf
|
||||
# https://github.com/9001/copyparty/tree/hovudstraum/docs/copyparty.d
|
||||
@@ -1,32 +0,0 @@
|
||||
# this will start `/usr/bin/copyparty-sfx.py`
|
||||
# and read config from `/etc/copyparty.d/*.conf`
|
||||
#
|
||||
# you probably want to:
|
||||
# change "User=cpp" and "/home/cpp/" to another user
|
||||
#
|
||||
# unless you add -q to disable logging, you may want to remove the
|
||||
# following line to allow buffering (slightly better performance):
|
||||
# Environment=PYTHONUNBUFFERED=x
|
||||
|
||||
[Unit]
|
||||
Description=copyparty file server
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
SyslogIdentifier=copyparty
|
||||
Environment=PYTHONUNBUFFERED=x
|
||||
WorkingDirectory=/var/lib/copyparty-jail
|
||||
ExecReload=/bin/kill -s USR1 $MAINPID
|
||||
|
||||
# user to run as + where the TLS certificate is (if any)
|
||||
User=cpp
|
||||
Environment=XDG_CONFIG_HOME=/home/cpp/.config
|
||||
|
||||
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
|
||||
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
# run copyparty
|
||||
ExecStart=/usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,3 +0,0 @@
|
||||
this is `/var/lib/copyparty-jail`, the fallback webroot when copyparty has not yet been configured
|
||||
|
||||
please add some `*.conf` files to `/etc/copyparty.d/`
|
||||
@@ -1,31 +0,0 @@
|
||||
# this will start `/usr/bin/copyparty-sfx.py`
|
||||
# in a chroot, preventing accidental access elsewhere
|
||||
# and read config from `/etc/copyparty.d/*.conf`
|
||||
#
|
||||
# expose additional filesystem locations to copyparty
|
||||
# by listing them between the last `1000` and `--`
|
||||
#
|
||||
# `1000 1000` = what user to run copyparty as
|
||||
#
|
||||
# unless you add -q to disable logging, you may want to remove the
|
||||
# following line to allow buffering (slightly better performance):
|
||||
# Environment=PYTHONUNBUFFERED=x
|
||||
|
||||
[Unit]
|
||||
Description=copyparty file server
|
||||
|
||||
[Service]
|
||||
SyslogIdentifier=prisonparty
|
||||
Environment=PYTHONUNBUFFERED=x
|
||||
WorkingDirectory=/var/lib/copyparty-jail
|
||||
ExecReload=/bin/kill -s USR1 $MAINPID
|
||||
|
||||
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
|
||||
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
# run copyparty
|
||||
ExecStart=/bin/bash /usr/bin/prisonparty /var/lib/copyparty-jail 1000 1000 /etc/copyparty.d -- \
|
||||
/usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,55 +0,0 @@
|
||||
{ lib, stdenv, makeWrapper, fetchurl, utillinux, python, jinja2, impacket, pyftpdlib, pyopenssl, pillow, pyvips, ffmpeg, mutagen,
|
||||
|
||||
# create thumbnails with Pillow; faster than FFmpeg / MediaProcessing
|
||||
withThumbnails ? true,
|
||||
|
||||
# create thumbnails with PyVIPS; even faster, uses more memory
|
||||
# -- can be combined with Pillow to support more filetypes
|
||||
withFastThumbnails ? false,
|
||||
|
||||
# enable FFmpeg; thumbnails for most filetypes (also video and audio), extract audio metadata, transcode audio to opus
|
||||
# -- possibly dangerous if you allow anonymous uploads, since FFmpeg has a huge attack surface
|
||||
# -- can be combined with Thumbnails and/or FastThumbnails, since FFmpeg is slower than both
|
||||
withMediaProcessing ? true,
|
||||
|
||||
# if MediaProcessing is not enabled, you probably want this instead (less accurate, but much safer and faster)
|
||||
withBasicAudioMetadata ? false,
|
||||
|
||||
# enable FTPS support in the FTP server
|
||||
withFTPS ? false,
|
||||
|
||||
# samba/cifs server; dangerous and buggy, enable if you really need it
|
||||
withSMB ? false,
|
||||
|
||||
}:
|
||||
|
||||
let
|
||||
pinData = lib.importJSON ./pin.json;
|
||||
pyEnv = python.withPackages (ps:
|
||||
with ps; [
|
||||
jinja2
|
||||
]
|
||||
++ lib.optional withSMB impacket
|
||||
++ lib.optional withFTPS pyopenssl
|
||||
++ lib.optional withThumbnails pillow
|
||||
++ lib.optional withFastThumbnails pyvips
|
||||
++ lib.optional withMediaProcessing ffmpeg
|
||||
++ lib.optional withBasicAudioMetadata mutagen
|
||||
);
|
||||
in stdenv.mkDerivation {
|
||||
pname = "copyparty";
|
||||
version = pinData.version;
|
||||
src = fetchurl {
|
||||
url = pinData.url;
|
||||
hash = pinData.hash;
|
||||
};
|
||||
buildInputs = [ makeWrapper ];
|
||||
dontUnpack = true;
|
||||
dontBuild = true;
|
||||
installPhase = ''
|
||||
install -Dm755 $src $out/share/copyparty-sfx.py
|
||||
makeWrapper ${pyEnv.interpreter} $out/bin/copyparty \
|
||||
--set PATH '${lib.makeBinPath ([ utillinux ] ++ lib.optional withMediaProcessing ffmpeg)}:$PATH' \
|
||||
--add-flags "$out/share/copyparty-sfx.py"
|
||||
'';
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"url": "https://github.com/9001/copyparty/releases/download/v1.6.14/copyparty-sfx.py",
|
||||
"version": "1.6.14",
|
||||
"hash": "sha256-8ylKIv3QhmBf6NFL/v9iDGz/RckBn9fUrxoN3Z4NOUc="
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Update the Nix package pin
|
||||
#
|
||||
# Usage: ./update.sh [PATH]
|
||||
# When the [PATH] is not set, it will fetch the latest release from the repo.
|
||||
# With [PATH] set, it will hash the given file and generate the URL,
|
||||
# base on the version contained within the file
|
||||
|
||||
import base64
|
||||
import json
|
||||
import hashlib
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
OUTPUT_FILE = Path("pin.json")
|
||||
TARGET_ASSET = "copyparty-sfx.py"
|
||||
HASH_TYPE = "sha256"
|
||||
LATEST_RELEASE_URL = "https://api.github.com/repos/9001/copyparty/releases/latest"
|
||||
DOWNLOAD_URL = lambda version: f"https://github.com/9001/copyparty/releases/download/v{version}/{TARGET_ASSET}"
|
||||
|
||||
|
||||
def get_formatted_hash(binary):
|
||||
hasher = hashlib.new("sha256")
|
||||
hasher.update(binary)
|
||||
asset_hash = hasher.digest()
|
||||
encoded_hash = base64.b64encode(asset_hash).decode("ascii")
|
||||
return f"{HASH_TYPE}-{encoded_hash}"
|
||||
|
||||
|
||||
def version_from_sfx(binary):
|
||||
result = re.search(b'^VER = "(.*)"$', binary, re.MULTILINE)
|
||||
if result:
|
||||
return result.groups(1)[0].decode("ascii")
|
||||
|
||||
raise ValueError("version not found in provided file")
|
||||
|
||||
|
||||
def remote_release_pin():
|
||||
import requests
|
||||
|
||||
response = requests.get(LATEST_RELEASE_URL).json()
|
||||
version = response["tag_name"].lstrip("v")
|
||||
asset_info = [a for a in response["assets"] if a["name"] == TARGET_ASSET][0]
|
||||
download_url = asset_info["browser_download_url"]
|
||||
asset = requests.get(download_url)
|
||||
formatted_hash = get_formatted_hash(asset.content)
|
||||
|
||||
result = {"url": download_url, "version": version, "hash": formatted_hash}
|
||||
return result
|
||||
|
||||
|
||||
def local_release_pin(path):
|
||||
asset = path.read_bytes()
|
||||
version = version_from_sfx(asset)
|
||||
download_url = DOWNLOAD_URL(version)
|
||||
formatted_hash = get_formatted_hash(asset)
|
||||
|
||||
result = {"url": download_url, "version": version, "hash": formatted_hash}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1:
|
||||
asset_path = Path(sys.argv[1])
|
||||
result = local_release_pin(asset_path)
|
||||
else:
|
||||
result = remote_release_pin()
|
||||
|
||||
print(result)
|
||||
json_result = json.dumps(result, indent=4)
|
||||
OUTPUT_FILE.write_text(json_result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,33 +0,0 @@
|
||||
# example resource files
|
||||
|
||||
can be provided to copyparty to tweak things
|
||||
|
||||
|
||||
|
||||
## example `.epilogue.html`
|
||||
save one of these as `.epilogue.html` inside a folder to customize it:
|
||||
|
||||
* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
|
||||
|
||||
|
||||
|
||||
## example browser-js
|
||||
point `--js-browser` to one of these by URL:
|
||||
|
||||
* [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
|
||||
* [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
|
||||
* [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
|
||||
|
||||
|
||||
|
||||
## example browser-css
|
||||
point `--css-browser` to one of these by URL:
|
||||
|
||||
* [`browser-icons.css`](browser-icons.css) adds filetype icons
|
||||
|
||||
|
||||
|
||||
## meadup.js
|
||||
|
||||
* turns copyparty into chromecast just more flexible (and probably way more buggy)
|
||||
* usage: put the js somewhere in the webroot and `--js-browser /memes/meadup.js`
|
||||
@@ -1,71 +0,0 @@
|
||||
/* video, alternative 1:
|
||||
top-left icon, just like the other formats
|
||||
=======================================================================
|
||||
|
||||
#ggrid>a:is(
|
||||
[href$=".mkv"i],
|
||||
[href$=".mp4"i],
|
||||
[href$=".webm"i],
|
||||
):before {
|
||||
content: '📺';
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/* video, alternative 2:
|
||||
play-icon in the middle of the thumbnail
|
||||
=======================================================================
|
||||
*/
|
||||
#ggrid>a:is(
|
||||
[href$=".mkv"i],
|
||||
[href$=".mp4"i],
|
||||
[href$=".webm"i],
|
||||
) {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
#ggrid>a:is(
|
||||
[href$=".mkv"i],
|
||||
[href$=".mp4"i],
|
||||
[href$=".webm"i],
|
||||
):before {
|
||||
content: '▶';
|
||||
opacity: .8;
|
||||
margin: 0;
|
||||
padding: 1em .5em 1em .7em;
|
||||
border-radius: 9em;
|
||||
line-height: 0;
|
||||
color: #fff;
|
||||
text-shadow: none;
|
||||
background: rgba(0, 0, 0, 0.7);
|
||||
left: calc(50% - 1em);
|
||||
top: calc(50% - 1.4em);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* audio */
|
||||
#ggrid>a:is(
|
||||
[href$=".mp3"i],
|
||||
[href$=".ogg"i],
|
||||
[href$=".opus"i],
|
||||
[href$=".flac"i],
|
||||
[href$=".m4a"i],
|
||||
[href$=".aac"i],
|
||||
):before {
|
||||
content: '🎵';
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* image */
|
||||
#ggrid>a:is(
|
||||
[href$=".jpg"i],
|
||||
[href$=".jpeg"i],
|
||||
[href$=".png"i],
|
||||
[href$=".gif"i],
|
||||
[href$=".webp"i],
|
||||
):before {
|
||||
content: '🎨';
|
||||
}
|
||||
@@ -1,506 +0,0 @@
|
||||
// USAGE:
|
||||
// place this file somewhere in the webroot and then
|
||||
// python3 -m copyparty --js-browser /memes/meadup.js
|
||||
//
|
||||
// FEATURES:
|
||||
// * adds an onscreen keyboard for operating a media center remotely,
|
||||
// relies on https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/very-bad-idea.py
|
||||
// * adds an interactive anime girl (if you can find the dependencies)
|
||||
|
||||
var hambagas = [
|
||||
"https://www.youtube.com/watch?v=pFA3KGp4GuU"
|
||||
];
|
||||
|
||||
// keybaord,
|
||||
// onscreen keyboard by @steinuil
|
||||
function initKeybaord(BASE_URL, HAMBAGA, consoleLog, consoleError) {
|
||||
document.querySelector('.keybaord-container').innerHTML = `
|
||||
<div class="keybaord-body">
|
||||
<div class="keybaord-row keybaord-row-1">
|
||||
<div class="keybaord-key" data-keybaord-key="Escape">
|
||||
esc
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F1">
|
||||
F1
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F2">
|
||||
F2
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F3">
|
||||
F3
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F4">
|
||||
F4
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F5">
|
||||
F5
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F6">
|
||||
F6
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F7">
|
||||
F7
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F8">
|
||||
F8
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F9">
|
||||
F9
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F10">
|
||||
F10
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F11">
|
||||
F11
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F12">
|
||||
F12
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Insert">
|
||||
ins
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Delete">
|
||||
del
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-2">
|
||||
<div class="keybaord-key" data-keybaord-key="\`">
|
||||
\`
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="1">
|
||||
1
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="2">
|
||||
2
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="3">
|
||||
3
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="4">
|
||||
4
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="5">
|
||||
5
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="6">
|
||||
6
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="7">
|
||||
7
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="8">
|
||||
8
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="9">
|
||||
9
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="0">
|
||||
0
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="-">
|
||||
-
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="=">
|
||||
=
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-backspace" data-keybaord-key="BackSpace">
|
||||
backspace
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-3">
|
||||
<div class="keybaord-key keybaord-tab" data-keybaord-key="Tab">
|
||||
tab
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="q">
|
||||
q
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="w">
|
||||
w
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="e">
|
||||
e
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="r">
|
||||
r
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="t">
|
||||
t
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="y">
|
||||
y
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="u">
|
||||
u
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="i">
|
||||
i
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="o">
|
||||
o
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="p">
|
||||
p
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="[">
|
||||
[
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="]">
|
||||
]
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-enter" data-keybaord-key="Return">
|
||||
enter
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-4">
|
||||
<div class="keybaord-key keybaord-capslock" data-keybaord-key="HAMBAGA">
|
||||
🍔
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="a">
|
||||
a
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="s">
|
||||
s
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="d">
|
||||
d
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="f">
|
||||
f
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="g">
|
||||
g
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="h">
|
||||
h
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="j">
|
||||
j
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="k">
|
||||
k
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="l">
|
||||
l
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key=";">
|
||||
;
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="'">
|
||||
'
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-backslash" data-keybaord-key="\\">
|
||||
\\
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-5">
|
||||
<div class="keybaord-key keybaord-lshift" data-keybaord-key="Shift_L">
|
||||
shift
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="\\">
|
||||
\\
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="z">
|
||||
z
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="x">
|
||||
x
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="c">
|
||||
c
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="v">
|
||||
v
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="b">
|
||||
b
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="n">
|
||||
n
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="m">
|
||||
m
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key=",">
|
||||
,
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key=".">
|
||||
.
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="/">
|
||||
/
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-rshift" data-keybaord-key="Shift_R">
|
||||
shift
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-6">
|
||||
<div class="keybaord-key keybaord-lctrl" data-keybaord-key="Control_L">
|
||||
ctrl
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-super" data-keybaord-key="Meta_L">
|
||||
win
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-alt" data-keybaord-key="Alt_L">
|
||||
alt
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-spacebar" data-keybaord-key="space">
|
||||
space
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-altgr" data-keybaord-key="Alt_R">
|
||||
altgr
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-what" data-keybaord-key="Menu">
|
||||
menu
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-rctrl" data-keybaord-key="Control_R">
|
||||
ctrl
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row">
|
||||
<div class="keybaord-key" data-keybaord-key="XF86AudioLowerVolume">
|
||||
🔉
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="XF86AudioRaiseVolume">
|
||||
🔊
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Left">
|
||||
⬅️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Down">
|
||||
⬇️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Up">
|
||||
⬆️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Right">
|
||||
➡️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Page_Up">
|
||||
PgUp
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Page_Down">
|
||||
PgDn
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Home">
|
||||
🏠
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="End">
|
||||
End
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
`;
|
||||
|
||||
function arraySample(array) {
|
||||
return array[Math.floor(Math.random() * array.length)];
|
||||
}
|
||||
|
||||
function sendMessage(msg) {
|
||||
return fetch(BASE_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
|
||||
},
|
||||
body: "msg=" + encodeURIComponent(msg),
|
||||
}).then(
|
||||
(r) => r.text(), // so the response body shows up in network tab
|
||||
(err) => consoleError(err)
|
||||
);
|
||||
}
|
||||
const MODIFIER_ON_CLASS = "keybaord-modifier-on";
|
||||
const KEY_DATASET = "data-keybaord-key";
|
||||
const KEY_CLASS = "keybaord-key";
|
||||
|
||||
const modifiers = new Set()
|
||||
|
||||
function toggleModifier(button, key) {
|
||||
button.classList.toggle(MODIFIER_ON_CLASS);
|
||||
if (modifiers.has(key)) {
|
||||
modifiers.delete(key);
|
||||
} else {
|
||||
modifiers.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
function popModifiers() {
|
||||
let modifierString = "";
|
||||
|
||||
modifiers.forEach((mod) => {
|
||||
document.querySelector("[" + KEY_DATASET + "='" + mod + "']")
|
||||
.classList.remove(MODIFIER_ON_CLASS);
|
||||
|
||||
modifierString += mod + "+";
|
||||
});
|
||||
|
||||
modifiers.clear();
|
||||
|
||||
return modifierString;
|
||||
}
|
||||
|
||||
Array.from(document.querySelectorAll("." + KEY_CLASS)).forEach((button) => {
|
||||
const key = button.dataset.keybaordKey;
|
||||
|
||||
button.addEventListener("click", (ev) => {
|
||||
switch (key) {
|
||||
case "HAMBAGA":
|
||||
sendMessage(arraySample(HAMBAGA));
|
||||
break;
|
||||
|
||||
case "Shift_L":
|
||||
case "Shift_R":
|
||||
|
||||
case "Control_L":
|
||||
case "Control_R":
|
||||
|
||||
case "Meta_L":
|
||||
|
||||
case "Alt_L":
|
||||
case "Alt_R":
|
||||
toggleModifier(button, key);
|
||||
break;
|
||||
|
||||
default: {
|
||||
const keyWithModifiers = popModifiers() + key;
|
||||
|
||||
consoleLog(keyWithModifiers);
|
||||
|
||||
sendMessage("key " + keyWithModifiers)
|
||||
.then(() => consoleLog(keyWithModifiers + " OK"));
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// keybaord integration
|
||||
(function () {
|
||||
var o = mknod('div');
|
||||
clmod(o, 'keybaord-container', 1);
|
||||
ebi('op_msg').appendChild(o);
|
||||
|
||||
o = mknod('style');
|
||||
o.innerHTML = `
|
||||
.keybaord-body {
|
||||
display: flex;
|
||||
flex-flow: column nowrap;
|
||||
margin: .6em 0;
|
||||
}
|
||||
|
||||
.keybaord-row {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.keybaord-key {
|
||||
border: 1px solid rgba(128,128,128,0.2);
|
||||
width: 41px;
|
||||
height: 40px;
|
||||
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.keybaord-key:active {
|
||||
background-color: lightgrey;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-modifier-on {
|
||||
background-color: lightblue;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-backspace {
|
||||
width: 82px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-tab {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-enter {
|
||||
width: 69px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-capslock {
|
||||
width: 80px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-backslash {
|
||||
width: 88px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-lshift {
|
||||
width: 65px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-rshift {
|
||||
width: 103px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-lctrl {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-super {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-alt {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-altgr {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-what {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-rctrl {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-spacebar {
|
||||
width: 302px;
|
||||
}
|
||||
`;
|
||||
document.head.appendChild(o);
|
||||
|
||||
initKeybaord('/', hambagas,
|
||||
(msg) => { toast.inf(2, msg.toString()) },
|
||||
(msg) => { toast.err(30, msg.toString()) });
|
||||
})();
|
||||
|
||||
|
||||
// live2d (dumb pointless meme)
|
||||
// dependencies for this part are not tracked in git
|
||||
// so delete this section if you wanna use this file
|
||||
// (or supply your own l2d model and js)
|
||||
(function () {
|
||||
var o = mknod('link');
|
||||
o.setAttribute('rel', 'stylesheet');
|
||||
o.setAttribute('href', "/bad-memes/pio.css");
|
||||
document.head.appendChild(o);
|
||||
|
||||
o = mknod('style');
|
||||
o.innerHTML = '.pio-container{text-shadow:none;z-index:1}';
|
||||
document.head.appendChild(o);
|
||||
|
||||
o = mknod('div');
|
||||
clmod(o, 'pio-container', 1);
|
||||
o.innerHTML = '<div class="pio-action"></div><canvas id="pio" width="280" height="500"></canvas>';
|
||||
document.body.appendChild(o);
|
||||
|
||||
var remaining = 3;
|
||||
for (var a of ['pio', 'l2d', 'fireworks']) {
|
||||
import_js(`/bad-memes/${a}.js`, function () {
|
||||
if (remaining --> 1)
|
||||
return;
|
||||
|
||||
o = mknod('script');
|
||||
o.innerHTML = 'var pio = new Paul_Pio({"selector":[],"mode":"fixed","hidden":false,"content":{"close":"ok bye"},"model":["/bad-memes/sagiri/model.json"]});';
|
||||
document.body.appendChild(o);
|
||||
});
|
||||
}
|
||||
})();
|
||||
@@ -1,46 +0,0 @@
|
||||
<!--
|
||||
NOTE: DEPRECATED; please use the javascript version instead:
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/minimal-up2k.js
|
||||
|
||||
----
|
||||
|
||||
save this as .epilogue.html inside a write-only folder to declutter the UI, makes it look like
|
||||
https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
|
||||
|
||||
only works if you disable the prologue/epilogue sandbox with --no-sb-lg
|
||||
which should probably be combined with --no-dot-ren to prevent damage
|
||||
(`no_sb_lg` can also be set per-volume with volflags)
|
||||
-->
|
||||
|
||||
<style>
|
||||
|
||||
/* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
|
||||
|
||||
#ops, #tree, #path, #wfp, /* main tabs and navigators (tree/breadcrumbs) */
|
||||
|
||||
#u2conf tr:first-child>td[rowspan]:not(#u2btn_cw), /* most of the config options */
|
||||
|
||||
#srch_dz, #srch_zd, /* the filesearch dropzone */
|
||||
|
||||
#u2cards, #u2etaw /* and the upload progress tabs */
|
||||
|
||||
{display: none !important} /* do it! */
|
||||
|
||||
|
||||
|
||||
/* add some margins because now it's weird */
|
||||
.opview {margin-top: 2.5em}
|
||||
#op_up2k {margin-top: 6em}
|
||||
|
||||
/* and embiggen the upload button */
|
||||
#u2conf #u2btn, #u2btn {padding:1.5em 0}
|
||||
|
||||
/* adjust the button area a bit */
|
||||
#u2conf.w, #u2conf.ww {width: 35em !important; margin: 5em auto}
|
||||
|
||||
/* a */
|
||||
#op_up2k {min-height: 0}
|
||||
|
||||
</style>
|
||||
|
||||
<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>
|
||||
@@ -1,59 +0,0 @@
|
||||
/*
|
||||
|
||||
makes the up2k ui REALLY minimal by hiding a bunch of stuff
|
||||
|
||||
almost the same as minimal-up2k.html except this one...:
|
||||
|
||||
-- applies to every write-only folder when used with --js-browser
|
||||
|
||||
-- only applies if javascript is enabled
|
||||
|
||||
-- doesn't hide the total upload ETA display
|
||||
|
||||
-- looks slightly better
|
||||
|
||||
*/
|
||||
|
||||
var u2min = `
|
||||
<style>
|
||||
|
||||
#ops, #path, #tree, #files, #wfp,
|
||||
#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
|
||||
display: none !important;
|
||||
}
|
||||
#u2conf {margin:5em auto 0 auto !important}
|
||||
#u2conf.ww {width:70em}
|
||||
#u2conf.w {width:50em}
|
||||
#u2conf.w .c,
|
||||
#u2conf.w #u2btn_cw {text-align:left}
|
||||
#u2conf.w #u2btn_cw {width:70%}
|
||||
#u2etaw {margin:3em auto}
|
||||
#u2etaw.w {
|
||||
text-align: center;
|
||||
margin: -3.5em auto 5em auto;
|
||||
}
|
||||
#u2etaw.w #u2etas {margin-right:-37em}
|
||||
#u2etaw.w #u2etas.o {margin-top:-2.2em}
|
||||
#u2etaw.ww {margin:-1em auto}
|
||||
#u2etaw.ww #u2etas {padding-left:4em}
|
||||
#u2etas {
|
||||
background: none !important;
|
||||
border: none !important;
|
||||
}
|
||||
#wrap {margin-left:2em !important}
|
||||
.logue {
|
||||
border: none !important;
|
||||
margin: 2em auto !important;
|
||||
}
|
||||
.logue:before {content:'' !important}
|
||||
|
||||
</style>
|
||||
|
||||
<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>
|
||||
`;
|
||||
|
||||
if (!has(perms, 'read')) {
|
||||
var e2 = mknod('div');
|
||||
e2.innerHTML = u2min;
|
||||
ebi('wrap').insertBefore(e2, QS('#wfp'));
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
/* untz untz untz untz */
|
||||
|
||||
(function () {
|
||||
|
||||
var can, ctx, W, H, fft, buf, bars, barw, pv,
|
||||
hue = 0,
|
||||
ibeat = 0,
|
||||
beats = [9001],
|
||||
beats_url = '',
|
||||
uofs = 0,
|
||||
ops = ebi('ops'),
|
||||
raving = false,
|
||||
recalc = 0,
|
||||
cdown = 0,
|
||||
FC = 0.9,
|
||||
css = `<style>
|
||||
|
||||
#fft {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
z-index: -1;
|
||||
}
|
||||
body {
|
||||
box-shadow: inset 0 0 0 white;
|
||||
}
|
||||
#ops>a,
|
||||
#path>a {
|
||||
display: inline-block;
|
||||
}
|
||||
/*
|
||||
body.untz {
|
||||
animation: untz-body 200ms ease-out;
|
||||
}
|
||||
@keyframes untz-body {
|
||||
0% {inset 0 0 20em white}
|
||||
100% {inset 0 0 0 white}
|
||||
}
|
||||
*/
|
||||
:root, html.a, html.b, html.c, html.d, html.e {
|
||||
--row-alt: rgba(48,52,78,0.2);
|
||||
}
|
||||
#files td {
|
||||
background: none;
|
||||
}
|
||||
|
||||
</style>`;
|
||||
|
||||
QS('body').appendChild(mknod('div', null, css));
|
||||
|
||||
function rave_load() {
|
||||
console.log('rave_load');
|
||||
can = mknod('canvas', 'fft');
|
||||
QS('body').appendChild(can);
|
||||
ctx = can.getContext('2d');
|
||||
|
||||
fft = new AnalyserNode(actx, {
|
||||
"fftSize": 2048,
|
||||
"maxDecibels": 0,
|
||||
"smoothingTimeConstant": 0.7,
|
||||
});
|
||||
ibeat = 0;
|
||||
beats = [9001];
|
||||
buf = new Uint8Array(fft.frequencyBinCount);
|
||||
bars = buf.length * FC;
|
||||
afilt.filters.push(fft);
|
||||
if (!raving) {
|
||||
raving = true;
|
||||
raver();
|
||||
}
|
||||
beats_url = mp.au.src.split('?')[0].replace(/(.*\/)(.*)/, '$1.beats/$2.txt');
|
||||
console.log("reading beats from", beats_url);
|
||||
var xhr = new XHR();
|
||||
xhr.open('GET', beats_url, true);
|
||||
xhr.onload = readbeats;
|
||||
xhr.url = beats_url;
|
||||
xhr.send();
|
||||
}
|
||||
|
||||
function rave_unload() {
|
||||
qsr('#fft');
|
||||
can = null;
|
||||
}
|
||||
|
||||
function readbeats() {
|
||||
if (this.url != beats_url)
|
||||
return console.log('old beats??', this.url, beats_url);
|
||||
|
||||
var sbeats = this.responseText.replace(/\r/g, '').split(/\n/g);
|
||||
if (sbeats.length < 3)
|
||||
return;
|
||||
|
||||
beats = [];
|
||||
for (var a = 0; a < sbeats.length; a++)
|
||||
beats.push(parseFloat(sbeats[a]));
|
||||
|
||||
var end = beats.slice(-2),
|
||||
t = end[1],
|
||||
d = t - end[0];
|
||||
|
||||
while (d > 0.1 && t < 1200)
|
||||
beats.push(t += d);
|
||||
}
|
||||
|
||||
function hrand() {
|
||||
return Math.random() - 0.5;
|
||||
}
|
||||
|
||||
function raver() {
|
||||
if (!can) {
|
||||
raving = false;
|
||||
return;
|
||||
}
|
||||
|
||||
requestAnimationFrame(raver);
|
||||
if (!mp || !mp.au || mp.au.paused)
|
||||
return;
|
||||
|
||||
if (--uofs >= 0) {
|
||||
document.body.style.marginLeft = hrand() * uofs + 'px';
|
||||
ebi('tree').style.marginLeft = hrand() * uofs + 'px';
|
||||
for (var a of QSA('#ops>a, #path>a, #pctl>a'))
|
||||
a.style.transform = 'translate(' + hrand() * uofs * 1 + 'px, ' + hrand() * uofs * 0.7 + 'px) rotate(' + Math.random() * uofs * 0.7 + 'deg)'
|
||||
}
|
||||
|
||||
if (--recalc < 0) {
|
||||
recalc = 60;
|
||||
var tree = ebi('tree'),
|
||||
x = tree.style.display == 'none' ? 0 : tree.offsetWidth;
|
||||
|
||||
//W = can.width = window.innerWidth - x;
|
||||
//H = can.height = window.innerHeight;
|
||||
//H = ebi('widget').offsetTop;
|
||||
W = can.width = bars;
|
||||
H = can.height = 512;
|
||||
barw = 1; //parseInt(0.8 + W / bars);
|
||||
can.style.left = x + 'px';
|
||||
can.style.width = (window.innerWidth - x) + 'px';
|
||||
can.style.height = ebi('widget').offsetTop + 'px';
|
||||
}
|
||||
|
||||
//if (--cdown == 1)
|
||||
// clmod(ops, 'untz');
|
||||
|
||||
fft.getByteFrequencyData(buf);
|
||||
|
||||
var imax = 0, vmax = 0;
|
||||
for (var a = 10; a < 50; a++)
|
||||
if (vmax < buf[a]) {
|
||||
vmax = buf[a];
|
||||
imax = a;
|
||||
}
|
||||
|
||||
hue = hue * 0.93 + imax * 0.07;
|
||||
|
||||
ctx.fillStyle = 'rgba(0,0,0,0)';
|
||||
ctx.fillRect(0, 0, W, H);
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
ctx.fillStyle = 'hsla(' + (hue * 2.5) + ',100%,50%,0.7)';
|
||||
|
||||
var x = 0, mul = (H / 256) * 0.5;
|
||||
for (var a = 0; a < buf.length * FC; a++) {
|
||||
var v = buf[a] * mul * (1 + 0.69 * a / buf.length);
|
||||
ctx.fillRect(x, H - v, barw, v);
|
||||
x += barw;
|
||||
}
|
||||
|
||||
var t = mp.au.currentTime + 0.05;
|
||||
|
||||
if (ibeat >= beats.length || beats[ibeat] > t)
|
||||
return;
|
||||
|
||||
while (ibeat < beats.length && beats[ibeat++] < t)
|
||||
continue;
|
||||
|
||||
return untz();
|
||||
|
||||
var cv = 0;
|
||||
for (var a = 0; a < 128; a++)
|
||||
cv += buf[a];
|
||||
|
||||
if (cv - pv > 1000) {
|
||||
console.log(pv, cv, cv - pv);
|
||||
if (cdown < 0) {
|
||||
clmod(ops, 'untz', 1);
|
||||
cdown = 20;
|
||||
}
|
||||
}
|
||||
pv = cv;
|
||||
}
|
||||
|
||||
function untz() {
|
||||
console.log('untz');
|
||||
uofs = 14;
|
||||
document.body.animate([
|
||||
{ boxShadow: 'inset 0 0 1em #f0c' },
|
||||
{ boxShadow: 'inset 0 0 20em #f0c', offset: 0.2 },
|
||||
{ boxShadow: 'inset 0 0 0 #f0c' },
|
||||
], { duration: 200, iterations: 1 });
|
||||
}
|
||||
|
||||
afilt.plugs.push({
|
||||
"en": true,
|
||||
"load": rave_load,
|
||||
"unload": rave_unload
|
||||
});
|
||||
|
||||
})();
|
||||
@@ -1,297 +0,0 @@
|
||||
// way more specific example --
|
||||
// assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
|
||||
// locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
|
||||
//
|
||||
// also tries to find the youtube-id in the embedded metadata
|
||||
//
|
||||
// assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
|
||||
|
||||
function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
|
||||
var passthru = up2k.uc.fsearch;
|
||||
if (passthru)
|
||||
return hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
|
||||
|
||||
a_up2k_namefilter(good_files, nil_files, bad_files, hooks).then(() => { });
|
||||
}
|
||||
|
||||
// ebi('op_up2k').appendChild(mknod('input','unick'));
|
||||
|
||||
function bstrpos(buf, ptn) {
|
||||
var ofs = 0,
|
||||
ch0 = ptn[0],
|
||||
sz = buf.byteLength;
|
||||
|
||||
while (true) {
|
||||
ofs = buf.indexOf(ch0, ofs);
|
||||
if (ofs < 0 || ofs >= sz)
|
||||
return -1;
|
||||
|
||||
for (var a = 1; a < ptn.length; a++)
|
||||
if (buf[ofs + a] !== ptn[a])
|
||||
break;
|
||||
|
||||
if (a === ptn.length)
|
||||
return ofs;
|
||||
|
||||
++ofs;
|
||||
}
|
||||
}
|
||||
|
||||
async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
|
||||
var t0 = Date.now(),
|
||||
yt_ids = new Set(),
|
||||
textdec = new TextDecoder('latin1'),
|
||||
md_ptn = new TextEncoder().encode('youtube.com/watch?v='),
|
||||
file_ids = [], // all IDs found for each good_files
|
||||
md_only = [], // `${id} ${fn}` where ID was only found in metadata
|
||||
mofs = 0,
|
||||
mnchk = 0,
|
||||
mfile = '',
|
||||
myid = localStorage.getItem('ytid_t0');
|
||||
|
||||
if (!myid)
|
||||
localStorage.setItem('ytid_t0', myid = Date.now());
|
||||
|
||||
for (var a = 0; a < good_files.length; a++) {
|
||||
var [fobj, name] = good_files[a],
|
||||
cname = name, // will clobber
|
||||
sz = fobj.size,
|
||||
ids = [],
|
||||
fn_ids = [],
|
||||
md_ids = [],
|
||||
id_ok = false,
|
||||
m;
|
||||
|
||||
// all IDs found in this file
|
||||
file_ids.push(ids);
|
||||
|
||||
// look for ID in filename; reduce the
|
||||
// metadata-scan intensity if the id looks safe
|
||||
m = /[\[(-]([\w-]{11})[\])]?\.(?:mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
|
||||
id_ok = !!m;
|
||||
|
||||
while (true) {
|
||||
// fuzzy catch-all;
|
||||
// some ytdl fork did %(title)-%(id).%(ext) ...
|
||||
m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(cname);
|
||||
if (!m)
|
||||
break;
|
||||
|
||||
cname = cname.replace(m[1], '');
|
||||
yt_ids.add(m[1]);
|
||||
fn_ids.unshift(m[1]);
|
||||
}
|
||||
|
||||
// look for IDs in video metadata,
|
||||
if (/\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name)) {
|
||||
toast.show('inf r', 0, `analyzing file ${a + 1} / ${good_files.length} :\n${name}\n\nhave analysed ${++mnchk} files in ${(Date.now() - t0) / 1000} seconds, ${humantime((good_files.length - (a + 1)) * (((Date.now() - t0) / 1000) / mnchk))} remaining,\n\nbiggest offset so far is ${mofs}, in this file:\n\n${mfile}`);
|
||||
|
||||
// check first and last 128 MiB;
|
||||
// pWxOroN5WCo.mkv @ 6edb98 (6.92M)
|
||||
// Nf-nN1wF5Xo.mp4 @ 4a98034 (74.6M)
|
||||
var chunksz = 1024 * 1024 * 2, // byte
|
||||
aspan = id_ok ? 128 : 512; // MiB
|
||||
|
||||
aspan = parseInt(Math.min(sz / 2, aspan * 1024 * 1024) / chunksz) * chunksz;
|
||||
if (!aspan)
|
||||
aspan = Math.min(sz, chunksz);
|
||||
|
||||
for (var side = 0; side < 2; side++) {
|
||||
var ofs = side ? Math.max(0, sz - aspan) : 0,
|
||||
nchunks = aspan / chunksz;
|
||||
|
||||
for (var chunk = 0; chunk < nchunks; chunk++) {
|
||||
var bchunk = await fobj.slice(ofs, ofs + chunksz + 16).arrayBuffer(),
|
||||
uchunk = new Uint8Array(bchunk, 0, bchunk.byteLength),
|
||||
bofs = bstrpos(uchunk, md_ptn),
|
||||
absofs = Math.min(ofs + bofs, (sz - ofs) + bofs),
|
||||
txt = bofs < 0 ? '' : textdec.decode(uchunk.subarray(bofs)),
|
||||
m;
|
||||
|
||||
//console.log(`side ${ side }, chunk ${ chunk }, ofs ${ ofs }, bchunk ${ bchunk.byteLength }, txt ${ txt.length }`);
|
||||
while (true) {
|
||||
// mkv/webm have [a-z] immediately after url
|
||||
m = /(youtube\.com\/watch\?v=[\w-]{11})/.exec(txt);
|
||||
if (!m)
|
||||
break;
|
||||
|
||||
txt = txt.replace(m[1], '');
|
||||
m = m[1].slice(-11);
|
||||
|
||||
console.log(`found ${m} @${bofs}, ${name} `);
|
||||
yt_ids.add(m);
|
||||
if (!has(fn_ids, m) && !has(md_ids, m)) {
|
||||
md_ids.push(m);
|
||||
md_only.push(`${m} ${name}`);
|
||||
}
|
||||
else
|
||||
// id appears several times; make it preferred
|
||||
md_ids.unshift(m);
|
||||
|
||||
// bail after next iteration
|
||||
chunk = nchunks - 1;
|
||||
side = 9;
|
||||
|
||||
if (mofs < absofs) {
|
||||
mofs = absofs;
|
||||
mfile = name;
|
||||
}
|
||||
}
|
||||
ofs += chunksz;
|
||||
if (ofs >= sz)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (var yi of md_ids)
|
||||
ids.push(yi);
|
||||
|
||||
for (var yi of fn_ids)
|
||||
if (!has(ids, yi))
|
||||
ids.push(yi);
|
||||
}
|
||||
|
||||
if (md_only.length)
|
||||
console.log('recovered the following youtube-IDs by inspecting metadata:\n\n' + md_only.join('\n'));
|
||||
else if (yt_ids.size)
|
||||
console.log('did not discover any additional youtube-IDs by inspecting metadata; all the IDs also existed in the filenames');
|
||||
else
|
||||
console.log('failed to find any youtube-IDs at all, sorry');
|
||||
|
||||
if (false) {
|
||||
var msg = `finished analysing ${mnchk} files in ${(Date.now() - t0) / 1000} seconds,\n\nbiggest offset was ${mofs} in this file:\n\n${mfile}`,
|
||||
mfun = function () { toast.ok(0, msg); };
|
||||
|
||||
mfun();
|
||||
setTimeout(mfun, 200);
|
||||
|
||||
return hooks[0]([], [], [], hooks.slice(1));
|
||||
}
|
||||
|
||||
var el = ebi('unick'), unick = el ? el.value : '';
|
||||
if (unick) {
|
||||
console.log(`sending uploader nickname [${unick}]`);
|
||||
fetch(document.location, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' },
|
||||
body: 'msg=' + encodeURIComponent(unick)
|
||||
});
|
||||
}
|
||||
|
||||
toast.inf(5, `running query for ${yt_ids.size} youtube-IDs...`);
|
||||
|
||||
var xhr = new XHR();
|
||||
xhr.open('POST', '/ytq', true);
|
||||
xhr.setRequestHeader('Content-Type', 'text/plain');
|
||||
xhr.onload = xhr.onerror = function () {
|
||||
if (this.status != 200)
|
||||
return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
|
||||
|
||||
process_id_list(this.responseText);
|
||||
};
|
||||
xhr.send(Array.from(yt_ids).join('\n'));
|
||||
|
||||
function process_id_list(txt) {
|
||||
var wanted_ids = new Set(txt.trim().split('\n')),
|
||||
name_id = {},
|
||||
wanted_names = new Set(), // basenames with a wanted ID -- not including relpath
|
||||
wanted_names_scoped = {}, // basenames with a wanted ID -> list of dirs to search under
|
||||
wanted_files = new Set(); // filedrops
|
||||
|
||||
for (var a = 0; a < good_files.length; a++) {
|
||||
var name = good_files[a][1];
|
||||
for (var b = 0; b < file_ids[a].length; b++)
|
||||
if (wanted_ids.has(file_ids[a][b])) {
|
||||
// let the next stage handle this to prevent dupes
|
||||
//wanted_files.add(good_files[a]);
|
||||
|
||||
var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
|
||||
if (!m)
|
||||
continue;
|
||||
|
||||
var [rd, fn] = vsplit(m[1]);
|
||||
|
||||
if (fn in wanted_names_scoped)
|
||||
wanted_names_scoped[fn].push(rd);
|
||||
else
|
||||
wanted_names_scoped[fn] = [rd];
|
||||
|
||||
wanted_names.add(fn);
|
||||
name_id[m[1]] = file_ids[a][b];
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// add all files with the same basename as each explicitly wanted file
|
||||
// (infojson/chatlog/etc when ID was discovered from metadata)
|
||||
for (var a = 0; a < good_files.length; a++) {
|
||||
var [rd, name] = vsplit(good_files[a][1]);
|
||||
for (var b = 0; b < 3; b++) {
|
||||
name = name.replace(/\.[^\.]+$/, '');
|
||||
if (!wanted_names.has(name))
|
||||
continue;
|
||||
|
||||
var vid_fp = false;
|
||||
for (var c of wanted_names_scoped[name])
|
||||
if (rd.startsWith(c))
|
||||
vid_fp = c + name;
|
||||
|
||||
if (!vid_fp)
|
||||
continue;
|
||||
|
||||
var subdir = name_id[vid_fp];
|
||||
subdir = `v${subdir.slice(0, 1)}/${subdir}-${myid}`;
|
||||
var newpath = subdir + '/' + good_files[a][1].split(/\//g).pop();
|
||||
|
||||
// check if this file is a dupe
|
||||
for (var c of good_files)
|
||||
if (c[1] == newpath)
|
||||
newpath = null;
|
||||
|
||||
if (!newpath)
|
||||
break;
|
||||
|
||||
good_files[a][1] = newpath;
|
||||
wanted_files.add(good_files[a]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function upload_filtered() {
|
||||
if (!wanted_files.size)
|
||||
return modal.alert('Good news -- turns out we already have all those.\n\nBut thank you for checking in!');
|
||||
|
||||
hooks[0](Array.from(wanted_files), nil_files, bad_files, hooks.slice(1));
|
||||
}
|
||||
|
||||
function upload_all() {
|
||||
hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
|
||||
}
|
||||
|
||||
var n_skip = good_files.length - wanted_files.size,
|
||||
msg = `you added ${good_files.length} files; ${good_files.length == n_skip ? 'all' : n_skip} of them were skipped --\neither because we already have them,\nor because there is no youtube-ID in your filenames.\n\n<code>OK</code> / <code>Enter</code> = continue uploading just the ${wanted_files.size} files we definitely need\n\n<code>Cancel</code> / <code>ESC</code> = override the filter; upload ALL the files you added`;
|
||||
|
||||
if (!n_skip)
|
||||
upload_filtered();
|
||||
else
|
||||
modal.confirm(msg, upload_filtered, upload_all);
|
||||
};
|
||||
}
|
||||
|
||||
up2k_hooks.push(function () {
|
||||
up2k.gotallfiles.unshift(up2k_namefilter);
|
||||
});
|
||||
|
||||
// persist/restore nickname field if present
|
||||
setInterval(function () {
|
||||
var o = ebi('unick');
|
||||
if (!o || document.activeElement == o)
|
||||
return;
|
||||
|
||||
o.oninput = function () {
|
||||
localStorage.setItem('unick', o.value);
|
||||
};
|
||||
o.value = localStorage.getItem('unick') || '';
|
||||
}, 1000);
|
||||
@@ -1,45 +0,0 @@
|
||||
// hooks into up2k
|
||||
|
||||
function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
|
||||
// is called when stuff is dropped into the browser,
|
||||
// after iterating through the directory tree and discovering all files,
|
||||
// before the upload confirmation dialogue is shown
|
||||
|
||||
// good_files will successfully upload
|
||||
// nil_files are empty files and will show an alert in the final hook
|
||||
// bad_files are unreadable and cannot be uploaded
|
||||
var file_lists = [good_files, nil_files, bad_files];
|
||||
|
||||
// build a list of filenames
|
||||
var filenames = [];
|
||||
for (var lst of file_lists)
|
||||
for (var ent of lst)
|
||||
filenames.push(ent[1]);
|
||||
|
||||
toast.inf(5, "running database query...");
|
||||
|
||||
// simulate delay while passing the list to some api for checking
|
||||
setTimeout(function () {
|
||||
|
||||
// only keep webm files as an example
|
||||
var new_lists = [];
|
||||
for (var lst of file_lists) {
|
||||
var keep = [];
|
||||
new_lists.push(keep);
|
||||
|
||||
for (var ent of lst)
|
||||
if (/\.webm$/.test(ent[1]))
|
||||
keep.push(ent);
|
||||
}
|
||||
|
||||
// finally, call the next hook in the chain
|
||||
[good_files, nil_files, bad_files] = new_lists;
|
||||
hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
|
||||
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
// register
|
||||
up2k_hooks.push(function () {
|
||||
up2k.gotallfiles.unshift(up2k_namefilter);
|
||||
});
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# PROVIDE: copyparty
|
||||
# REQUIRE: networking
|
||||
# KEYWORD:
|
||||
|
||||
. /etc/rc.subr
|
||||
|
||||
name="copyparty"
|
||||
rcvar="copyparty_enable"
|
||||
copyparty_user="copyparty"
|
||||
copyparty_args="-e2dsa -v /storage:/storage:r" # change as you see fit
|
||||
copyparty_command="/usr/local/bin/python3.8 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
|
||||
pidfile="/var/run/copyparty/${name}.pid"
|
||||
command="/usr/sbin/daemon"
|
||||
command_args="-P ${pidfile} -r -f ${copyparty_command}"
|
||||
|
||||
stop_postcmd="copyparty_shutdown"
|
||||
|
||||
copyparty_shutdown()
|
||||
{
|
||||
if [ -e "${pidfile}" ]; then
|
||||
echo "Stopping supervising daemon."
|
||||
kill -s TERM `cat ${pidfile}`
|
||||
fi
|
||||
}
|
||||
|
||||
load_rc_config $name
|
||||
: ${copyparty_enable:=no}
|
||||
|
||||
run_rc_command "$1"
|
||||
@@ -1,23 +0,0 @@
|
||||
# systemd service which generates a new TLS certificate on each boot,
|
||||
# that way the one-year expiry time won't cause any issues --
|
||||
# just have everyone trust the ca.pem once every 10 years
|
||||
#
|
||||
# assumptions/placeholder values:
|
||||
# * this script and copyparty runs as user "cpp"
|
||||
# * copyparty repo is at ~cpp/dev/copyparty
|
||||
# * CA is named partylan
|
||||
# * server IPs = 10.1.2.3 and 192.168.123.1
|
||||
# * server hostname = party.lan
|
||||
|
||||
[Unit]
|
||||
Description=copyparty certificate generator
|
||||
Before=copyparty.service
|
||||
|
||||
[Service]
|
||||
User=cpp
|
||||
Type=oneshot
|
||||
SyslogIdentifier=cpp-cert
|
||||
ExecStart=/bin/bash -c 'cd ~/dev/copyparty/contrib && ./cfssl.sh partylan 10.1.2.3,192.168.123.1,party.lan y'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -2,64 +2,18 @@
|
||||
# and share '/mnt' with anonymous read+write
|
||||
#
|
||||
# installation:
|
||||
# wget https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py -O /usr/local/bin/copyparty-sfx.py
|
||||
# cp -pv copyparty.service /etc/systemd/system/
|
||||
# restorecon -vr /etc/systemd/system/copyparty.service # on fedora/rhel
|
||||
# firewall-cmd --permanent --add-port={80,443,3923}/tcp # --zone=libvirt
|
||||
# firewall-cmd --reload
|
||||
# systemctl daemon-reload && systemctl enable --now copyparty
|
||||
#
|
||||
# if it fails to start, first check this: systemctl status copyparty
|
||||
# then try starting it while viewing logs: journalctl -fan 100
|
||||
# cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
|
||||
#
|
||||
# you may want to:
|
||||
# change "User=cpp" and "/home/cpp/" to another user
|
||||
# remove the nft lines to only listen on port 3923
|
||||
# and in the ExecStart= line:
|
||||
# change '/usr/bin/python3' to another interpreter
|
||||
# change '/mnt::rw' to another location or permission-set
|
||||
# add '-q' to disable logging on busy servers
|
||||
# add '-i 127.0.0.1' to only allow local connections
|
||||
# add '-e2dsa' to enable filesystem scanning + indexing
|
||||
# add '-e2ts' to enable metadata indexing
|
||||
#
|
||||
# with `Type=notify`, copyparty will signal systemd when it is ready to
|
||||
# accept connections; correctly delaying units depending on copyparty.
|
||||
# But note that journalctl will get the timestamps wrong due to
|
||||
# python disabling line-buffering, so messages are out-of-order:
|
||||
# https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
|
||||
#
|
||||
# unless you add -q to disable logging, you may want to remove the
|
||||
# following line to allow buffering (slightly better performance):
|
||||
# Environment=PYTHONUNBUFFERED=x
|
||||
#
|
||||
# keep ExecStartPre before ExecStart, at least on rhel8
|
||||
# change '/usr/bin/python' to another interpreter
|
||||
# change '/mnt::a' to another location or permission-set
|
||||
|
||||
[Unit]
|
||||
Description=copyparty file server
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
SyslogIdentifier=copyparty
|
||||
Environment=PYTHONUNBUFFERED=x
|
||||
ExecReload=/bin/kill -s USR1 $MAINPID
|
||||
|
||||
# user to run as + where the TLS certificate is (if any)
|
||||
User=cpp
|
||||
Environment=XDG_CONFIG_HOME=/home/cpp/.config
|
||||
|
||||
# OPTIONAL: setup forwarding from ports 80 and 443 to port 3923
|
||||
ExecStartPre=+/bin/bash -c 'nft -n -a list table nat | awk "/ to :3923 /{print\$NF}" | xargs -rL1 nft delete rule nat prerouting handle; true'
|
||||
ExecStartPre=+nft add table ip nat
|
||||
ExecStartPre=+nft -- add chain ip nat prerouting { type nat hook prerouting priority -100 \; }
|
||||
ExecStartPre=+nft add rule ip nat prerouting tcp dport 80 redirect to :3923
|
||||
ExecStartPre=+nft add rule ip nat prerouting tcp dport 443 redirect to :3923
|
||||
|
||||
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
|
||||
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
# copyparty settings
|
||||
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -e2d -v /mnt::rw
|
||||
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
|
||||
ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# this will start `/usr/local/bin/copyparty-sfx.py`
|
||||
# in a chroot, preventing accidental access elsewhere
|
||||
# and share '/mnt' with anonymous read+write
|
||||
#
|
||||
# installation:
|
||||
# 1) put copyparty-sfx.py and prisonparty.sh in /usr/local/bin
|
||||
# 2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
|
||||
#
|
||||
# expose additional filesystem locations to copyparty
|
||||
# by listing them between the last `1000` and `--`
|
||||
#
|
||||
# `1000 1000` = what user to run copyparty as
|
||||
#
|
||||
# you may want to:
|
||||
# change '/mnt::rw' to another location or permission-set
|
||||
# (remember to change the '/mnt' chroot arg too)
|
||||
#
|
||||
# unless you add -q to disable logging, you may want to remove the
|
||||
# following line to allow buffering (slightly better performance):
|
||||
# Environment=PYTHONUNBUFFERED=x
|
||||
|
||||
[Unit]
|
||||
Description=copyparty file server
|
||||
|
||||
[Service]
|
||||
SyslogIdentifier=prisonparty
|
||||
Environment=PYTHONUNBUFFERED=x
|
||||
WorkingDirectory=/var/lib/copyparty-jail
|
||||
ExecReload=/bin/kill -s USR1 $MAINPID
|
||||
|
||||
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
|
||||
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
# run copyparty
|
||||
ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
|
||||
/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,45 +0,0 @@
|
||||
@echo off
|
||||
rem removes the 47.6 MiB filesize limit when downloading from webdav
|
||||
rem + optionally allows/enables password-auth over plaintext http
|
||||
rem + optionally helps disable wpad, removing the 10sec latency
|
||||
|
||||
net session >nul 2>&1
|
||||
if %errorlevel% neq 0 (
|
||||
echo sorry, you must run this as administrator
|
||||
pause
|
||||
exit /b
|
||||
)
|
||||
|
||||
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v FileSizeLimitInBytes /t REG_DWORD /d 0xffffffff /f
|
||||
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters /v FsCtlRequestTimeoutInSec /t REG_DWORD /d 0xffffffff /f
|
||||
|
||||
echo(
|
||||
echo OK;
|
||||
echo allow webdav basic-auth over plaintext http?
|
||||
echo Y: login works, but the password will be visible in wireshark etc
|
||||
echo N: login will NOT work unless you use https and valid certificates
|
||||
choice
|
||||
if %errorlevel% equ 1 (
|
||||
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v BasicAuthLevel /t REG_DWORD /d 0x2 /f
|
||||
rem default is 1 (require tls)
|
||||
)
|
||||
|
||||
echo(
|
||||
echo OK;
|
||||
echo do you want to disable wpad?
|
||||
echo can give a HUGE speed boost depending on network settings
|
||||
choice
|
||||
if %errorlevel% equ 1 (
|
||||
echo(
|
||||
echo i'm about to open the [Connections] tab in [Internet Properties] for you;
|
||||
echo please click [LAN settings] and disable [Automatically detect settings]
|
||||
echo(
|
||||
pause
|
||||
control inetcpl.cpl,,4
|
||||
)
|
||||
|
||||
net stop webclient
|
||||
net start webclient
|
||||
echo(
|
||||
echo OK; all done
|
||||
pause
|
||||
@@ -1,53 +1,50 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
|
||||
try:
|
||||
from typing import TYPE_CHECKING
|
||||
except:
|
||||
TYPE_CHECKING = False
|
||||
|
||||
if True:
|
||||
from typing import Any, Callable
|
||||
|
||||
PY2 = sys.version_info < (3,)
|
||||
if not PY2:
|
||||
unicode: Callable[[Any], str] = str
|
||||
else:
|
||||
PY2 = sys.version_info[0] == 2
|
||||
if PY2:
|
||||
sys.dont_write_bytecode = True
|
||||
unicode = unicode # noqa: F821 # pylint: disable=undefined-variable,self-assigning-variable
|
||||
|
||||
WINDOWS: Any = (
|
||||
[int(x) for x in platform.version().split(".")]
|
||||
if platform.system() == "Windows"
|
||||
else False
|
||||
)
|
||||
WINDOWS = False
|
||||
if platform.system() == "Windows":
|
||||
WINDOWS = [int(x) for x in platform.version().split(".")]
|
||||
|
||||
VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
|
||||
# introduced in anniversary update
|
||||
|
||||
ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"]
|
||||
ANYWIN = WINDOWS or sys.platform in ["msys"]
|
||||
|
||||
MACOS = platform.system() == "Darwin"
|
||||
|
||||
EXE = bool(getattr(sys, "frozen", False))
|
||||
|
||||
try:
|
||||
CORES = len(os.sched_getaffinity(0))
|
||||
except:
|
||||
CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
|
||||
|
||||
|
||||
class EnvParams(object):
|
||||
def __init__(self) -> None:
|
||||
def __init__(self):
|
||||
self.t0 = time.time()
|
||||
self.mod = ""
|
||||
self.cfg = ""
|
||||
self.ox = getattr(sys, "oxidized", None)
|
||||
self.mod = os.path.dirname(os.path.realpath(__file__))
|
||||
if self.mod.endswith("__init__"):
|
||||
self.mod = os.path.dirname(self.mod)
|
||||
|
||||
if sys.platform == "win32":
|
||||
self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
|
||||
elif sys.platform == "darwin":
|
||||
self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
|
||||
else:
|
||||
self.cfg = os.path.normpath(
|
||||
os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
|
||||
+ "/copyparty"
|
||||
)
|
||||
|
||||
self.cfg = self.cfg.replace("\\", "/")
|
||||
try:
|
||||
os.makedirs(self.cfg)
|
||||
except:
|
||||
if not os.path.isdir(self.cfg):
|
||||
raise
|
||||
|
||||
|
||||
E = EnvParams()
|
||||
|
||||
1197
copyparty/__main__.py
Executable file → Normal file
1197
copyparty/__main__.py
Executable file → Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,8 @@
|
||||
# coding: utf-8
|
||||
|
||||
VERSION = (1, 6, 15)
|
||||
CODENAME = "cors k"
|
||||
BUILD_DT = (2023, 4, 26)
|
||||
VERSION = (0, 11, 29)
|
||||
CODENAME = "the grid"
|
||||
BUILD_DT = (2021, 6, 30)
|
||||
|
||||
S_VERSION = ".".join(map(str, VERSION))
|
||||
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
|
||||
|
||||
1792
copyparty/authsrv.py
1792
copyparty/authsrv.py
File diff suppressed because it is too large
Load Diff
@@ -1,81 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
from ..util import SYMTIME, fsdec, fsenc
|
||||
from . import path as path
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Any, Optional
|
||||
|
||||
_ = (path,)
|
||||
__all__ = ["path"]
|
||||
|
||||
# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
|
||||
# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
|
||||
|
||||
|
||||
def chmod(p: str, mode: int) -> None:
|
||||
return os.chmod(fsenc(p), mode)
|
||||
|
||||
|
||||
def listdir(p: str = ".") -> list[str]:
|
||||
return [fsdec(x) for x in os.listdir(fsenc(p))]
|
||||
|
||||
|
||||
def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> bool:
|
||||
bname = fsenc(name)
|
||||
try:
|
||||
os.makedirs(bname, mode)
|
||||
return True
|
||||
except:
|
||||
if not exist_ok or not os.path.isdir(bname):
|
||||
raise
|
||||
return False
|
||||
|
||||
|
||||
def mkdir(p: str, mode: int = 0o755) -> None:
|
||||
return os.mkdir(fsenc(p), mode)
|
||||
|
||||
|
||||
def open(p: str, *a, **ka) -> int:
|
||||
return os.open(fsenc(p), *a, **ka)
|
||||
|
||||
|
||||
def rename(src: str, dst: str) -> None:
|
||||
return os.rename(fsenc(src), fsenc(dst))
|
||||
|
||||
|
||||
def replace(src: str, dst: str) -> None:
|
||||
return os.replace(fsenc(src), fsenc(dst))
|
||||
|
||||
|
||||
def rmdir(p: str) -> None:
|
||||
return os.rmdir(fsenc(p))
|
||||
|
||||
|
||||
def stat(p: str) -> os.stat_result:
|
||||
return os.stat(fsenc(p))
|
||||
|
||||
|
||||
def unlink(p: str) -> None:
|
||||
return os.unlink(fsenc(p))
|
||||
|
||||
|
||||
def utime(
|
||||
p: str, times: Optional[tuple[float, float]] = None, follow_symlinks: bool = True
|
||||
) -> None:
|
||||
if SYMTIME:
|
||||
return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks)
|
||||
else:
|
||||
return os.utime(fsenc(p), times)
|
||||
|
||||
|
||||
if hasattr(os, "lstat"):
|
||||
|
||||
def lstat(p: str) -> os.stat_result:
|
||||
return os.lstat(fsenc(p))
|
||||
|
||||
else:
|
||||
lstat = stat
|
||||
@@ -1,45 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
from ..util import SYMTIME, fsdec, fsenc
|
||||
|
||||
|
||||
def abspath(p: str) -> str:
|
||||
return fsdec(os.path.abspath(fsenc(p)))
|
||||
|
||||
|
||||
def exists(p: str) -> bool:
|
||||
return os.path.exists(fsenc(p))
|
||||
|
||||
|
||||
def getmtime(p: str, follow_symlinks: bool = True) -> float:
|
||||
if not follow_symlinks and SYMTIME:
|
||||
return os.lstat(fsenc(p)).st_mtime
|
||||
else:
|
||||
return os.path.getmtime(fsenc(p))
|
||||
|
||||
|
||||
def getsize(p: str) -> int:
|
||||
return os.path.getsize(fsenc(p))
|
||||
|
||||
|
||||
def isfile(p: str) -> bool:
|
||||
return os.path.isfile(fsenc(p))
|
||||
|
||||
|
||||
def isdir(p: str) -> bool:
|
||||
return os.path.isdir(fsenc(p))
|
||||
|
||||
|
||||
def islink(p: str) -> bool:
|
||||
return os.path.islink(fsenc(p))
|
||||
|
||||
|
||||
def lexists(p: str) -> bool:
|
||||
return os.path.lexists(fsenc(p))
|
||||
|
||||
|
||||
def realpath(p: str) -> str:
|
||||
return fsdec(os.path.realpath(fsenc(p)))
|
||||
@@ -1,64 +1,70 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import threading
|
||||
|
||||
import queue
|
||||
|
||||
from .__init__ import CORES, TYPE_CHECKING
|
||||
from .broker_mpw import MpWorker
|
||||
from .__init__ import PY2, WINDOWS, VT100
|
||||
from .broker_util import try_exec
|
||||
from .util import Daemon, mp
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Any
|
||||
from .broker_mpw import MpWorker
|
||||
from .util import mp
|
||||
|
||||
|
||||
class MProcess(mp.Process):
|
||||
def __init__(
|
||||
self,
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]],
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]],
|
||||
target: Any,
|
||||
args: Any,
|
||||
) -> None:
|
||||
super(MProcess, self).__init__(target=target, args=args)
|
||||
self.q_pend = q_pend
|
||||
self.q_yield = q_yield
|
||||
if PY2 and not WINDOWS:
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from StringIO import StringIO as MemesIO # pylint: disable=import-error
|
||||
|
||||
|
||||
class BrokerMp(object):
|
||||
"""external api; manages MpWorkers"""
|
||||
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
def __init__(self, hub):
|
||||
self.hub = hub
|
||||
self.log = hub.log
|
||||
self.args = hub.args
|
||||
|
||||
self.procs = []
|
||||
self.retpend = {}
|
||||
self.retpend_mutex = threading.Lock()
|
||||
self.mutex = threading.Lock()
|
||||
|
||||
self.num_workers = self.args.j or CORES
|
||||
self.log("broker", "booting {} subprocesses".format(self.num_workers))
|
||||
for n in range(1, self.num_workers + 1):
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
|
||||
cores = self.args.j
|
||||
if not cores:
|
||||
cores = mp.cpu_count()
|
||||
|
||||
self.log("broker", "booting {} subprocesses".format(cores))
|
||||
for n in range(cores):
|
||||
q_pend = mp.Queue(1)
|
||||
q_yield = mp.Queue(64)
|
||||
|
||||
proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
|
||||
proc.q_pend = q_pend
|
||||
proc.q_yield = q_yield
|
||||
proc.nid = n
|
||||
proc.clients = {}
|
||||
proc.workload = 0
|
||||
|
||||
thr = threading.Thread(
|
||||
target=self.collector, args=(proc,), name="mp-collector"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
|
||||
Daemon(self.collector, "mp-sink-{}".format(n), (proc,))
|
||||
self.procs.append(proc)
|
||||
proc.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if not self.args.q:
|
||||
thr = threading.Thread(
|
||||
target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def shutdown(self):
|
||||
self.log("broker", "shutting down")
|
||||
for n, proc in enumerate(self.procs):
|
||||
thr = threading.Thread(
|
||||
target=proc.q_pend.put((0, "shutdown", [])),
|
||||
target=proc.q_pend.put([0, "shutdown", []]),
|
||||
name="mp-shutdown-{}-{}".format(n, len(self.procs)),
|
||||
)
|
||||
thr.start()
|
||||
@@ -74,12 +80,7 @@ class BrokerMp(object):
|
||||
|
||||
procs.pop()
|
||||
|
||||
def reload(self) -> None:
|
||||
self.log("broker", "reloading")
|
||||
for _, proc in enumerate(self.procs):
|
||||
proc.q_pend.put((0, "reload", []))
|
||||
|
||||
def collector(self, proc: MProcess) -> None:
|
||||
def collector(self, proc):
|
||||
"""receive message from hub in other process"""
|
||||
while True:
|
||||
msg = proc.q_yield.get()
|
||||
@@ -88,41 +89,77 @@ class BrokerMp(object):
|
||||
if dest == "log":
|
||||
self.log(*args)
|
||||
|
||||
elif dest == "workload":
|
||||
with self.mutex:
|
||||
proc.workload = args[0]
|
||||
|
||||
elif dest == "httpdrop":
|
||||
addr = args[0]
|
||||
|
||||
with self.mutex:
|
||||
del proc.clients[addr]
|
||||
if not proc.clients:
|
||||
proc.workload = 0
|
||||
|
||||
self.hub.tcpsrv.num_clients.add(-1)
|
||||
|
||||
elif dest == "retq":
|
||||
# response from previous ipc call
|
||||
raise Exception("invalid broker_mp usage")
|
||||
with self.retpend_mutex:
|
||||
retq = self.retpend.pop(retq_id)
|
||||
|
||||
retq.put(args)
|
||||
|
||||
else:
|
||||
# new ipc invoking managed service in hub
|
||||
try:
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
# TODO will deadlock if dest performs another ipc
|
||||
rv = try_exec(retq_id, obj, *args)
|
||||
except:
|
||||
rv = ["exception", "stack", traceback.format_exc()]
|
||||
# TODO will deadlock if dest performs another ipc
|
||||
rv = try_exec(retq_id, obj, *args)
|
||||
|
||||
if retq_id:
|
||||
proc.q_pend.put((retq_id, "retq", rv))
|
||||
proc.q_pend.put([retq_id, "retq", rv])
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
def put(self, want_retval, dest, *args):
|
||||
"""
|
||||
send message to non-hub component in other process,
|
||||
returns a Queue object which eventually contains the response if want_retval
|
||||
(not-impl here since nothing uses it yet)
|
||||
"""
|
||||
if dest == "listen":
|
||||
for p in self.procs:
|
||||
p.q_pend.put((0, dest, [args[0], len(self.procs)]))
|
||||
if dest == "httpconn":
|
||||
sck, addr = args
|
||||
sck2 = sck
|
||||
if PY2:
|
||||
buf = MemesIO()
|
||||
ForkingPickler(buf).dump(sck)
|
||||
sck2 = buf.getvalue()
|
||||
|
||||
elif dest == "set_netdevs":
|
||||
for p in self.procs:
|
||||
p.q_pend.put((0, dest, list(args)))
|
||||
proc = sorted(self.procs, key=lambda x: x.workload)[0]
|
||||
proc.q_pend.put([0, dest, [sck2, addr]])
|
||||
|
||||
elif dest == "cb_httpsrv_up":
|
||||
self.hub.cb_httpsrv_up()
|
||||
with self.mutex:
|
||||
proc.clients[addr] = 50
|
||||
proc.workload += 50
|
||||
|
||||
else:
|
||||
raise Exception("what is " + str(dest))
|
||||
|
||||
def debug_load_balancer(self):
|
||||
fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
|
||||
if not VT100:
|
||||
fmt = "({}{:4})"
|
||||
|
||||
last = ""
|
||||
while self.procs:
|
||||
msg = ""
|
||||
for proc in self.procs:
|
||||
msg += fmt.format(len(proc.clients), proc.workload)
|
||||
|
||||
if msg != last:
|
||||
last = msg
|
||||
with self.hub.log_mutex:
|
||||
print(msg)
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
@@ -1,84 +1,68 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
from copyparty.authsrv import AuthSrv
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import threading
|
||||
|
||||
import queue
|
||||
|
||||
from .__init__ import ANYWIN
|
||||
from .authsrv import AuthSrv
|
||||
from .broker_util import BrokerCli, ExceptionalQueue
|
||||
from .__init__ import PY2, WINDOWS
|
||||
from .broker_util import ExceptionalQueue
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import FAKE_MP, Daemon, HMaccas
|
||||
from .util import FAKE_MP
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from types import FrameType
|
||||
|
||||
from typing import Any, Optional, Union
|
||||
if PY2 and not WINDOWS:
|
||||
import pickle # nosec
|
||||
|
||||
|
||||
class MpWorker(BrokerCli):
|
||||
class MpWorker(object):
|
||||
"""one single mp instance"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]],
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]],
|
||||
args: argparse.Namespace,
|
||||
n: int,
|
||||
) -> None:
|
||||
super(MpWorker, self).__init__()
|
||||
|
||||
def __init__(self, q_pend, q_yield, args, n):
|
||||
self.q_pend = q_pend
|
||||
self.q_yield = q_yield
|
||||
self.args = args
|
||||
self.n = n
|
||||
|
||||
self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
|
||||
|
||||
self.retpend: dict[int, Any] = {}
|
||||
self.retpend = {}
|
||||
self.retpend_mutex = threading.Lock()
|
||||
self.mutex = threading.Lock()
|
||||
self.workload_thr_alive = False
|
||||
|
||||
# we inherited signal_handler from parent,
|
||||
# replace it with something harmless
|
||||
if not FAKE_MP:
|
||||
sigs = [signal.SIGINT, signal.SIGTERM]
|
||||
if not ANYWIN:
|
||||
sigs.append(signal.SIGUSR1)
|
||||
|
||||
for sig in sigs:
|
||||
signal.signal(sig, self.signal_handler)
|
||||
signal.signal(signal.SIGINT, self.signal_handler)
|
||||
|
||||
# starting to look like a good idea
|
||||
self.asrv = AuthSrv(args, None, False)
|
||||
|
||||
# instantiate all services here (TODO: inheritance?)
|
||||
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
|
||||
self.httpsrv = HttpSrv(self, n)
|
||||
self.httpsrv = HttpSrv(self, True)
|
||||
self.httpsrv.disconnect_func = self.httpdrop
|
||||
|
||||
# on winxp and some other platforms,
|
||||
# use thr.join() to block all signals
|
||||
Daemon(self.main, "mpw-main").join()
|
||||
thr = threading.Thread(target=self.main, name="mpw-main")
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
thr.join()
|
||||
|
||||
def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
|
||||
def signal_handler(self, signal, frame):
|
||||
# print('k')
|
||||
pass
|
||||
|
||||
def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.q_yield.put((0, "log", [src, msg, c]))
|
||||
def log(self, src, msg, c=0):
|
||||
self.q_yield.put([0, "log", [src, msg, c]])
|
||||
|
||||
def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
pass
|
||||
|
||||
def logw(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def logw(self, msg, c=0):
|
||||
self.log("mp{}".format(self.n), msg, c)
|
||||
|
||||
def main(self) -> None:
|
||||
def httpdrop(self, addr):
|
||||
self.q_yield.put([0, "httpdrop", [addr]])
|
||||
|
||||
def main(self):
|
||||
while True:
|
||||
retq_id, dest, args = self.q_pend.get()
|
||||
|
||||
@@ -89,16 +73,24 @@ class MpWorker(BrokerCli):
|
||||
sys.exit(0)
|
||||
return
|
||||
|
||||
elif dest == "reload":
|
||||
self.logw("mpw.asrv reloading")
|
||||
self.asrv.reload()
|
||||
self.logw("mpw.asrv reloaded")
|
||||
elif dest == "httpconn":
|
||||
sck, addr = args
|
||||
if PY2:
|
||||
sck = pickle.loads(sck) # nosec
|
||||
|
||||
elif dest == "listen":
|
||||
self.httpsrv.listen(args[0], args[1])
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
|
||||
|
||||
elif dest == "set_netdevs":
|
||||
self.httpsrv.set_netdevs(args[0])
|
||||
self.httpsrv.accept(sck, addr)
|
||||
|
||||
with self.mutex:
|
||||
if not self.workload_thr_alive:
|
||||
self.workload_thr_alive = True
|
||||
thr = threading.Thread(
|
||||
target=self.thr_workload, name="mpw-workload"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
elif dest == "retq":
|
||||
# response from previous ipc call
|
||||
@@ -110,14 +102,28 @@ class MpWorker(BrokerCli):
|
||||
else:
|
||||
raise Exception("what is " + str(dest))
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
retq = ExceptionalQueue(1)
|
||||
retq_id = id(retq)
|
||||
with self.retpend_mutex:
|
||||
self.retpend[retq_id] = retq
|
||||
def put(self, want_retval, dest, *args):
|
||||
if want_retval:
|
||||
retq = ExceptionalQueue(1)
|
||||
retq_id = id(retq)
|
||||
with self.retpend_mutex:
|
||||
self.retpend[retq_id] = retq
|
||||
else:
|
||||
retq = None
|
||||
retq_id = 0
|
||||
|
||||
self.q_yield.put((retq_id, dest, list(args)))
|
||||
self.q_yield.put([retq_id, dest, args])
|
||||
return retq
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
self.q_yield.put((0, dest, list(args)))
|
||||
def thr_workload(self):
|
||||
"""announce workloads to MpSrv (the mp controller / loadbalancer)"""
|
||||
# avoid locking in extract_filedata by tracking difference here
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
with self.mutex:
|
||||
if self.httpsrv.num_clients() == 0:
|
||||
# no clients rn, termiante thread
|
||||
self.workload_thr_alive = False
|
||||
return
|
||||
|
||||
self.q_yield.put([0, "workload", [self.httpsrv.workload]])
|
||||
|
||||
@@ -1,73 +1,56 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .broker_util import BrokerCli, ExceptionalQueue, try_exec
|
||||
from .authsrv import AuthSrv
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import HMaccas
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Any
|
||||
from .broker_util import ExceptionalQueue, try_exec
|
||||
|
||||
|
||||
class BrokerThr(BrokerCli):
|
||||
class BrokerThr(object):
|
||||
"""external api; behaves like BrokerMP but using plain threads"""
|
||||
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
super(BrokerThr, self).__init__()
|
||||
|
||||
def __init__(self, hub):
|
||||
self.hub = hub
|
||||
self.log = hub.log
|
||||
self.args = hub.args
|
||||
self.asrv = hub.asrv
|
||||
|
||||
self.mutex = threading.Lock()
|
||||
self.num_workers = 1
|
||||
|
||||
# instantiate all services here (TODO: inheritance?)
|
||||
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
|
||||
self.httpsrv = HttpSrv(self, None)
|
||||
self.reload = self.noop
|
||||
self.httpsrv = HttpSrv(self)
|
||||
self.httpsrv.disconnect_func = self.httpdrop
|
||||
|
||||
def shutdown(self) -> None:
|
||||
def shutdown(self):
|
||||
# self.log("broker", "shutting down")
|
||||
self.httpsrv.shutdown()
|
||||
|
||||
def noop(self) -> None:
|
||||
pass
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
def put(self, want_retval, dest, *args):
|
||||
if dest == "httpconn":
|
||||
sck, addr = args
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
|
||||
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
self.httpsrv.accept(sck, addr)
|
||||
|
||||
rv = try_exec(True, obj, *args)
|
||||
else:
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
# pretend we're broker_mp
|
||||
retq = ExceptionalQueue(1)
|
||||
retq.put(rv)
|
||||
return retq
|
||||
# TODO will deadlock if dest performs another ipc
|
||||
rv = try_exec(want_retval, obj, *args)
|
||||
if not want_retval:
|
||||
return
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
if dest == "listen":
|
||||
self.httpsrv.listen(args[0], 1)
|
||||
return
|
||||
# pretend we're broker_mp
|
||||
retq = ExceptionalQueue(1)
|
||||
retq.put(rv)
|
||||
return retq
|
||||
|
||||
if dest == "set_netdevs":
|
||||
self.httpsrv.set_netdevs(args[0])
|
||||
return
|
||||
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
try_exec(False, obj, *args)
|
||||
def httpdrop(self, addr):
|
||||
self.hub.tcpsrv.num_clients.add(-1)
|
||||
|
||||
@@ -1,28 +1,17 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
|
||||
import traceback
|
||||
|
||||
from queue import Queue
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .authsrv import AuthSrv
|
||||
from .util import HMaccas, Pebkac
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from .util import RootLogger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import Pebkac, Queue
|
||||
|
||||
|
||||
class ExceptionalQueue(Queue, object):
|
||||
def get(self, block: bool = True, timeout: Optional[float] = None) -> Any:
|
||||
def get(self, block=True, timeout=None):
|
||||
rv = super(ExceptionalQueue, self).get(block, timeout)
|
||||
|
||||
# TODO: how expensive is this?
|
||||
if isinstance(rv, list):
|
||||
if rv[0] == "exception":
|
||||
if rv[1] == "pebkac":
|
||||
@@ -33,29 +22,7 @@ class ExceptionalQueue(Queue, object):
|
||||
return rv
|
||||
|
||||
|
||||
class BrokerCli(object):
|
||||
"""
|
||||
helps mypy understand httpsrv.broker but still fails a few levels deeper,
|
||||
for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
|
||||
"""
|
||||
|
||||
log: "RootLogger"
|
||||
args: argparse.Namespace
|
||||
asrv: AuthSrv
|
||||
httpsrv: "HttpSrv"
|
||||
iphash: HMaccas
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
return ExceptionalQueue(1)
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
|
||||
def try_exec(want_retval, func, *args):
|
||||
try:
|
||||
return func(*args)
|
||||
|
||||
|
||||
152
copyparty/cfg.py
152
copyparty/cfg.py
@@ -1,152 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
# awk -F\" '/add_argument\("-[^-]/{print(substr($2,2))}' copyparty/__main__.py | sort | tr '\n' ' '
|
||||
zs = "a c e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vp e2vu ed emp i j lo mcr mte mth mtm mtp nb nc nid nih nw p q s ss sss v z zv"
|
||||
onedash = set(zs.split())
|
||||
|
||||
|
||||
def vf_bmap() -> dict[str, str]:
|
||||
"""argv-to-volflag: simple bools"""
|
||||
ret = {
|
||||
"never_symlink": "neversymlink",
|
||||
"no_dedup": "copydupes",
|
||||
"no_dupe": "nodupe",
|
||||
"no_forget": "noforget",
|
||||
"dav_rt": "davrt",
|
||||
}
|
||||
for k in (
|
||||
"dotsrch",
|
||||
"e2t",
|
||||
"e2ts",
|
||||
"e2tsr",
|
||||
"e2v",
|
||||
"e2vu",
|
||||
"e2vp",
|
||||
"hardlink",
|
||||
"magic",
|
||||
"no_sb_md",
|
||||
"no_sb_lg",
|
||||
"rand",
|
||||
"xdev",
|
||||
"xlink",
|
||||
"xvol",
|
||||
):
|
||||
ret[k] = k
|
||||
return ret
|
||||
|
||||
|
||||
def vf_vmap() -> dict[str, str]:
|
||||
"""argv-to-volflag: simple values"""
|
||||
ret = {}
|
||||
for k in ("lg_sbf", "md_sbf"):
|
||||
ret[k] = k
|
||||
return ret
|
||||
|
||||
|
||||
def vf_cmap() -> dict[str, str]:
|
||||
"""argv-to-volflag: complex/lists"""
|
||||
ret = {}
|
||||
for k in ("dbd", "html_head", "mte", "mth", "nrand"):
|
||||
ret[k] = k
|
||||
return ret
|
||||
|
||||
|
||||
permdescs = {
|
||||
"r": "read; list folder contents, download files",
|
||||
"w": 'write; upload files; need "r" to see the uploads',
|
||||
"m": 'move; move files and folders; need "w" at destination',
|
||||
"d": "delete; permanently delete files and folders",
|
||||
"g": "get; download files, but cannot see folder contents",
|
||||
"G": 'upget; same as "g" but can see filekeys of their own uploads',
|
||||
}
|
||||
|
||||
|
||||
flagcats = {
|
||||
"uploads, general": {
|
||||
"nodupe": "rejects existing files (instead of symlinking them)",
|
||||
"hardlink": "does dedup with hardlinks instead of symlinks",
|
||||
"neversymlink": "disables symlink fallback; full copy instead",
|
||||
"copydupes": "disables dedup, always saves full copies of dupes",
|
||||
"daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
|
||||
"nosub": "forces all uploads into the top folder of the vfs",
|
||||
"magic": "enables filetype detection for nameless uploads",
|
||||
"gz": "allows server-side gzip of uploads with ?gz (also c,xz)",
|
||||
"pk": "forces server-side compression, optional arg: xz,9",
|
||||
},
|
||||
"upload rules": {
|
||||
"maxn=250,600": "max 250 uploads over 15min",
|
||||
"maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g)",
|
||||
"rand": "force randomized filenames, 9 chars long by default",
|
||||
"nrand=N": "randomized filenames are N chars long",
|
||||
"sz=1k-3m": "allow filesizes between 1 KiB and 3MiB",
|
||||
"df=1g": "ensure 1 GiB free disk space",
|
||||
},
|
||||
"upload rotation\n(moves all uploads into the specified folder structure)": {
|
||||
"rotn=100,3": "3 levels of subfolders with 100 entries in each",
|
||||
"rotf=%Y-%m/%d-%H": "date-formatted organizing",
|
||||
"lifetime=3600": "uploads are deleted after 1 hour",
|
||||
},
|
||||
"database, general": {
|
||||
"e2d": "enable database; makes files searchable + enables upload dedup",
|
||||
"e2ds": "scan writable folders for new files on startup; also sets -e2d",
|
||||
"e2dsa": "scans all folders for new files on startup; also sets -e2d",
|
||||
"e2t": "enable multimedia indexing; makes it possible to search for tags",
|
||||
"e2ts": "scan existing files for tags on startup; also sets -e2t",
|
||||
"e2tsa": "delete all metadata from DB (full rescan); also sets -e2ts",
|
||||
"d2ts": "disables metadata collection for existing files",
|
||||
"d2ds": "disables onboot indexing, overrides -e2ds*",
|
||||
"d2t": "disables metadata collection, overrides -e2t*",
|
||||
"d2v": "disables file verification, overrides -e2v*",
|
||||
"d2d": "disables all database stuff, overrides -e2*",
|
||||
"hist=/tmp/cdb": "puts thumbnails and indexes at that location",
|
||||
"scan=60": "scan for new files every 60sec, same as --re-maxage",
|
||||
"nohash=\\.iso$": "skips hashing file contents if path matches *.iso",
|
||||
"noidx=\\.iso$": "fully ignores the contents at paths matching *.iso",
|
||||
"noforget": "don't forget files when deleted from disk",
|
||||
"dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
|
||||
"xlink": "cross-volume dupe detection / linking",
|
||||
"xdev": "do not descend into other filesystems",
|
||||
"xvol": "skip symlinks leaving the volume root",
|
||||
"dotsrch": "show dotfiles in search results",
|
||||
"nodotsrch": "hide dotfiles in search results (default)",
|
||||
},
|
||||
'database, audio tags\n"mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...': {
|
||||
"mtp=.bpm=f,audio-bpm.py": 'uses the "audio-bpm.py" program to\ngenerate ".bpm" tags from uploads (f = overwrite tags)',
|
||||
"mtp=ahash,vhash=media-hash.py": "collects two tags at once",
|
||||
},
|
||||
"thumbnails": {
|
||||
"dthumb": "disables all thumbnails",
|
||||
"dvthumb": "disables video thumbnails",
|
||||
"dathumb": "disables audio thumbnails (spectrograms)",
|
||||
"dithumb": "disables image thumbnails",
|
||||
},
|
||||
"event hooks\n(better explained in --help-hooks)": {
|
||||
"xbu=CMD": "execute CMD before a file upload starts",
|
||||
"xau=CMD": "execute CMD after a file upload finishes",
|
||||
"xiu=CMD": "execute CMD after all uploads finish and volume is idle",
|
||||
"xbr=CMD": "execute CMD before a file rename/move",
|
||||
"xar=CMD": "execute CMD after a file rename/move",
|
||||
"xbd=CMD": "execute CMD before a file delete",
|
||||
"xad=CMD": "execute CMD after a file delete",
|
||||
"xm=CMD": "execute CMD on message",
|
||||
},
|
||||
"client and ux": {
|
||||
"html_head=TXT": "includes TXT in the <head>",
|
||||
"robots": "allows indexing by search engines (default)",
|
||||
"norobots": "kindly asks search engines to leave",
|
||||
"no_sb_md": "disable js sandbox for markdown files",
|
||||
"no_sb_lg": "disable js sandbox for prologue/epilogue",
|
||||
"sb_md": "enable js sandbox for markdown files (default)",
|
||||
"sb_lg": "enable js sandbox for prologue/epilogue (default)",
|
||||
"md_sbf": "list of markdown-sandbox safeguards to disable",
|
||||
"lg_sbf": "list of *logue-sandbox safeguards to disable",
|
||||
},
|
||||
"others": {
|
||||
"fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission',
|
||||
"davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
flagdescs = {k.split("=")[0]: v for tab in flagcats.values() for k, v in tab.items()}
|
||||
@@ -1,72 +0,0 @@
|
||||
import importlib
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from .__init__ import PY2
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def get_ET() -> ET.XMLParser:
|
||||
pn = "xml.etree.ElementTree"
|
||||
cn = "_elementtree"
|
||||
|
||||
cmod = sys.modules.pop(cn, None)
|
||||
if not cmod:
|
||||
return ET.XMLParser # type: ignore
|
||||
|
||||
pmod = sys.modules.pop(pn)
|
||||
sys.modules[cn] = None # type: ignore
|
||||
|
||||
ret = importlib.import_module(pn)
|
||||
for name, mod in ((pn, pmod), (cn, cmod)):
|
||||
if mod:
|
||||
sys.modules[name] = mod
|
||||
else:
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
sys.modules["xml.etree"].ElementTree = pmod # type: ignore
|
||||
ret.ParseError = ET.ParseError # type: ignore
|
||||
return ret.XMLParser # type: ignore
|
||||
|
||||
|
||||
XMLParser: ET.XMLParser = get_ET()
|
||||
|
||||
|
||||
class DXMLParser(XMLParser): # type: ignore
|
||||
def __init__(self) -> None:
|
||||
tb = ET.TreeBuilder()
|
||||
super(DXMLParser, self).__init__(target=tb)
|
||||
|
||||
p = self._parser if PY2 else self.parser
|
||||
p.StartDoctypeDeclHandler = self.nope
|
||||
p.EntityDeclHandler = self.nope
|
||||
p.UnparsedEntityDeclHandler = self.nope
|
||||
p.ExternalEntityRefHandler = self.nope
|
||||
|
||||
def nope(self, *a: Any, **ka: Any) -> None:
|
||||
raise BadXML("{}, {}".format(a, ka))
|
||||
|
||||
|
||||
class BadXML(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def parse_xml(txt: str) -> ET.Element:
|
||||
parser = DXMLParser()
|
||||
parser.feed(txt)
|
||||
return parser.close() # type: ignore
|
||||
|
||||
|
||||
def mktnod(name: str, text: str) -> ET.Element:
|
||||
el = ET.Element(name)
|
||||
el.text = text
|
||||
return el
|
||||
|
||||
|
||||
def mkenod(name: str, sub_el: Optional[ET.Element] = None) -> ET.Element:
|
||||
el = ET.Element(name)
|
||||
if sub_el is not None:
|
||||
el.append(sub_el)
|
||||
return el
|
||||
@@ -1,152 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from .__init__ import ANYWIN, MACOS
|
||||
from .authsrv import AXS, VFS
|
||||
from .bos import bos
|
||||
from .util import chkcmd, min_ex
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Optional, Union
|
||||
|
||||
from .util import RootLogger
|
||||
|
||||
|
||||
class Fstab(object):
|
||||
def __init__(self, log: "RootLogger"):
|
||||
self.log_func = log
|
||||
|
||||
self.trusted = False
|
||||
self.tab: Optional[VFS] = None
|
||||
self.cache: dict[str, str] = {}
|
||||
self.age = 0.0
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("fstab", msg, c)
|
||||
|
||||
def get(self, path: str) -> str:
|
||||
if len(self.cache) > 9000:
|
||||
self.age = time.time()
|
||||
self.tab = None
|
||||
self.cache = {}
|
||||
|
||||
fs = "ext4"
|
||||
msg = "failed to determine filesystem at [{}]; assuming {}\n{}"
|
||||
|
||||
if ANYWIN:
|
||||
fs = "vfat"
|
||||
try:
|
||||
path = self._winpath(path)
|
||||
except:
|
||||
self.log(msg.format(path, fs, min_ex()), 3)
|
||||
return fs
|
||||
|
||||
path = path.lstrip("/")
|
||||
try:
|
||||
return self.cache[path]
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
fs = self.get_w32(path) if ANYWIN else self.get_unix(path)
|
||||
except:
|
||||
self.log(msg.format(path, fs, min_ex()), 3)
|
||||
|
||||
fs = fs.lower()
|
||||
self.cache[path] = fs
|
||||
self.log("found {} at {}".format(fs, path))
|
||||
return fs
|
||||
|
||||
def _winpath(self, path: str) -> str:
|
||||
# try to combine volume-label + st_dev (vsn)
|
||||
path = path.replace("/", "\\")
|
||||
vid = path.split(":", 1)[0].strip("\\").split("\\", 1)[0]
|
||||
try:
|
||||
return "{}*{}".format(vid, bos.stat(path).st_dev)
|
||||
except:
|
||||
return vid
|
||||
|
||||
def build_fallback(self) -> None:
|
||||
self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
|
||||
self.trusted = False
|
||||
|
||||
def build_tab(self) -> None:
|
||||
self.log("building tab")
|
||||
|
||||
sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
|
||||
if MACOS:
|
||||
sptn = r"^.*? on (.*) \(([^ ]+), .*"
|
||||
|
||||
ptn = re.compile(sptn)
|
||||
so, _ = chkcmd(["mount"])
|
||||
tab1: list[tuple[str, str]] = []
|
||||
for ln in so.split("\n"):
|
||||
m = ptn.match(ln)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
zs1, zs2 = m.groups()
|
||||
tab1.append((str(zs1), str(zs2)))
|
||||
|
||||
tab1.sort(key=lambda x: (len(x[0]), x[0]))
|
||||
path1, fs1 = tab1[0]
|
||||
tab = VFS(self.log_func, fs1, path1, AXS(), {})
|
||||
for path, fs in tab1[1:]:
|
||||
tab.add(fs, path.lstrip("/"))
|
||||
|
||||
self.tab = tab
|
||||
|
||||
def relabel(self, path: str, nval: str) -> None:
|
||||
assert self.tab
|
||||
self.cache = {}
|
||||
if ANYWIN:
|
||||
path = self._winpath(path)
|
||||
|
||||
path = path.lstrip("/")
|
||||
ptn = re.compile(r"^[^\\/]*")
|
||||
vn, rem = self.tab._find(path)
|
||||
if not self.trusted:
|
||||
# no mtab access; have to build as we go
|
||||
if "/" in rem:
|
||||
self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
|
||||
if rem:
|
||||
self.tab.add(nval, path)
|
||||
else:
|
||||
vn.realpath = nval
|
||||
|
||||
return
|
||||
|
||||
visit = [vn]
|
||||
while visit:
|
||||
vn = visit.pop()
|
||||
vn.realpath = ptn.sub(nval, vn.realpath)
|
||||
visit.extend(list(vn.nodes.values()))
|
||||
|
||||
def get_unix(self, path: str) -> str:
|
||||
if not self.tab:
|
||||
try:
|
||||
self.build_tab()
|
||||
self.trusted = True
|
||||
except:
|
||||
# prisonparty or other restrictive environment
|
||||
self.log("failed to build tab:\n{}".format(min_ex()), 3)
|
||||
self.build_fallback()
|
||||
|
||||
assert self.tab
|
||||
ret = self.tab._find(path)[0]
|
||||
if self.trusted or path == ret.vpath:
|
||||
return ret.realpath.split("/")[0]
|
||||
else:
|
||||
return "idk"
|
||||
|
||||
def get_w32(self, path: str) -> str:
|
||||
if not self.tab:
|
||||
self.build_fallback()
|
||||
|
||||
assert self.tab
|
||||
ret = self.tab._find(path)[0]
|
||||
return ret.realpath
|
||||
@@ -1,506 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
|
||||
from pyftpdlib.filesystems import AbstractedFS, FilesystemError
|
||||
from pyftpdlib.handlers import FTPHandler
|
||||
from pyftpdlib.servers import FTPServer
|
||||
|
||||
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E
|
||||
from .authsrv import VFS
|
||||
from .bos import bos
|
||||
from .util import (
|
||||
Daemon,
|
||||
Pebkac,
|
||||
exclude_dotfiles,
|
||||
fsenc,
|
||||
ipnorm,
|
||||
pybin,
|
||||
relchk,
|
||||
runhook,
|
||||
sanitize_fn,
|
||||
vjoin,
|
||||
)
|
||||
|
||||
try:
|
||||
from pyftpdlib.ioloop import IOLoop
|
||||
except ImportError:
|
||||
p = os.path.join(E.mod, "vend")
|
||||
print("loading asynchat from " + p)
|
||||
sys.path.append(p)
|
||||
from pyftpdlib.ioloop import IOLoop
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
import typing
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class FtpAuth(DummyAuthorizer):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
super(FtpAuth, self).__init__()
|
||||
self.hub = hub
|
||||
|
||||
def validate_authentication(
|
||||
self, username: str, password: str, handler: Any
|
||||
) -> None:
|
||||
handler.username = "{}:{}".format(username, password)
|
||||
handler.uname = "*"
|
||||
|
||||
ip = handler.addr[0]
|
||||
if ip.startswith("::ffff:"):
|
||||
ip = ip[7:]
|
||||
|
||||
ip = ipnorm(ip)
|
||||
bans = self.hub.bans
|
||||
if ip in bans:
|
||||
rt = bans[ip] - time.time()
|
||||
if rt < 0:
|
||||
logging.info("client unbanned")
|
||||
del bans[ip]
|
||||
else:
|
||||
raise AuthenticationFailed("banned")
|
||||
|
||||
asrv = self.hub.asrv
|
||||
if username == "anonymous":
|
||||
uname = "*"
|
||||
else:
|
||||
uname = asrv.iacct.get(password, "") or asrv.iacct.get(username, "") or "*"
|
||||
|
||||
if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
|
||||
g = self.hub.gpwd
|
||||
if g.lim:
|
||||
bonk, ip = g.bonk(ip, handler.username)
|
||||
if bonk:
|
||||
logging.warning("client banned: invalid passwords")
|
||||
bans[ip] = bonk
|
||||
|
||||
raise AuthenticationFailed("Authentication failed.")
|
||||
|
||||
handler.uname = uname
|
||||
|
||||
def get_home_dir(self, username: str) -> str:
|
||||
return "/"
|
||||
|
||||
def has_user(self, username: str) -> bool:
|
||||
asrv = self.hub.asrv
|
||||
return username in asrv.acct or username in asrv.iacct
|
||||
|
||||
def has_perm(self, username: str, perm: int, path: Optional[str] = None) -> bool:
|
||||
return True # handled at filesystem layer
|
||||
|
||||
def get_perms(self, username: str) -> str:
|
||||
return "elradfmwMT"
|
||||
|
||||
def get_msg_login(self, username: str) -> str:
|
||||
return "sup {}".format(username)
|
||||
|
||||
def get_msg_quit(self, username: str) -> str:
|
||||
return "cya"
|
||||
|
||||
|
||||
class FtpFs(AbstractedFS):
|
||||
def __init__(
|
||||
self, root: str, cmd_channel: Any
|
||||
) -> None: # pylint: disable=super-init-not-called
|
||||
self.h = cmd_channel # type: FTPHandler
|
||||
self.cmd_channel = cmd_channel # type: FTPHandler
|
||||
self.hub: "SvcHub" = cmd_channel.hub
|
||||
self.args = cmd_channel.args
|
||||
self.uname = cmd_channel.uname
|
||||
|
||||
self.cwd = "/" # pyftpdlib convention of leading slash
|
||||
self.root = "/var/lib/empty"
|
||||
|
||||
self.can_read = self.can_write = self.can_move = False
|
||||
self.can_delete = self.can_get = self.can_upget = False
|
||||
|
||||
self.listdirinfo = self.listdir
|
||||
self.chdir(".")
|
||||
|
||||
def die(self, msg):
|
||||
self.h.die(msg)
|
||||
raise Exception()
|
||||
|
||||
def v2a(
|
||||
self,
|
||||
vpath: str,
|
||||
r: bool = False,
|
||||
w: bool = False,
|
||||
m: bool = False,
|
||||
d: bool = False,
|
||||
) -> tuple[str, VFS, str]:
|
||||
try:
|
||||
vpath = vpath.replace("\\", "/").lstrip("/")
|
||||
rd, fn = os.path.split(vpath)
|
||||
if ANYWIN and relchk(rd):
|
||||
logging.warning("malicious vpath: %s", vpath)
|
||||
self.die("Unsupported characters in filepath")
|
||||
|
||||
fn = sanitize_fn(fn or "", "", [".prologue.html", ".epilogue.html"])
|
||||
vpath = vjoin(rd, fn)
|
||||
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
|
||||
if not vfs.realpath:
|
||||
self.die("No filesystem mounted at this path")
|
||||
|
||||
return os.path.join(vfs.realpath, rem), vfs, rem
|
||||
except Pebkac as ex:
|
||||
self.die(str(ex))
|
||||
|
||||
def rv2a(
|
||||
self,
|
||||
vpath: str,
|
||||
r: bool = False,
|
||||
w: bool = False,
|
||||
m: bool = False,
|
||||
d: bool = False,
|
||||
) -> tuple[str, VFS, str]:
|
||||
return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
|
||||
|
||||
def ftp2fs(self, ftppath: str) -> str:
|
||||
# return self.v2a(ftppath)
|
||||
return ftppath # self.cwd must be vpath
|
||||
|
||||
def fs2ftp(self, fspath: str) -> str:
|
||||
# raise NotImplementedError()
|
||||
return fspath
|
||||
|
||||
def validpath(self, path: str) -> bool:
|
||||
if "/.hist/" in path:
|
||||
if "/up2k." in path or path.endswith("/dir.txt"):
|
||||
self.die("Access to this file is forbidden")
|
||||
|
||||
return True
|
||||
|
||||
def open(self, filename: str, mode: str) -> typing.IO[Any]:
|
||||
r = "r" in mode
|
||||
w = "w" in mode or "a" in mode or "+" in mode
|
||||
|
||||
ap = self.rv2a(filename, r, w)[0]
|
||||
if w:
|
||||
try:
|
||||
st = bos.stat(ap)
|
||||
td = time.time() - st.st_mtime
|
||||
except:
|
||||
td = 0
|
||||
|
||||
if td < -1 or td > self.args.ftp_wt:
|
||||
self.die("Cannot open existing file for writing")
|
||||
|
||||
self.validpath(ap)
|
||||
return open(fsenc(ap), mode)
|
||||
|
||||
def chdir(self, path: str) -> None:
|
||||
nwd = join(self.cwd, path)
|
||||
vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
|
||||
ap = vfs.canonical(rem)
|
||||
if not bos.path.isdir(ap):
|
||||
# returning 550 is library-default and suitable
|
||||
self.die("Failed to change directory")
|
||||
|
||||
self.cwd = nwd
|
||||
(
|
||||
self.can_read,
|
||||
self.can_write,
|
||||
self.can_move,
|
||||
self.can_delete,
|
||||
self.can_get,
|
||||
self.can_upget,
|
||||
) = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.uname)
|
||||
|
||||
def mkdir(self, path: str) -> None:
|
||||
ap = self.rv2a(path, w=True)[0]
|
||||
bos.makedirs(ap) # filezilla expects this
|
||||
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
vpath = join(self.cwd, path).lstrip("/")
|
||||
try:
|
||||
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
|
||||
|
||||
fsroot, vfs_ls1, vfs_virt = vfs.ls(
|
||||
rem,
|
||||
self.uname,
|
||||
not self.args.no_scandir,
|
||||
[[True, False], [False, True]],
|
||||
)
|
||||
vfs_ls = [x[0] for x in vfs_ls1]
|
||||
vfs_ls.extend(vfs_virt.keys())
|
||||
|
||||
if not self.args.ed:
|
||||
vfs_ls = exclude_dotfiles(vfs_ls)
|
||||
|
||||
vfs_ls.sort()
|
||||
return vfs_ls
|
||||
except:
|
||||
if vpath:
|
||||
# display write-only folders as empty
|
||||
return []
|
||||
|
||||
# return list of volumes
|
||||
r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
|
||||
return list(sorted(list(r.keys())))
|
||||
|
||||
def rmdir(self, path: str) -> None:
|
||||
ap = self.rv2a(path, d=True)[0]
|
||||
bos.rmdir(ap)
|
||||
|
||||
def remove(self, path: str) -> None:
|
||||
if self.args.no_del:
|
||||
self.die("The delete feature is disabled in server config")
|
||||
|
||||
vp = join(self.cwd, path).lstrip("/")
|
||||
try:
|
||||
self.hub.up2k.handle_rm(self.uname, self.h.cli_ip, [vp], [])
|
||||
except Exception as ex:
|
||||
self.die(str(ex))
|
||||
|
||||
def rename(self, src: str, dst: str) -> None:
|
||||
if not self.can_move:
|
||||
self.die("Not allowed for user " + self.h.uname)
|
||||
|
||||
if self.args.no_mv:
|
||||
self.die("The rename/move feature is disabled in server config")
|
||||
|
||||
svp = join(self.cwd, src).lstrip("/")
|
||||
dvp = join(self.cwd, dst).lstrip("/")
|
||||
try:
|
||||
self.hub.up2k.handle_mv(self.uname, svp, dvp)
|
||||
except Exception as ex:
|
||||
self.die(str(ex))
|
||||
|
||||
def chmod(self, path: str, mode: str) -> None:
|
||||
pass
|
||||
|
||||
def stat(self, path: str) -> os.stat_result:
|
||||
try:
|
||||
ap = self.rv2a(path, r=True)[0]
|
||||
return bos.stat(ap)
|
||||
except:
|
||||
ap = self.rv2a(path)[0]
|
||||
st = bos.stat(ap)
|
||||
if not stat.S_ISDIR(st.st_mode):
|
||||
raise
|
||||
|
||||
return st
|
||||
|
||||
def utime(self, path: str, timeval: float) -> None:
|
||||
ap = self.rv2a(path, w=True)[0]
|
||||
return bos.utime(ap, (timeval, timeval))
|
||||
|
||||
def lstat(self, path: str) -> os.stat_result:
|
||||
ap = self.rv2a(path)[0]
|
||||
return bos.stat(ap)
|
||||
|
||||
def isfile(self, path: str) -> bool:
|
||||
try:
|
||||
st = self.stat(path)
|
||||
return stat.S_ISREG(st.st_mode)
|
||||
except:
|
||||
return False # expected for mojibake in ftp_SIZE()
|
||||
|
||||
def islink(self, path: str) -> bool:
|
||||
ap = self.rv2a(path)[0]
|
||||
return bos.path.islink(ap)
|
||||
|
||||
def isdir(self, path: str) -> bool:
|
||||
try:
|
||||
st = self.stat(path)
|
||||
return stat.S_ISDIR(st.st_mode)
|
||||
except:
|
||||
return True
|
||||
|
||||
def getsize(self, path: str) -> int:
|
||||
ap = self.rv2a(path)[0]
|
||||
return bos.path.getsize(ap)
|
||||
|
||||
def getmtime(self, path: str) -> float:
|
||||
ap = self.rv2a(path)[0]
|
||||
return bos.path.getmtime(ap)
|
||||
|
||||
def realpath(self, path: str) -> str:
|
||||
return path
|
||||
|
||||
def lexists(self, path: str) -> bool:
|
||||
ap = self.rv2a(path)[0]
|
||||
return bos.path.lexists(ap)
|
||||
|
||||
def get_user_by_uid(self, uid: int) -> str:
|
||||
return "root"
|
||||
|
||||
def get_group_by_uid(self, gid: int) -> str:
|
||||
return "root"
|
||||
|
||||
|
||||
class FtpHandler(FTPHandler):
|
||||
abstracted_fs = FtpFs
|
||||
hub: "SvcHub"
|
||||
args: argparse.Namespace
|
||||
uname: str
|
||||
|
||||
def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
|
||||
self.hub: "SvcHub" = FtpHandler.hub
|
||||
self.args: argparse.Namespace = FtpHandler.args
|
||||
self.uname = "*"
|
||||
|
||||
if PY2:
|
||||
FTPHandler.__init__(self, conn, server, ioloop)
|
||||
else:
|
||||
super(FtpHandler, self).__init__(conn, server, ioloop)
|
||||
|
||||
cip = self.remote_ip
|
||||
self.cli_ip = cip[7:] if cip.startswith("::ffff:") else cip
|
||||
|
||||
# abspath->vpath mapping to resolve log_transfer paths
|
||||
self.vfs_map: dict[str, str] = {}
|
||||
|
||||
# reduce non-debug logging
|
||||
self.log_cmds_list = [x for x in self.log_cmds_list if x not in ("CWD", "XCWD")]
|
||||
|
||||
def die(self, msg):
|
||||
self.respond("550 {}".format(msg))
|
||||
raise FilesystemError(msg)
|
||||
|
||||
def ftp_STOR(self, file: str, mode: str = "w") -> Any:
|
||||
# Optional[str]
|
||||
vp = join(self.fs.cwd, file).lstrip("/")
|
||||
ap, vfs, rem = self.fs.v2a(vp)
|
||||
self.vfs_map[ap] = vp
|
||||
xbu = vfs.flags.get("xbu")
|
||||
if xbu and not runhook(
|
||||
None,
|
||||
xbu,
|
||||
ap,
|
||||
vfs.canonical(rem),
|
||||
"",
|
||||
self.uname,
|
||||
0,
|
||||
0,
|
||||
self.cli_ip,
|
||||
0,
|
||||
"",
|
||||
):
|
||||
self.die("Upload blocked by xbu server config")
|
||||
|
||||
# print("ftp_STOR: {} {} => {}".format(vp, mode, ap))
|
||||
ret = FTPHandler.ftp_STOR(self, file, mode)
|
||||
# print("ftp_STOR: {} {} OK".format(vp, mode))
|
||||
return ret
|
||||
|
||||
def log_transfer(
|
||||
self,
|
||||
cmd: str,
|
||||
filename: bytes,
|
||||
receive: bool,
|
||||
completed: bool,
|
||||
elapsed: float,
|
||||
bytes: int,
|
||||
) -> Any:
|
||||
# None
|
||||
ap = filename.decode("utf-8", "replace")
|
||||
vp = self.vfs_map.pop(ap, None)
|
||||
# print("xfer_end: {} => {}".format(ap, vp))
|
||||
if vp:
|
||||
vp, fn = os.path.split(vp)
|
||||
vfs, rem = self.hub.asrv.vfs.get(vp, self.uname, False, True)
|
||||
vfs, rem = vfs.get_dbv(rem)
|
||||
self.hub.up2k.hash_file(
|
||||
vfs.realpath,
|
||||
vfs.vpath,
|
||||
vfs.flags,
|
||||
rem,
|
||||
fn,
|
||||
self.cli_ip,
|
||||
time.time(),
|
||||
self.uname,
|
||||
)
|
||||
|
||||
return FTPHandler.log_transfer(
|
||||
self, cmd, filename, receive, completed, elapsed, bytes
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
from pyftpdlib.handlers import TLS_FTPHandler
|
||||
|
||||
class SftpHandler(FtpHandler, TLS_FTPHandler):
|
||||
pass
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class Ftpd(object):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
self.hub = hub
|
||||
self.args = hub.args
|
||||
|
||||
hs = []
|
||||
if self.args.ftp:
|
||||
hs.append([FtpHandler, self.args.ftp])
|
||||
if self.args.ftps:
|
||||
try:
|
||||
h1 = SftpHandler
|
||||
except:
|
||||
t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
|
||||
print(t.format(pybin))
|
||||
sys.exit(1)
|
||||
|
||||
h1.certfile = self.args.cert
|
||||
h1.tls_control_required = True
|
||||
h1.tls_data_required = True
|
||||
|
||||
hs.append([h1, self.args.ftps])
|
||||
|
||||
for h_lp in hs:
|
||||
h2, lp = h_lp
|
||||
h2.hub = hub
|
||||
h2.args = hub.args
|
||||
h2.authorizer = FtpAuth(hub)
|
||||
|
||||
if self.args.ftp_pr:
|
||||
p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
|
||||
if self.args.ftp and self.args.ftps:
|
||||
# divide port range in half
|
||||
d = int((p2 - p1) / 2)
|
||||
if lp == self.args.ftp:
|
||||
p2 = p1 + d
|
||||
else:
|
||||
p1 += d + 1
|
||||
|
||||
h2.passive_ports = list(range(p1, p2 + 1))
|
||||
|
||||
if self.args.ftp_nat:
|
||||
h2.masquerade_address = self.args.ftp_nat
|
||||
|
||||
lgr = logging.getLogger("pyftpdlib")
|
||||
lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO)
|
||||
|
||||
ips = self.args.i
|
||||
if "::" in ips:
|
||||
ips.append("0.0.0.0")
|
||||
|
||||
ioloop = IOLoop()
|
||||
for ip in ips:
|
||||
for h, lp in hs:
|
||||
try:
|
||||
FTPServer((ip, int(lp)), h, ioloop)
|
||||
except:
|
||||
if ip != "0.0.0.0" or "::" not in ips:
|
||||
raise
|
||||
|
||||
Daemon(ioloop.loop, "ftp")
|
||||
|
||||
|
||||
def join(p1: str, p2: str) -> str:
|
||||
w = os.path.join(p1, p2.replace("\\", "/"))
|
||||
return os.path.normpath(w).replace("\\", "/")
|
||||
3182
copyparty/httpcli.py
3182
copyparty/httpcli.py
File diff suppressed because it is too large
Load Diff
@@ -1,38 +1,24 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse # typechk
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import threading # typechk
|
||||
import os
|
||||
import time
|
||||
import socket
|
||||
|
||||
HAVE_SSL = True
|
||||
try:
|
||||
HAVE_SSL = True
|
||||
import ssl
|
||||
except:
|
||||
HAVE_SSL = False
|
||||
|
||||
from . import util as Util
|
||||
from .__init__ import TYPE_CHECKING, EnvParams
|
||||
from .authsrv import AuthSrv # typechk
|
||||
from .__init__ import E
|
||||
from .util import Unrecv
|
||||
from .httpcli import HttpCli
|
||||
from .ico import Ico
|
||||
from .mtag import HAVE_FFMPEG
|
||||
from .th_cli import ThumbCli
|
||||
from .th_srv import HAVE_PIL, HAVE_VIPS
|
||||
from .u2idx import U2idx
|
||||
from .util import HMaccas, shut_socket
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Optional, Pattern, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
|
||||
|
||||
PTN_HTTP = re.compile(br"[A-Z]{3}[A-Z ]")
|
||||
from .th_cli import ThumbCli
|
||||
from .th_srv import HAVE_PIL
|
||||
from .ico import Ico
|
||||
|
||||
|
||||
class HttpConn(object):
|
||||
@@ -41,50 +27,39 @@ class HttpConn(object):
|
||||
creates an HttpCli for each request (Connection: Keep-Alive)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, sck: socket.socket, addr: tuple[str, int], hsrv: "HttpSrv"
|
||||
) -> None:
|
||||
def __init__(self, sck, addr, hsrv):
|
||||
self.s = sck
|
||||
self.sr: Optional[Util._Unrecv] = None
|
||||
self.cli: Optional[HttpCli] = None
|
||||
self.addr = addr
|
||||
self.hsrv = hsrv
|
||||
|
||||
self.mutex: threading.Lock = hsrv.mutex # mypy404
|
||||
self.args: argparse.Namespace = hsrv.args # mypy404
|
||||
self.E: EnvParams = self.args.E
|
||||
self.asrv: AuthSrv = hsrv.asrv # mypy404
|
||||
self.args = hsrv.args
|
||||
self.asrv = hsrv.asrv
|
||||
self.is_mp = hsrv.is_mp
|
||||
self.cert_path = hsrv.cert_path
|
||||
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
|
||||
self.iphash: HMaccas = hsrv.broker.iphash
|
||||
self.bans: dict[str, int] = hsrv.bans
|
||||
self.aclose: dict[str, int] = hsrv.aclose
|
||||
|
||||
enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
|
||||
self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404
|
||||
self.ico: Ico = Ico(self.args) # mypy404
|
||||
enth = HAVE_PIL and not self.args.no_thumb
|
||||
self.thumbcli = ThumbCli(hsrv.broker) if enth else None
|
||||
self.ico = Ico(self.args)
|
||||
|
||||
self.t0: float = time.time() # mypy404
|
||||
self.freshen_pwd: float = 0.0
|
||||
self.t0 = time.time()
|
||||
self.stopping = False
|
||||
self.nreq: int = -1 # mypy404
|
||||
self.nbyte: int = 0 # mypy404
|
||||
self.u2idx: Optional[U2idx] = None
|
||||
self.log_func: "Util.RootLogger" = hsrv.log # mypy404
|
||||
self.log_src: str = "httpconn" # mypy404
|
||||
self.lf_url: Optional[Pattern[str]] = (
|
||||
re.compile(self.args.lf_url) if self.args.lf_url else None
|
||||
) # mypy404
|
||||
self.nreq = 0
|
||||
self.nbyte = 0
|
||||
self.workload = 0
|
||||
self.u2idx = None
|
||||
self.log_func = hsrv.log
|
||||
self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
|
||||
self.set_rproxy()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
def shutdown(self):
|
||||
self.stopping = True
|
||||
try:
|
||||
shut_socket(self.log, self.s, 1)
|
||||
self.s.shutdown(socket.SHUT_RDWR)
|
||||
self.s.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
def set_rproxy(self, ip: Optional[str] = None) -> str:
|
||||
def set_rproxy(self, ip=None):
|
||||
if ip is None:
|
||||
color = 36
|
||||
ip = self.addr[0]
|
||||
@@ -97,38 +72,35 @@ class HttpConn(object):
|
||||
self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
|
||||
return self.log_src
|
||||
|
||||
def respath(self, res_name: str) -> str:
|
||||
return os.path.join(self.E.mod, "web", res_name)
|
||||
def respath(self, res_name):
|
||||
return os.path.join(E.mod, "web", res_name)
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def log(self, msg, c=0):
|
||||
self.log_func(self.log_src, msg, c)
|
||||
|
||||
def get_u2idx(self) -> Optional[U2idx]:
|
||||
# grab from a pool of u2idx instances;
|
||||
# sqlite3 fully parallelizes under python threads
|
||||
# but avoid running out of FDs by creating too many
|
||||
def get_u2idx(self):
|
||||
if not self.u2idx:
|
||||
self.u2idx = self.hsrv.get_u2idx(str(self.addr))
|
||||
self.u2idx = U2idx(self)
|
||||
|
||||
return self.u2idx
|
||||
|
||||
def _detect_https(self) -> bool:
|
||||
def _detect_https(self):
|
||||
method = None
|
||||
if self.cert_path:
|
||||
try:
|
||||
method = self.s.recv(4, socket.MSG_PEEK)
|
||||
except socket.timeout:
|
||||
return False
|
||||
return
|
||||
except AttributeError:
|
||||
# jython does not support msg_peek; forget about https
|
||||
method = self.s.recv(4)
|
||||
self.sr = Util.Unrecv(self.s, self.log)
|
||||
self.sr = Unrecv(self.s)
|
||||
self.sr.buf = method
|
||||
|
||||
# jython used to do this, they stopped since it's broken
|
||||
# but reimplementing sendall is out of scope for now
|
||||
if not getattr(self.s, "sendall", None):
|
||||
self.s.sendall = self.s.send # type: ignore
|
||||
self.s.sendall = self.s.send
|
||||
|
||||
if len(method) != 4:
|
||||
err = "need at least 4 bytes in the first packet; got {}".format(
|
||||
@@ -138,20 +110,17 @@ class HttpConn(object):
|
||||
self.log(err)
|
||||
|
||||
self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
|
||||
return False
|
||||
return
|
||||
|
||||
return not method or not bool(PTN_HTTP.match(method))
|
||||
|
||||
def run(self) -> None:
|
||||
self.s.settimeout(10)
|
||||
return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
|
||||
|
||||
def run(self):
|
||||
self.sr = None
|
||||
if self.args.https_only:
|
||||
is_https = True
|
||||
elif self.args.http_only or not HAVE_SSL:
|
||||
is_https = False
|
||||
else:
|
||||
# raise Exception("asdf")
|
||||
is_https = self._detect_https()
|
||||
|
||||
if is_https:
|
||||
@@ -180,15 +149,14 @@ class HttpConn(object):
|
||||
self.s = ctx.wrap_socket(self.s, server_side=True)
|
||||
msg = [
|
||||
"\033[1;3{:d}m{}".format(c, s)
|
||||
for c, s in zip([0, 5, 0], self.s.cipher()) # type: ignore
|
||||
for c, s in zip([0, 5, 0], self.s.cipher())
|
||||
]
|
||||
self.log(" ".join(msg) + "\033[0m")
|
||||
|
||||
if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
|
||||
ciphers = self.s.shared_ciphers()
|
||||
assert ciphers
|
||||
overlap = [str(y[::-1]) for y in ciphers]
|
||||
self.log("TLS cipher overlap:" + "\n".join(overlap))
|
||||
overlap = [y[::-1] for y in self.s.shared_ciphers()]
|
||||
lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
|
||||
self.log("\n".join(lines))
|
||||
for k, v in [
|
||||
["compression", self.s.compression()],
|
||||
["ALPN proto", self.s.selected_alpn_protocol()],
|
||||
@@ -199,7 +167,11 @@ class HttpConn(object):
|
||||
except Exception as ex:
|
||||
em = str(ex)
|
||||
|
||||
if "ALERT_CERTIFICATE_UNKNOWN" in em:
|
||||
if "ALERT_BAD_CERTIFICATE" in em:
|
||||
# firefox-linux if there is no exception yet
|
||||
self.log("client rejected our certificate (nice)")
|
||||
|
||||
elif "ALERT_CERTIFICATE_UNKNOWN" in em:
|
||||
# android-chrome keeps doing this
|
||||
pass
|
||||
|
||||
@@ -209,14 +181,15 @@ class HttpConn(object):
|
||||
return
|
||||
|
||||
if not self.sr:
|
||||
self.sr = Util.Unrecv(self.s, self.log)
|
||||
self.sr = Unrecv(self.s)
|
||||
|
||||
while not self.stopping:
|
||||
self.nreq += 1
|
||||
self.cli = HttpCli(self)
|
||||
if not self.cli.run():
|
||||
return
|
||||
if self.is_mp:
|
||||
self.workload += 50
|
||||
if self.workload >= 2 ** 31:
|
||||
self.workload = 100
|
||||
|
||||
if self.u2idx:
|
||||
self.hsrv.put_u2idx(str(self.addr), self.u2idx)
|
||||
self.u2idx = None
|
||||
self.nreq += 1
|
||||
cli = HttpCli(self)
|
||||
if not cli.run():
|
||||
return
|
||||
|
||||
@@ -1,29 +1,17 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import base64
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import queue
|
||||
|
||||
from .__init__ import ANYWIN, CORES, EXE, MACOS, TYPE_CHECKING, EnvParams
|
||||
|
||||
try:
|
||||
MNFE = ModuleNotFoundError
|
||||
except:
|
||||
MNFE = ImportError
|
||||
import base64
|
||||
import struct
|
||||
import socket
|
||||
import threading
|
||||
|
||||
try:
|
||||
import jinja2
|
||||
except MNFE:
|
||||
if EXE:
|
||||
raise
|
||||
|
||||
except ImportError:
|
||||
print(
|
||||
"""\033[1;31m
|
||||
you do not have jinja2 installed,\033[33m
|
||||
@@ -38,31 +26,8 @@ except MNFE:
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
from .bos import bos
|
||||
from .__init__ import E, MACOS
|
||||
from .httpconn import HttpConn
|
||||
from .u2idx import U2idx
|
||||
from .util import (
|
||||
E_SCK,
|
||||
FHC,
|
||||
Daemon,
|
||||
Garda,
|
||||
Magician,
|
||||
Netdev,
|
||||
NetMap,
|
||||
ipnorm,
|
||||
min_ex,
|
||||
shut_socket,
|
||||
spack,
|
||||
start_log_thrs,
|
||||
start_stackmon,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .broker_util import BrokerCli
|
||||
from .ssdp import SSDPr
|
||||
|
||||
if True: # pylint: disable=using-constant-test
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class HttpSrv(object):
|
||||
@@ -71,356 +36,89 @@ class HttpSrv(object):
|
||||
relying on MpSrv for performance (HttpSrv is just plain threads)
|
||||
"""
|
||||
|
||||
def __init__(self, broker: "BrokerCli", nid: Optional[int]) -> None:
|
||||
def __init__(self, broker, is_mp=False):
|
||||
self.broker = broker
|
||||
self.nid = nid
|
||||
self.is_mp = is_mp
|
||||
self.args = broker.args
|
||||
self.E: EnvParams = self.args.E
|
||||
self.log = broker.log
|
||||
self.asrv = broker.asrv
|
||||
|
||||
# redefine in case of multiprocessing
|
||||
socket.setdefaulttimeout(120)
|
||||
|
||||
nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
|
||||
self.magician = Magician()
|
||||
self.nm = NetMap([], {})
|
||||
self.ssdp: Optional["SSDPr"] = None
|
||||
self.gpwd = Garda(self.args.ban_pw)
|
||||
self.g404 = Garda(self.args.ban_404)
|
||||
self.bans: dict[str, int] = {}
|
||||
self.aclose: dict[str, int] = {}
|
||||
|
||||
self.bound: set[tuple[str, int]] = set()
|
||||
self.name = "hsrv" + nsuf
|
||||
self.disconnect_func = None
|
||||
self.mutex = threading.Lock()
|
||||
self.stopping = False
|
||||
|
||||
self.tp_nthr = 0 # actual
|
||||
self.tp_ncli = 0 # fading
|
||||
self.tp_time = 0.0 # latest worker collect
|
||||
self.tp_q: Optional[queue.LifoQueue[Any]] = (
|
||||
None if self.args.no_htp else queue.LifoQueue()
|
||||
)
|
||||
self.t_periodic: Optional[threading.Thread] = None
|
||||
|
||||
self.u2fh = FHC()
|
||||
self.srvs: list[socket.socket] = []
|
||||
self.ncli = 0 # exact
|
||||
self.clients: set[HttpConn] = set() # laggy
|
||||
self.nclimax = 0
|
||||
self.cb_ts = 0.0
|
||||
self.cb_v = ""
|
||||
|
||||
self.u2idx_free: dict[str, U2idx] = {}
|
||||
self.u2idx_n = 0
|
||||
self.clients = {}
|
||||
self.workload = 0
|
||||
self.workload_thr_alive = False
|
||||
self.cb_ts = 0
|
||||
self.cb_v = 0
|
||||
|
||||
env = jinja2.Environment()
|
||||
env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
|
||||
jn = ["splash", "svcs", "browser", "browser2", "msg", "md", "mde", "cf"]
|
||||
self.j2 = {x: env.get_template(x + ".html") for x in jn}
|
||||
zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
|
||||
self.prism = os.path.exists(zs)
|
||||
env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
|
||||
self.j2 = {
|
||||
x: env.get_template(x + ".html")
|
||||
for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
|
||||
}
|
||||
|
||||
self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
|
||||
if not self.args.no_dav:
|
||||
zs = "PROPFIND PROPPATCH LOCK UNLOCK MKCOL COPY MOVE"
|
||||
self.mallow += zs.split()
|
||||
|
||||
if self.args.zs:
|
||||
from .ssdp import SSDPr
|
||||
|
||||
self.ssdp = SSDPr(broker)
|
||||
|
||||
cert_path = self.args.cert
|
||||
if bos.path.exists(cert_path):
|
||||
cert_path = os.path.join(E.cfg, "cert.pem")
|
||||
if os.path.exists(cert_path):
|
||||
self.cert_path = cert_path
|
||||
else:
|
||||
self.cert_path = ""
|
||||
self.cert_path = None
|
||||
|
||||
if self.tp_q:
|
||||
self.start_threads(4)
|
||||
|
||||
if nid:
|
||||
if self.args.stackmon:
|
||||
start_stackmon(self.args.stackmon, nid)
|
||||
|
||||
if self.args.log_thrs:
|
||||
start_log_thrs(self.log, self.args.log_thrs, nid)
|
||||
|
||||
self.th_cfg: dict[str, Any] = {}
|
||||
Daemon(self.post_init, "hsrv-init2")
|
||||
|
||||
def post_init(self) -> None:
|
||||
try:
|
||||
x = self.broker.ask("thumbsrv.getcfg")
|
||||
self.th_cfg = x.get()
|
||||
except:
|
||||
pass
|
||||
|
||||
def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
|
||||
ips = set()
|
||||
for ip, _ in self.bound:
|
||||
ips.add(ip)
|
||||
|
||||
self.nm = NetMap(list(ips), netdevs)
|
||||
|
||||
def start_threads(self, n: int) -> None:
|
||||
self.tp_nthr += n
|
||||
if self.args.log_htp:
|
||||
self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
|
||||
|
||||
for _ in range(n):
|
||||
Daemon(self.thr_poolw, self.name + "-poolw")
|
||||
|
||||
def stop_threads(self, n: int) -> None:
|
||||
self.tp_nthr -= n
|
||||
if self.args.log_htp:
|
||||
self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
|
||||
|
||||
assert self.tp_q
|
||||
for _ in range(n):
|
||||
self.tp_q.put(None)
|
||||
|
||||
def periodic(self) -> None:
|
||||
while True:
|
||||
time.sleep(2 if self.tp_ncli or self.ncli else 10)
|
||||
with self.mutex:
|
||||
self.u2fh.clean()
|
||||
if self.tp_q:
|
||||
self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
|
||||
if self.tp_nthr > self.tp_ncli + 8:
|
||||
self.stop_threads(4)
|
||||
|
||||
if not self.ncli and not self.u2fh.cache and self.tp_nthr <= 8:
|
||||
self.t_periodic = None
|
||||
return
|
||||
|
||||
def listen(self, sck: socket.socket, nlisteners: int) -> None:
|
||||
if self.args.j != 1:
|
||||
# lost in the pickle; redefine
|
||||
if not ANYWIN or self.args.reuseaddr:
|
||||
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
|
||||
sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
sck.settimeout(None) # < does not inherit, ^ opts above do
|
||||
|
||||
ip, port = sck.getsockname()[:2]
|
||||
self.srvs.append(sck)
|
||||
self.bound.add((ip, port))
|
||||
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
|
||||
Daemon(
|
||||
self.thr_listen,
|
||||
"httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
|
||||
(sck,),
|
||||
)
|
||||
|
||||
def thr_listen(self, srv_sck: socket.socket) -> None:
|
||||
"""listens on a shared tcp server"""
|
||||
ip, port = srv_sck.getsockname()[:2]
|
||||
fno = srv_sck.fileno()
|
||||
hip = "[{}]".format(ip) if ":" in ip else ip
|
||||
msg = "subscribed @ {}:{} f{} p{}".format(hip, port, fno, os.getpid())
|
||||
self.log(self.name, msg)
|
||||
|
||||
def fun() -> None:
|
||||
self.broker.say("cb_httpsrv_up")
|
||||
|
||||
threading.Thread(target=fun, name="sig-hsrv-up1").start()
|
||||
|
||||
while not self.stopping:
|
||||
if self.args.log_conn:
|
||||
self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90")
|
||||
|
||||
spins = 0
|
||||
while self.ncli >= self.nclimax:
|
||||
if not spins:
|
||||
self.log(self.name, "at connection limit; waiting", 3)
|
||||
|
||||
spins += 1
|
||||
time.sleep(0.1)
|
||||
if spins != 50 or not self.args.aclose:
|
||||
continue
|
||||
|
||||
ipfreq: dict[str, int] = {}
|
||||
with self.mutex:
|
||||
for c in self.clients:
|
||||
ip = ipnorm(c.ip)
|
||||
try:
|
||||
ipfreq[ip] += 1
|
||||
except:
|
||||
ipfreq[ip] = 1
|
||||
|
||||
ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0]
|
||||
if n < self.nclimax / 2:
|
||||
continue
|
||||
|
||||
self.aclose[ip] = int(time.time() + self.args.aclose * 60)
|
||||
nclose = 0
|
||||
nloris = 0
|
||||
nconn = 0
|
||||
with self.mutex:
|
||||
for c in self.clients:
|
||||
cip = ipnorm(c.ip)
|
||||
if ip != cip:
|
||||
continue
|
||||
|
||||
nconn += 1
|
||||
try:
|
||||
if (
|
||||
c.nreq >= 1
|
||||
or not c.cli
|
||||
or c.cli.in_hdr_recv
|
||||
or c.cli.keepalive
|
||||
):
|
||||
Daemon(c.shutdown)
|
||||
nclose += 1
|
||||
if c.nreq <= 0 and (not c.cli or c.cli.in_hdr_recv):
|
||||
nloris += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
t = "{} downgraded to connection:close for {} min; dropped {}/{} connections"
|
||||
self.log(self.name, t.format(ip, self.args.aclose, nclose, nconn), 1)
|
||||
|
||||
if nloris < nconn / 2:
|
||||
continue
|
||||
|
||||
t = "slowloris (idle-conn): {} banned for {} min"
|
||||
self.log(self.name, t.format(ip, self.args.loris, nclose), 1)
|
||||
self.bans[ip] = int(time.time() + self.args.loris * 60)
|
||||
|
||||
if self.args.log_conn:
|
||||
self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90")
|
||||
|
||||
try:
|
||||
sck, saddr = srv_sck.accept()
|
||||
cip, cport = saddr[:2]
|
||||
if cip.startswith("::ffff:"):
|
||||
cip = cip[7:]
|
||||
|
||||
addr = (cip, cport)
|
||||
except (OSError, socket.error) as ex:
|
||||
if self.stopping:
|
||||
break
|
||||
|
||||
self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
|
||||
time.sleep(0.02)
|
||||
continue
|
||||
|
||||
if self.args.log_conn:
|
||||
t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
|
||||
"-" * 3, ip, port % 8, port
|
||||
)
|
||||
self.log("%s %s" % addr, t, c="90")
|
||||
|
||||
self.accept(sck, addr)
|
||||
|
||||
def accept(self, sck: socket.socket, addr: tuple[str, int]) -> None:
|
||||
def accept(self, sck, addr):
|
||||
"""takes an incoming tcp connection and creates a thread to handle it"""
|
||||
now = time.time()
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
|
||||
|
||||
if now - (self.tp_time or now) > 300:
|
||||
t = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
|
||||
self.log(self.name, t.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
|
||||
self.tp_time = 0
|
||||
self.tp_q = None
|
||||
|
||||
with self.mutex:
|
||||
self.ncli += 1
|
||||
if not self.t_periodic:
|
||||
name = "hsrv-pt"
|
||||
if self.nid:
|
||||
name += "-{}".format(self.nid)
|
||||
|
||||
self.t_periodic = Daemon(self.periodic, name)
|
||||
|
||||
if self.tp_q:
|
||||
self.tp_time = self.tp_time or now
|
||||
self.tp_ncli = max(self.tp_ncli, self.ncli)
|
||||
if self.tp_nthr < self.ncli + 4:
|
||||
self.start_threads(8)
|
||||
|
||||
self.tp_q.put((sck, addr))
|
||||
return
|
||||
|
||||
if not self.args.no_htp:
|
||||
t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
|
||||
self.log(self.name, t, 1)
|
||||
|
||||
Daemon(
|
||||
self.thr_client,
|
||||
"httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
|
||||
(sck, addr),
|
||||
thr = threading.Thread(
|
||||
target=self.thr_client,
|
||||
args=(sck, addr),
|
||||
name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def thr_poolw(self) -> None:
|
||||
assert self.tp_q
|
||||
while True:
|
||||
task = self.tp_q.get()
|
||||
if not task:
|
||||
break
|
||||
|
||||
with self.mutex:
|
||||
self.tp_time = 0
|
||||
def num_clients(self):
|
||||
with self.mutex:
|
||||
return len(self.clients)
|
||||
|
||||
def shutdown(self):
|
||||
clients = list(self.clients.keys())
|
||||
for cli in clients:
|
||||
try:
|
||||
sck, addr = task
|
||||
me = threading.current_thread()
|
||||
me.name = "httpconn-{}-{}".format(
|
||||
addr[0].split(".", 2)[-1][-6:], addr[1]
|
||||
)
|
||||
self.thr_client(sck, addr)
|
||||
me.name = self.name + "-poolw"
|
||||
except Exception as ex:
|
||||
if str(ex).startswith("client d/c "):
|
||||
self.log(self.name, "thr_client: " + str(ex), 6)
|
||||
else:
|
||||
self.log(self.name, "thr_client: " + min_ex(), 3)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.stopping = True
|
||||
for srv in self.srvs:
|
||||
try:
|
||||
srv.close()
|
||||
cli.shutdown()
|
||||
except:
|
||||
pass
|
||||
|
||||
thrs = []
|
||||
clients = list(self.clients)
|
||||
for cli in clients:
|
||||
t = threading.Thread(target=cli.shutdown)
|
||||
thrs.append(t)
|
||||
t.start()
|
||||
self.log("httpsrv-n", "ok bye")
|
||||
|
||||
if self.tp_q:
|
||||
self.stop_threads(self.tp_nthr)
|
||||
for _ in range(10):
|
||||
time.sleep(0.05)
|
||||
if self.tp_q.empty():
|
||||
break
|
||||
|
||||
for t in thrs:
|
||||
t.join()
|
||||
|
||||
self.log(self.name, "ok bye")
|
||||
|
||||
def thr_client(self, sck: socket.socket, addr: tuple[str, int]) -> None:
|
||||
def thr_client(self, sck, addr):
|
||||
"""thread managing one tcp client"""
|
||||
sck.settimeout(120)
|
||||
|
||||
cli = HttpConn(sck, addr, self)
|
||||
with self.mutex:
|
||||
self.clients.add(cli)
|
||||
self.clients[cli] = 0
|
||||
|
||||
if self.is_mp:
|
||||
self.workload += 50
|
||||
if not self.workload_thr_alive:
|
||||
self.workload_thr_alive = True
|
||||
thr = threading.Thread(
|
||||
target=self.thr_workload, name="httpsrv-workload"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
# print("{}\n".format(len(self.clients)), end="")
|
||||
fno = sck.fileno()
|
||||
try:
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="90")
|
||||
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
|
||||
|
||||
cli.run()
|
||||
|
||||
except (OSError, socket.error) as ex:
|
||||
if ex.errno not in E_SCK:
|
||||
if ex.errno not in [10038, 10054, 107, 57, 9]:
|
||||
self.log(
|
||||
"%s %s" % addr,
|
||||
"run({}): {}".format(fno, ex),
|
||||
@@ -430,29 +128,61 @@ class HttpSrv(object):
|
||||
finally:
|
||||
sck = cli.s
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="90")
|
||||
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
|
||||
|
||||
try:
|
||||
fno = sck.fileno()
|
||||
shut_socket(cli.log, sck)
|
||||
sck.shutdown(socket.SHUT_RDWR)
|
||||
sck.close()
|
||||
except (OSError, socket.error) as ex:
|
||||
if not MACOS:
|
||||
self.log(
|
||||
"%s %s" % addr,
|
||||
"shut({}): {}".format(fno, ex),
|
||||
c="90",
|
||||
c="1;30",
|
||||
)
|
||||
if ex.errno not in E_SCK:
|
||||
if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
|
||||
# 10038 No longer considered a socket
|
||||
# 10054 Foribly closed by remote
|
||||
# 107 Transport endpoint not connected
|
||||
# 57 Socket is not connected
|
||||
# 49 Can't assign requested address (wifi down)
|
||||
# 9 Bad file descriptor
|
||||
raise
|
||||
finally:
|
||||
with self.mutex:
|
||||
self.clients.remove(cli)
|
||||
self.ncli -= 1
|
||||
del self.clients[cli]
|
||||
|
||||
if cli.u2idx:
|
||||
self.put_u2idx(str(addr), cli.u2idx)
|
||||
if self.disconnect_func:
|
||||
self.disconnect_func(addr) # pylint: disable=not-callable
|
||||
|
||||
def cachebuster(self) -> str:
|
||||
def thr_workload(self):
|
||||
"""indicates the python interpreter workload caused by this HttpSrv"""
|
||||
# avoid locking in extract_filedata by tracking difference here
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
with self.mutex:
|
||||
if not self.clients:
|
||||
# no clients rn, termiante thread
|
||||
self.workload_thr_alive = False
|
||||
self.workload = 0
|
||||
return
|
||||
|
||||
total = 0
|
||||
with self.mutex:
|
||||
for cli in self.clients.keys():
|
||||
now = cli.workload
|
||||
delta = now - self.clients[cli]
|
||||
if delta < 0:
|
||||
# was reset in HttpCli to prevent overflow
|
||||
delta = now
|
||||
|
||||
total += delta
|
||||
self.clients[cli] = now
|
||||
|
||||
self.workload = total
|
||||
|
||||
def cachebuster(self):
|
||||
if time.time() - self.cb_ts < 1:
|
||||
return self.cb_v
|
||||
|
||||
@@ -460,44 +190,16 @@ class HttpSrv(object):
|
||||
if time.time() - self.cb_ts < 1:
|
||||
return self.cb_v
|
||||
|
||||
v = self.E.t0
|
||||
v = E.t0
|
||||
try:
|
||||
with os.scandir(os.path.join(self.E.mod, "web")) as dh:
|
||||
with os.scandir(os.path.join(E.mod, "web")) as dh:
|
||||
for fh in dh:
|
||||
inf = fh.stat()
|
||||
inf = fh.stat(follow_symlinks=False)
|
||||
v = max(v, inf.st_mtime)
|
||||
except:
|
||||
pass
|
||||
|
||||
v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
|
||||
v = base64.urlsafe_b64encode(struct.pack(">xxL", int(v)))
|
||||
self.cb_v = v.decode("ascii")[-4:]
|
||||
self.cb_ts = time.time()
|
||||
return self.cb_v
|
||||
|
||||
def get_u2idx(self, ident: str) -> Optional[U2idx]:
|
||||
utab = self.u2idx_free
|
||||
for _ in range(100): # 5/0.05 = 5sec
|
||||
with self.mutex:
|
||||
if utab:
|
||||
if ident in utab:
|
||||
return utab.pop(ident)
|
||||
|
||||
return utab.pop(list(utab.keys())[0])
|
||||
|
||||
if self.u2idx_n < CORES:
|
||||
self.u2idx_n += 1
|
||||
return U2idx(self)
|
||||
|
||||
time.sleep(0.05)
|
||||
# not using conditional waits, on a hunch that
|
||||
# average performance will be faster like this
|
||||
# since most servers won't be fully saturated
|
||||
|
||||
return None
|
||||
|
||||
def put_u2idx(self, ident: str, u2idx: U2idx) -> None:
|
||||
with self.mutex:
|
||||
while ident in self.u2idx_free:
|
||||
ident += "a"
|
||||
|
||||
self.u2idx_free[ident] = u2idx
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user