Compare commits

...

94 Commits

Author SHA1 Message Date
ed
02910b0020 v0.11.45 2021-07-20 23:23:08 +02:00
ed
23b8901c9c include localstore on the crashpage 2021-07-20 23:22:35 +02:00
ed
99f6ed0cd7 up2k-cli: avoid loading sha.js multiple times 2021-07-20 23:14:30 +02:00
ed
890c310880 another attempt at fixing tooltips on iphone 2021-07-20 23:07:15 +02:00
ed
0194eeb31f add login/permissions indicator 2021-07-20 22:42:03 +02:00
ed
f9be4c62b1 v0.11.44 2021-07-20 01:03:08 +02:00
ed
027e8c18f1 sfx: option to remove mouse cursor 2021-07-20 01:00:28 +02:00
ed
4a3bb35a95 sfx: option to remove scp.woff2 2021-07-20 00:45:54 +02:00
ed
4bfb0d4494 notes 2021-07-19 23:46:44 +02:00
ed
7e0ef03a1e fix audio player edgecase (continue into next folder with sidebar closed) 2021-07-19 23:10:48 +02:00
ed
f7dbd95a54 v0.11.43 2021-07-19 01:56:19 +02:00
ed
515ee2290b v0.11.42 2021-07-18 23:22:09 +02:00
ed
b0c78910bb fix tabchange triggering tooltips 2021-07-18 23:21:36 +02:00
ed
f4ca62b664 reattach tooltips on column show/hide 2021-07-18 23:14:57 +02:00
ed
8eb8043a3d fix 3rdparty namecase 2021-07-18 22:50:29 +02:00
ed
3e8541362a keep active dir scrolled into view on keybd nav 2021-07-18 22:32:34 +02:00
ed
789724e348 use preferred key notation in search results 2021-07-18 21:50:57 +02:00
ed
5125b9532f fix multiple whitespace in query translator 2021-07-18 21:39:28 +02:00
ed
ebc9de02b0 case-insensitive tag search 2021-07-18 21:34:36 +02:00
ed
ec788fa491 mutagen fixes:
* extract codec and format info
* add FFprobe as fallback when mutagen fails
* add option to blacklist FFprobe for tags
2021-07-18 19:57:31 +02:00
ed
9b5e264574 systemd: fix name in journalctl 2021-07-17 19:14:15 +02:00
ed
57c297274b v0.11.41 2021-07-17 17:53:34 +02:00
ed
e9bf092317 tweak audio drawer tab 2021-07-17 17:24:48 +02:00
ed
d173887324 explain confusing behavior in journalctl 2021-07-17 16:45:49 +02:00
ed
99820d854c oh that wasnt enough ok then 2021-07-17 16:45:25 +02:00
ed
62df0a0eb2 thx osx 2021-07-17 16:43:22 +02:00
ed
600e9ac947 try to workaround iphones not hiding tooltips 2021-07-17 16:03:21 +02:00
ed
3ca41be2b4 do up2k snapshot on shutdown 2021-07-17 14:48:35 +02:00
ed
5c7debd900 improve signal handling + emit sd-notify on start 2021-07-17 04:15:07 +02:00
ed
7fa5b23ce3 sfx: fix color bleed on flock errors 2021-07-17 04:12:14 +02:00
ed
ff82738aaf vscode: support whitespace in python binary path 2021-07-17 04:11:14 +02:00
ed
bf5ee9d643 colum header tooltips 2021-07-17 02:52:55 +02:00
ed
72a8593ecd gridmode shortcut in the audio drawer 2021-07-17 01:45:05 +02:00
ed
bc3bbe07d4 combine tabs on narrow screens 2021-07-17 01:21:49 +02:00
ed
c7cb64bfef gallery: add hotkey list button 2021-07-17 01:14:14 +02:00
ed
629f537d06 add more hotkey tooltips 2021-07-17 01:05:26 +02:00
ed
9e988041b8 cosmetics 2021-07-16 02:56:21 +02:00
ed
f9a8b5c9d7 update readme 2021-07-16 02:44:06 +02:00
ed
b9c3538253 nope, not doing this 2021-07-15 23:49:30 +02:00
ed
2bc0cdf017 fix md-editor hotkeys on dvorak 2021-07-15 23:24:10 +02:00
ed
02a91f60d4 playing some golf 2021-07-15 23:19:37 +02:00
ed
fae83da197 v0.11.40 2021-07-15 01:13:15 +02:00
ed
0fe4aa6418 ux tweaks 2021-07-15 01:04:38 +02:00
ed
21a51bf0dc make it feel like home 2021-07-15 00:50:43 +02:00
ed
bcb353cc30 allow ctrl-clicking primary tabs 2021-07-15 00:37:14 +02:00
ed
6af4508518 adjust the sfx edit warning 2021-07-15 00:26:33 +02:00
ed
6a559bc28a gallery: dispose videos to stop buffering 2021-07-15 00:22:26 +02:00
ed
0f5026cd20 gallery: option to autoplay next video on end 2021-07-15 00:04:33 +02:00
ed
a91b80a311 gallery: add video loop hotkey R 2021-07-14 09:42:38 +02:00
ed
ec534701c8 gallery: pause/resume audio player on video 2021-07-14 09:40:12 +02:00
ed
af5169f67f gallery: fix hotkeys + focus 2021-07-14 09:35:50 +02:00
ed
18676c5e65 better crash page 2021-07-14 09:34:42 +02:00
ed
e2df6fda7b update hotkeys 2021-07-13 02:20:52 +02:00
ed
e9ae9782fe v0.11.39 2021-07-13 00:54:23 +02:00
ed
016dba4ca9 v0.11.38 2021-07-13 00:35:34 +02:00
ed
39c7ef305f add a link to clear settings on the js crash page 2021-07-13 00:33:46 +02:00
ed
849c1dc848 video-player: add hotkeys m=mute, f=fullscreen 2021-07-13 00:23:48 +02:00
ed
61414014fe gallery: fix link overlapping image 2021-07-13 00:14:06 +02:00
ed
578a915884 stack/thread monitors in mpw + better thread names 2021-07-12 23:03:52 +02:00
ed
eacafb8a63 add option to log summary of running threads 2021-07-12 22:57:37 +02:00
ed
4446760f74 fix link to ?stack on rootless configs 2021-07-12 22:55:38 +02:00
ed
6da2a083f9 v0.11.37 2021-07-12 00:51:59 +02:00
ed
8837c8f822 print zip/tar errors to log 2021-07-12 00:47:22 +02:00
ed
bac301ed66 get rid of iffy default-args 2021-07-12 00:15:13 +02:00
ed
061db3906d v0.11.36 2021-07-11 06:39:58 +02:00
ed
fd7df5c952 v0.11.35 2021-07-11 06:22:56 +02:00
ed
a270019147 easier to tell youre trying to watch a video that firefox cant deal with 2021-07-11 06:21:25 +02:00
ed
55e0209901 add video-player keybinds 2021-07-11 06:12:24 +02:00
ed
2b255fbbed add in-gallery video playback 2021-07-11 03:25:46 +02:00
ed
8a2345a0fb top of the sandwich fell off 2021-07-11 02:06:18 +02:00
ed
bfa9f535aa more context in exceptions 2021-07-11 01:59:07 +02:00
ed
f757623ad8 make bdmv thumbnails 2021-07-09 20:09:32 +02:00
ed
3c7465e268 option to disable thumbcache eviction 2021-07-09 19:55:17 +02:00
ed
108665fc4f v0.11.34 2021-07-09 17:12:21 +02:00
ed
ed519c9138 add performance notes 2021-07-09 17:10:37 +02:00
ed
2dd2e2c57e discard logs in mpw 2021-07-09 17:01:11 +02:00
ed
6c3a976222 scale max-clients to mp-workers 2021-07-09 16:48:02 +02:00
ed
80cc26bd95 fix max-client limit 2021-07-09 16:33:11 +02:00
ed
970fb84fd8 hex looks better 2021-07-09 16:11:33 +02:00
ed
20cbcf6931 logging + shutdown cleanup 2021-07-09 16:07:16 +02:00
ed
8fcde2a579 move tcp accept into mp-worker 2021-07-09 15:49:36 +02:00
ed
b32d1f8ad3 make ?stack work anywhere 2021-07-09 13:46:42 +02:00
ed
03513e0cb1 effectively pointless but cool 2021-07-09 03:41:44 +02:00
ed
e041a2b197 fix centos7 support 2021-07-08 23:35:28 +02:00
ed
d7d625be2a v0.11.33 2021-07-07 10:45:47 +02:00
ed
4121266678 v0.11.32 2021-07-06 21:58:03 +02:00
ed
22971a6be4 up2k-cli: add turbo button 2021-07-06 21:43:07 +02:00
ed
efbf8d7e0d better handling of invalid requests 2021-07-06 01:03:09 +02:00
ed
397396ea4a apply -nw to PUT uploads too 2021-07-06 00:49:39 +02:00
ed
e59b077c21 announce the rotates 2021-07-06 00:43:37 +02:00
ed
4bc39f3084 add logrotate 2021-07-06 00:23:51 +02:00
ed
21c3570786 detect more recursive symlinks 2021-07-05 23:50:03 +02:00
ed
2f85c1fb18 add logging to file 2021-07-05 23:30:33 +02:00
ed
1e27a4c2df make thumb-dir.txt unretrievable 2021-07-05 00:21:33 +02:00
44 changed files with 1893 additions and 966 deletions

5
.vscode/tasks.json vendored
View File

@@ -9,7 +9,10 @@
{
"label": "no_dbg",
"type": "shell",
"command": "${config:python.pythonPath} .vscode/launch.py"
"command": "${config:python.pythonPath}",
"args": [
".vscode/launch.py"
]
}
]
}

141
README.md
View File

@@ -46,6 +46,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
* [browser support](#browser-support)
* [client examples](#client-examples)
* [up2k](#up2k)
* [performance](#performance)
* [dependencies](#dependencies)
* [optional dependencies](#optional-dependencies)
* [install recommended deps](#install-recommended-deps)
@@ -58,6 +59,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
* [just the sfx](#just-the-sfx)
* [complete release](#complete-release)
* [todo](#todo)
* [discarded ideas](#discarded-ideas)
## quickstart
@@ -68,7 +70,7 @@ running the sfx without arguments (for example doubleclicking it on Windows) wil
some recommended options:
* `-e2dsa` enables general file indexing, see [search configuration](#search-configuration)
* `-e2ts` enables audio metadata indexing (needs either FFprobe or mutagen), see [optional dependencies](#optional-dependencies)
* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
* `-v /mnt/music:/music:r:afoo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, with user `foo` as `a`dmin (read/write), password `bar`
* the syntax is `-v src:dst:perm:perm:...` so local-path, url-path, and one or more permissions to set
* replace `:r:afoo` with `:rfoo` to only make the folder readable by `foo` and nobody else
@@ -113,7 +115,7 @@ summary: all planned features work! now please enjoy the bloatening
* backend stuff
* ☑ sanic multipart parser
*load balancer (multiprocessing)
*multiprocessing (actual multithreading)
* ☑ volumes (mountpoints)
* ☑ accounts
* upload
@@ -129,10 +131,10 @@ summary: all planned features work! now please enjoy the bloatening
* ☑ tree-view
* ☑ audio player (with OS media controls)
* ☑ thumbnails
* ☑ images using Pillow
* ☑ videos using FFmpeg
* ...of images using Pillow
* ...of videos using FFmpeg
* ☑ cache eviction (max-age; maybe max-size eventually)
* ☑ image gallery
* ☑ image gallery with webm player
* ☑ SPA (browse while uploading)
* if you use the file-tree on the left only, not folders in the file list
* server indexing
@@ -153,10 +155,10 @@ small collection of user feedback
# bugs
* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
* Windows: python 3.7 and older cannot read tags with FFprobe, so use Mutagen or upgrade
* Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
* Windows: python 2.7 cannot handle filenames with mojibake
* MacOS: `--th-ff-jpg` may fix thumbnails using macports-FFmpeg
* `--th-ff-jpg` may fix video thumbnails on some FFmpeg versions
## general bugs
@@ -171,9 +173,12 @@ small collection of user feedback
* Windows: folders cannot be accessed if the name ends with `.`
* python or windows bug
* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
* Windows: msys2-python 3.8.6 occasionally throws `RuntimeError: release unlocked lock` when leaving a scoped mutex in up2k
* this is an msys2 bug, the regular windows edition of python is fine
* VirtualBox: sqlite throws `Disk I/O Error` when running in a VM and the up2k database is in a vboxsf
* use `--hist` or the `hist` volflag (`-v [...]:chist=/tmp/foo`) to place the db inside the vm instead
# the browser
@@ -187,28 +192,46 @@ small collection of user feedback
* `[📂]` mkdir, create directories
* `[📝]` new-md, create a new markdown document
* `[📟]` send-msg, either to server-log or into textfiles if `--urlform save`
* `[⚙️]` client configuration options
* `[🎺]` audio-player config options
* `[⚙️]` general client config options
## hotkeys
the browser has the following hotkeys
the browser has the following hotkeys (assumes qwerty, ignores actual layout)
* `B` toggle breadcrumbs / directory tree
* `I/K` prev/next folder
* `M` parent folder
* `M` parent folder (or unexpand current)
* `G` toggle list / grid view
* `T` toggle thumbnails / icons
* when playing audio:
* `0..9` jump to 10%..90%
* `U/O` skip 10sec back/forward
* `J/L` prev/next song
* `U/O` skip 10sec back/forward
* `0..9` jump to 0%..90%
* `P` play/pause (also starts playing the folder)
* when viewing images / playing videos:
* `J/L, Left/Right` prev/next file
* `Home/End` first/last file
* `Esc` close viewer
* videos:
* `U/O` skip 10sec back/forward
* `P/K/Space` play/pause
* `F` fullscreen
* `C` continue playing next video
* `R` loop
* `M` mute
* when tree-sidebar is open:
* `A/D` adjust tree width
* in the grid view:
* `S` toggle multiselect
* shift+`A/D` zoom
* in the markdown editor:
* `^s` save
* `^h` header
* `^k` autoformat table
* `^u` jump to next unicode character
* `^e` toggle editor / preview
* `^up, ^down` jump paragraphs
## tree-mode
@@ -221,9 +244,9 @@ click `[-]` and `[+]` (or hotkeys `A`/`D`) to adjust the size, and the `[a]` tog
![copyparty-thumbs-fs8](https://user-images.githubusercontent.com/241032/120070302-10836b00-c08a-11eb-8eb4-82004a34c342.png)
it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how destructive your users are
it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how dangerous your users are
images named `folder.jpg` and `folder.png` become the thumbnail of the folder they're in
images with the following names (see `--th-covers`) become the thumbnail of the folder they're in: `folder.png`, `folder.jpg`, `cover.png`, `cover.jpg`
in the grid/thumbnail view, if the audio player panel is open, songs will start playing when clicked
@@ -240,9 +263,10 @@ the `zip` link next to folders can produce various types of zip/tar files using
| `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
* hidden files (dotfiles) are excluded unless `-ed`
* the up2k.db is always excluded
* `up2k.db` and `dir.txt` is always excluded
* `zip_crc` will take longer to download since the server has to read each file twice
* please let me know if you find a program old enough to actually need this
* this is only to support MS-DOS PKZIP v2.04g (october 1993) and older
* how are you accessing copyparty actually
you can also zip a selection of files or folders by clicking them in the browser, that brings up a selection editor and zip button in the bottom right
@@ -257,9 +281,11 @@ two upload methods are available in the html client:
up2k has several advantages:
* you can drop folders into the browser (files are added recursively)
* files are processed in chunks, and each chunk is checksummed
* uploads resume if they are interrupted (for example by a reboot)
* uploads autoresume if they are interrupted by network issues
* uploads resume if you reboot your browser or pc, just upload the same files again
* server detects any corruption; the client reuploads affected chunks
* the client doesn't upload anything that already exists on the server
* much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
* the last-modified timestamp of the file is preserved
see [up2k](#up2k) for details on how it works
@@ -292,11 +318,11 @@ in the `[🚀 up2k]` tab, after toggling the `[🔎]` switch green, any files/fo
files go into `[ok]` if they exist (and you get a link to where it is), otherwise they land in `[ng]`
* the main reason filesearch is combined with the uploader is cause the code was too spaghetti to separate it out somewhere else, this is no longer the case but now i've warmed up to the idea too much
adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files
adding the same file multiple times is blocked, so if you first search for a file and then decide to upload it, you have to click the `[cleanup]` button to discard `[done]` files (or just refresh the page)
note that since up2k has to read the file twice, `[🎈 bup]` can be up to 2x faster in extreme cases (if your internet connection is faster than the read-speed of your HDD)
up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well thanks to tls also functioning as an integrity check
up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
## markdown viewer
@@ -334,11 +360,11 @@ searching relies on two databases, the up2k filetree (`-e2d`) and the metadata t
through arguments:
* `-e2d` enables file indexing on upload
* `-e2ds` scans writable folders on startup
* `-e2ds` scans writable folders for new files on startup
* `-e2dsa` scans all mounted volumes (including readonly ones)
* `-e2t` enables metadata indexing on upload
* `-e2ts` scans for tags in all files that don't have tags yet
* `-e2tsr` deletes all existing tags, so a full reindex
* `-e2tsr` deletes all existing tags, does a full reindex
the same arguments can be set as volume flags, in addition to `d2d` and `d2t` for disabling:
* `-v ~/music::r:ce2dsa:ce2tsr` does a full reindex of everything on startup
@@ -346,11 +372,11 @@ the same arguments can be set as volume flags, in addition to `d2d` and `d2t` fo
* `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
note:
* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those
* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those, unless there is a new copyparty version with new parsers and the release note says otherwise
* the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
you can choose to only index filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash` or the volume-flag `cdhash`, this has the following consequences:
* initial indexing is way faster, especially when the volume is on a networked disk
* initial indexing is way faster, especially when the volume is on a network disk
* makes it impossible to [file-search](#file-search)
* if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
@@ -383,17 +409,17 @@ tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric val
see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
`--no-mutagen` disables mutagen and uses ffprobe instead, which...
* is about 20x slower than mutagen
* catches a few tags that mutagen doesn't
`--no-mutagen` disables Mutagen and uses FFprobe instead, which...
* is about 20x slower than Mutagen
* catches a few tags that Mutagen doesn't
* melodic key, video resolution, framerate, pixfmt
* avoids pulling any GPL code into copyparty
* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
* more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve
## file parser plugins
copyparty can invoke external programs to collect additional metadata for files using `mtp` (as argument or volume flag), there is a default timeout of 30sec
copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volume flag), there is a default timeout of 30sec
* `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
* `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
@@ -427,11 +453,13 @@ copyparty can invoke external programs to collect additional metadata for files
| zip selection | - | yep | yep | yep | yep | yep | yep | yep |
| directory tree | - | - | `*1` | yep | yep | yep | yep | yep |
| up2k | - | - | yep | yep | yep | yep | yep | yep |
| icons work | - | - | yep | yep | yep | yep | yep | yep |
| markdown editor | - | - | yep | yep | yep | yep | yep | yep |
| markdown viewer | - | - | yep | yep | yep | yep | yep | yep |
| play mp3/m4a | - | yep | yep | yep | yep | yep | yep | yep |
| play ogg/opus | - | - | - | - | yep | yep | `*2` | yep |
| thumbnail view | - | - | - | - | yep | yep | yep | yep |
| image viewer | - | - | - | - | yep | yep | yep | yep |
| **= feature =** | ie6 | ie9 | ie10 | ie11 | ff 52 | c 49 | iOS | Andr |
* internet explorer 6 to 8 behave the same
* firefox 52 and chrome 49 are the last winxp versions
@@ -449,7 +477,7 @@ quick summary of more eccentric web-browsers trying to view a directory index:
| **w3m** (0.5.3/macports) | can browse, login, upload at 100kB/s, mkdir/msg |
| **netsurf** (3.10/arch) | is basically ie6 with much better css (javascript has almost no effect) |
| **ie4** and **netscape** 4.0 | can browse (text is yellow on white), upload with `?b=u` |
| **SerenityOS** (22d13d8) | hits a page fault, works with `?b=u`, file input not-impl, url params are multiplying |
| **SerenityOS** (7e98457) | hits a page fault, works with `?b=u`, file upload not-impl |
# client examples
@@ -494,6 +522,23 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
# performance
defaults are good for most cases, don't mind the `cannot efficiently use multiple CPU cores` message, it's very unlikely to be a problem
below are some tweaks roughly ordered by usefulness:
* `-q` disables logging and can help a bunch, even when combined with `-lo` to redirect logs to file
* `--http-only` or `--https-only` (unless you want to support both protocols) will reduce the delay before a new connection is established
* `--hist` pointing to a fast location (ssd) will make directory listings and searches faster when `-e2d` or `-e2t` is set
* `--no-hash` when indexing a network-disk if you don't care about the actual filehashes and only want the names/tags searchable
* `-j` enables multiprocessing (actual multithreading) and can make copyparty perform better in cpu-intensive workloads, for example:
* huge amount of short-lived connections
* really heavy traffic (downloads/uploads)
...however it adds an overhead to internal communication so it might be a net loss, see if it works 4 u
# dependencies
* `jinja2` (is built into the SFX)
@@ -503,18 +548,18 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
enable music tags:
* either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
* or `ffprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
enable image thumbnails:
enable thumbnails of images:
* `Pillow` (requires py2.7 or py3.5+)
enable video thumbnails:
enable thumbnails of videos:
* `ffmpeg` and `ffprobe` somewhere in `$PATH`
enable reading HEIF pictures:
enable thumbnails of HEIF pictures:
* `pyheif-pillow-opener` (requires Linux or a C compiler)
enable reading AVIF pictures:
enable thumbnails of AVIF pictures:
* `pillow-avif-plugin`
@@ -528,7 +573,7 @@ python -m pip install --user -U jinja2 mutagen Pillow
some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)
these are standalone programs and will never be imported / evaluated by copyparty
these are standalone programs and will never be imported / evaluated by copyparty, and must be enabled through `-mtp` configs
# sfx
@@ -544,10 +589,10 @@ pls note that `copyparty-sfx.sh` will fail if you rename `copyparty-sfx.py` to `
## sfx repack
if you don't need all the features you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except for either msys2 or WSL if you're on windows)
* `724K` original size as of v0.4.0
* `256K` after `./scripts/make-sfx.sh re no-ogv`
* `164K` after `./scripts/make-sfx.sh re no-ogv no-cm`
if you don't need all the features, you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except if you're on windows then you need msys2 or WSL)
* `525k` size of original sfx.py as of v0.11.30
* `315k` after `./scripts/make-sfx.sh re no-ogv`
* `223k` after `./scripts/make-sfx.sh re no-ogv no-cm`
the features you can opt to drop are
* `ogv`.js, the opus/vorbis decoder which is needed by apple devices to play foss audio files
@@ -594,7 +639,7 @@ rm -rf copyparty/web/deps
curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
python3 x.py -h
rm x.py
mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/
mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
```
then build the sfx using any of the following examples:
@@ -622,14 +667,16 @@ in the `scripts` folder:
roughly sorted by priority
* hls framework for Someone Else to drop code into :^)
* readme.md as epilogue
## discarded ideas
* reduce up2k roundtrips
* start from a chunk index and just go
* terminate client on bad data
* logging to file
discarded ideas
* not worth the effort, just throw enough conncetions at it
* single sha512 across all up2k chunks?
* crypto.subtle cannot into streaming, would have to use hashwasm, expensive
* separate sqlite table per tag

View File

@@ -345,7 +345,7 @@ class Gateway(object):
except:
pass
def sendreq(self, *args, headers={}, **kwargs):
def sendreq(self, meth, path, headers, **kwargs):
if self.password:
headers["Cookie"] = "=".join(["cppwd", self.password])
@@ -354,21 +354,21 @@ class Gateway(object):
if c.rx_path:
raise Exception()
c.request(*list(args), headers=headers, **kwargs)
c.request(meth, path, headers=headers, **kwargs)
c.rx = c.getresponse()
return c
except:
tid = threading.current_thread().ident
dbg(
"\033[1;37;44mbad conn {:x}\n {}\n {}\033[0m".format(
tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
"\033[1;37;44mbad conn {:x}\n {} {}\n {}\033[0m".format(
tid, meth, path, c.rx_path if c else "(null)"
)
)
self.closeconn(c)
c = self.getconn()
try:
c.request(*list(args), headers=headers, **kwargs)
c.request(meth, path, headers=headers, **kwargs)
c.rx = c.getresponse()
return c
except:
@@ -386,7 +386,7 @@ class Gateway(object):
path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
c = self.sendreq("GET", web_path)
c = self.sendreq("GET", web_path, {})
if c.rx.status != 200:
self.closeconn(c)
log(
@@ -440,7 +440,7 @@ class Gateway(object):
)
)
c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
c = self.sendreq("GET", web_path, {"Range": hdr_range})
if c.rx.status != http.client.PARTIAL_CONTENT:
self.closeconn(c)
raise Exception(

View File

@@ -54,10 +54,13 @@ MACOS = platform.system() == "Darwin"
info = log = dbg = None
print("{} v{} @ {}".format(
platform.python_implementation(),
".".join([str(x) for x in sys.version_info]),
sys.executable))
print(
"{} v{} @ {}".format(
platform.python_implementation(),
".".join([str(x) for x in sys.version_info]),
sys.executable,
)
)
try:
@@ -299,14 +302,14 @@ class Gateway(object):
except:
pass
def sendreq(self, *args, headers={}, **kwargs):
def sendreq(self, meth, path, headers, **kwargs):
tid = get_tid()
if self.password:
headers["Cookie"] = "=".join(["cppwd", self.password])
try:
c = self.getconn(tid)
c.request(*list(args), headers=headers, **kwargs)
c.request(meth, path, headers=headers, **kwargs)
return c.getresponse()
except:
dbg("bad conn")
@@ -314,7 +317,7 @@ class Gateway(object):
self.closeconn(tid)
try:
c = self.getconn(tid)
c.request(*list(args), headers=headers, **kwargs)
c.request(meth, path, headers=headers, **kwargs)
return c.getresponse()
except:
info("http connection failed:\n" + traceback.format_exc())
@@ -331,7 +334,7 @@ class Gateway(object):
path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
r = self.sendreq("GET", web_path)
r = self.sendreq("GET", web_path, {})
if r.status != 200:
self.closeconn()
log(
@@ -368,7 +371,7 @@ class Gateway(object):
)
)
r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
r = self.sendreq("GET", web_path, {"Range": hdr_range})
if r.status != http.client.PARTIAL_CONTENT:
self.closeconn()
raise Exception(

View File

@@ -1,7 +1,15 @@
# when running copyparty behind a reverse-proxy,
# make sure that copyparty allows at least as many clients as the proxy does,
# so run copyparty with -nc 512 if your nginx has the default limits
# (worker_processes 1, worker_connections 512)
# when running copyparty behind a reverse proxy,
# the following arguments are recommended:
#
# -nc 512 important, see next paragraph
# --http-only lower latency on initial connection
# -i 127.0.0.1 only accept connections from nginx
#
# -nc must match or exceed the webserver's max number of concurrent clients;
# nginx default is 512 (worker_processes 1, worker_connections 512)
#
# you may also consider adding -j0 for CPU-intensive configurations
# (not that i can really think of any good examples)
upstream cpp {
server 127.0.0.1:3923;

View File

@@ -7,11 +7,19 @@
# you may want to:
# change '/usr/bin/python' to another interpreter
# change '/mnt::a' to another location or permission-set
#
# with `Type=notify`, copyparty will signal systemd when it is ready to
# accept connections; correctly delaying units depending on copyparty.
# But note that journalctl will get the timestamps wrong due to
# python disabling line-buffering, so messages are out-of-order:
# https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
[Unit]
Description=copyparty file server
[Service]
Type=notify
SyslogIdentifier=copyparty
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'

View File

@@ -9,6 +9,9 @@ import os
PY2 = sys.version_info[0] == 2
if PY2:
sys.dont_write_bytecode = True
unicode = unicode
else:
unicode = str
WINDOWS = False
if platform.system() == "Windows":

View File

@@ -20,10 +20,10 @@ import threading
import traceback
from textwrap import dedent
from .__init__ import E, WINDOWS, VT100, PY2
from .__init__ import E, WINDOWS, VT100, PY2, unicode
from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
from .svchub import SvcHub
from .util import py_desc, align_tab, IMPLICATIONS, alltrace
from .util import py_desc, align_tab, IMPLICATIONS
HAVE_SSL = True
try:
@@ -31,6 +31,8 @@ try:
except:
HAVE_SSL = False
printed = ""
class RiceFormatter(argparse.HelpFormatter):
def _get_help_string(self, action):
@@ -61,8 +63,15 @@ class Dodge11874(RiceFormatter):
super(Dodge11874, self).__init__(*args, **kwargs)
def lprint(*a, **ka):
global printed
printed += " ".join(unicode(x) for x in a) + ka.get("end", "\n")
print(*a, **ka)
def warn(msg):
print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
def ensure_locale():
@@ -73,7 +82,7 @@ def ensure_locale():
]:
try:
locale.setlocale(locale.LC_ALL, x)
print("Locale:", x)
lprint("Locale:", x)
break
except:
continue
@@ -94,7 +103,7 @@ def ensure_cert():
try:
if filecmp.cmp(cert_cfg, cert_insec):
print(
lprint(
"\033[33m using default TLS certificate; https will be insecure."
+ "\033[36m\n certificate location: {}\033[0m\n".format(cert_cfg)
)
@@ -123,7 +132,7 @@ def configure_ssl_ver(al):
if "help" in sslver:
avail = [terse_sslver(x[6:]) for x in flags]
avail = " ".join(sorted(avail) + ["all"])
print("\navailable ssl/tls versions:\n " + avail)
lprint("\navailable ssl/tls versions:\n " + avail)
sys.exit(0)
al.ssl_flags_en = 0
@@ -143,7 +152,7 @@ def configure_ssl_ver(al):
for k in ["ssl_flags_en", "ssl_flags_de"]:
num = getattr(al, k)
print("{}: {:8x} ({})".format(k, num, num))
lprint("{}: {:8x} ({})".format(k, num, num))
# think i need that beer now
@@ -160,13 +169,13 @@ def configure_ssl_ciphers(al):
try:
ctx.set_ciphers(al.ciphers)
except:
print("\n\033[1;31mfailed to set ciphers\033[0m\n")
lprint("\n\033[1;31mfailed to set ciphers\033[0m\n")
if not hasattr(ctx, "get_ciphers"):
print("cannot read cipher list: openssl or python too old")
lprint("cannot read cipher list: openssl or python too old")
else:
ciphers = [x["description"] for x in ctx.get_ciphers()]
print("\n ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
lprint("\n ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
if is_help:
sys.exit(0)
@@ -182,16 +191,6 @@ def sighandler(sig=None, frame=None):
print("\n".join(msg))
def stackmon(fp, ival):
ctr = 0
while True:
ctr += 1
time.sleep(ival)
st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
with open(fp, "wb") as f:
f.write(st.encode("utf-8", "replace"))
def run_argparse(argv, formatter):
ap = argparse.ArgumentParser(
formatter_class=formatter,
@@ -249,30 +248,32 @@ def run_argparse(argv, formatter):
),
)
# fmt: off
ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
ap.add_argument("-ed", action="store_true", help="enable ?dots")
ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
u = unicode
ap2 = ap.add_argument_group('general options')
ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, USER:PASS; example [ed:wark")
ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
ap2.add_argument("-ed", action="store_true", help="enable ?dots")
ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
ap2 = ap.add_argument_group('network options')
ap2.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
ap2.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
ap2 = ap.add_argument_group('SSL/TLS options')
ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ssl/tls ciphers; [help] shows available ciphers")
ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [help] shows available ciphers")
ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets")
ap2 = ap.add_argument_group('opt-outs')
ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
@@ -281,14 +282,16 @@ def run_argparse(argv, formatter):
ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
ap2 = ap.add_argument_group('safety options')
ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
ap2.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt")
ap2 = ap.add_argument_group('logging options')
ap2.add_argument("-q", action="store_true", help="quiet")
ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
ap2.add_argument("--log-htp", action="store_true", help="print http-server threadpool scaling")
ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
ap2 = ap.add_argument_group('admin panel options')
ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
@@ -303,9 +306,9 @@ def run_argparse(argv, formatter):
ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval")
ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval; 0=disabled")
ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
ap2.add_argument("--th-covers", metavar="N,N", type=str, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
ap2 = ap.add_argument_group('database options')
ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
@@ -314,24 +317,27 @@ def run_argparse(argv, formatter):
ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
ap2.add_argument("--hist", metavar="PATH", type=str, help="where to store volume state")
ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume state")
ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,ac,vc,res,.fps")
ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
ap2.add_argument("-mtp", metavar="M=[f,]bin", type=u, action="append", help="read tag M using bin")
ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
ap2 = ap.add_argument_group('appearance options')
ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
ap2 = ap.add_argument_group('debug options')
ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
ap2.add_argument("--stackmon", metavar="P,S", help="write stacktrace to Path every S second")
ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
return ap.parse_args(args=argv[1:])
# fmt: on
@@ -348,7 +354,7 @@ def main(argv=None):
desc = py_desc().replace("[", "\033[1;30m[")
f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
ensure_locale()
if HAVE_SSL:
@@ -362,7 +368,7 @@ def main(argv=None):
continue
msg = "\033[1;31mWARNING:\033[0;1m\n {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
print(msg.format(dk, nk))
lprint(msg.format(dk, nk))
argv[idx] = nk
time.sleep(2)
@@ -371,16 +377,6 @@ def main(argv=None):
except AssertionError:
al = run_argparse(argv, Dodge11874)
if al.stackmon:
fp, f = al.stackmon.rsplit(",", 1)
f = int(f)
t = threading.Thread(
target=stackmon,
args=(fp, f),
)
t.daemon = True
t.start()
# propagate implications
for k1, k2 in IMPLICATIONS:
if getattr(al, k1):
@@ -416,7 +412,7 @@ def main(argv=None):
# signal.signal(signal.SIGINT, sighandler)
SvcHub(al).run()
SvcHub(al, argv, printed).run()
if __name__ == "__main__":

View File

@@ -1,8 +1,8 @@
# coding: utf-8
VERSION = (0, 11, 31)
VERSION = (0, 11, 45)
CODENAME = "the grid"
BUILD_DT = (2021, 7, 4)
BUILD_DT = (2021, 7, 20)
S_VERSION = ".".join(map(str, VERSION))
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

View File

@@ -10,13 +10,14 @@ import hashlib
import threading
from .__init__ import WINDOWS
from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir, nuprint
from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir
class VFS(object):
"""single level in the virtual fs"""
def __init__(self, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}):
def __init__(self, log, realpath, vpath, uread, uwrite, uadm, flags):
self.log = log
self.realpath = realpath # absolute path on host filesystem
self.vpath = vpath # absolute path in the virtual filesystem
self.uread = uread # users who can read this
@@ -62,6 +63,7 @@ class VFS(object):
return self.nodes[name].add(src, dst)
vn = VFS(
self.log,
os.path.join(self.realpath, name) if self.realpath else None,
"{}/{}".format(self.vpath, name).lstrip("/"),
self.uread,
@@ -79,7 +81,7 @@ class VFS(object):
# leaf does not exist; create and keep permissions blank
vp = "{}/{}".format(self.vpath, dst).lstrip("/")
vn = VFS(src, vp)
vn = VFS(self.log, src, vp, [], [], [], {})
vn.dbv = self.dbv or self
self.nodes[dst] = vn
return vn
@@ -181,7 +183,7 @@ class VFS(object):
"""return user-readable [fsdir,real,virt] items at vpath"""
virt_vis = {} # nodes readable by user
abspath = self.canonical(rem)
real = list(statdir(nuprint, scandir, lstat, abspath))
real = list(statdir(self.log, scandir, lstat, abspath))
real.sort()
if not rem:
for name, vn2 in sorted(self.nodes.items()):
@@ -208,8 +210,13 @@ class VFS(object):
rem, uname, scandir, incl_wo=False, lstat=lstat
)
if seen and not fsroot.startswith(seen[-1]) and fsroot in seen:
print("bailing from symlink loop,\n {}\n {}".format(seen[-1], fsroot))
if (
seen
and (not fsroot.startswith(seen[-1]) or fsroot == seen[-1])
and fsroot in seen
):
m = "bailing from symlink loop,\n prev: {}\n curr: {}\n from: {}/{}"
self.log("vfs.walk", m.format(seen[-1], fsroot, self.vpath, rem), 3)
return
seen = seen[:] + [fsroot]
@@ -242,6 +249,10 @@ class VFS(object):
if flt:
flt = {k: True for k in flt}
f1 = "{0}.hist{0}up2k.".format(os.sep)
f2a = os.sep + "dir.txt"
f2b = "{0}.hist{0}".format(os.sep)
for vpath, apath, files, rd, vd in self.walk(
"", vrem, [], uname, dots, scandir, False
):
@@ -275,7 +286,11 @@ class VFS(object):
del vd[x]
# up2k filetring based on actual abspath
files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
files = [
x
for x in files
if f1 not in x[1] and (not x[1].endswith(f2a) or f2b not in x[1])
]
for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
yield f
@@ -466,7 +481,7 @@ class AuthSrv(object):
)
except:
m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
print(m.format(cfg_fn, self.line_ctr))
self.log(m.format(cfg_fn, self.line_ctr), 1)
raise
# case-insensitive; normalize
@@ -482,10 +497,10 @@ class AuthSrv(object):
if not mount:
# -h says our defaults are CWD at root and read/write for everyone
vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
vfs = VFS(self.log_func, os.path.abspath("."), "", ["*"], ["*"], ["*"], {})
elif "" not in mount:
# there's volumes but no root; make root inaccessible
vfs = VFS(None, "")
vfs = VFS(self.log_func, None, "", [], [], [], {})
vfs.flags["d2d"] = True
maxdepth = 0
@@ -497,7 +512,13 @@ class AuthSrv(object):
if dst == "":
# rootfs was mapped; fully replaces the default CWD vfs
vfs = VFS(
mount[dst], dst, mread[dst], mwrite[dst], madm[dst], mflags[dst]
self.log_func,
mount[dst],
dst,
mread[dst],
mwrite[dst],
madm[dst],
mflags[dst],
)
continue
@@ -780,7 +801,7 @@ class AuthSrv(object):
msg = [x[1] for x in files]
if msg:
nuprint("\n".join(msg))
self.log("\n" + "\n".join(msg))
if n_bads and flag_p:
raise Exception("found symlink leaving volume, and strict is set")

View File

@@ -4,17 +4,11 @@ from __future__ import print_function, unicode_literals
import time
import threading
from .__init__ import PY2, WINDOWS, VT100
from .broker_util import try_exec
from .broker_mpw import MpWorker
from .util import mp
if PY2 and not WINDOWS:
from multiprocessing.reduction import ForkingPickler
from StringIO import StringIO as MemesIO # pylint: disable=import-error
class BrokerMp(object):
"""external api; manages MpWorkers"""
@@ -33,19 +27,17 @@ class BrokerMp(object):
cores = mp.cpu_count()
self.log("broker", "booting {} subprocesses".format(cores))
for n in range(cores):
for n in range(1, cores + 1):
q_pend = mp.Queue(1)
q_yield = mp.Queue(64)
proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
proc.q_pend = q_pend
proc.q_yield = q_yield
proc.nid = n
proc.clients = {}
proc.workload = 0
thr = threading.Thread(
target=self.collector, args=(proc,), name="mp-collector"
target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
)
thr.daemon = True
thr.start()
@@ -53,13 +45,6 @@ class BrokerMp(object):
self.procs.append(proc)
proc.start()
if not self.args.q:
thr = threading.Thread(
target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
)
thr.daemon = True
thr.start()
def shutdown(self):
self.log("broker", "shutting down")
for n, proc in enumerate(self.procs):
@@ -89,20 +74,6 @@ class BrokerMp(object):
if dest == "log":
self.log(*args)
elif dest == "workload":
with self.mutex:
proc.workload = args[0]
elif dest == "httpdrop":
addr = args[0]
with self.mutex:
del proc.clients[addr]
if not proc.clients:
proc.workload = 0
self.hub.tcpsrv.num_clients.add(-1)
elif dest == "retq":
# response from previous ipc call
with self.retpend_mutex:
@@ -128,38 +99,9 @@ class BrokerMp(object):
returns a Queue object which eventually contains the response if want_retval
(not-impl here since nothing uses it yet)
"""
if dest == "httpconn":
sck, addr = args
sck2 = sck
if PY2:
buf = MemesIO()
ForkingPickler(buf).dump(sck)
sck2 = buf.getvalue()
proc = sorted(self.procs, key=lambda x: x.workload)[0]
proc.q_pend.put([0, dest, [sck2, addr]])
with self.mutex:
proc.clients[addr] = 50
proc.workload += 50
if dest == "listen":
for p in self.procs:
p.q_pend.put([0, dest, [args[0], len(self.procs)]])
else:
raise Exception("what is " + str(dest))
def debug_load_balancer(self):
fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
if not VT100:
fmt = "({}{:4})"
last = ""
while self.procs:
msg = ""
for proc in self.procs:
msg += fmt.format(len(proc.clients), proc.workload)
if msg != last:
last = msg
with self.hub.log_mutex:
print(msg)
time.sleep(0.1)

View File

@@ -3,18 +3,13 @@ from __future__ import print_function, unicode_literals
from copyparty.authsrv import AuthSrv
import sys
import time
import signal
import threading
from .__init__ import PY2, WINDOWS
from .broker_util import ExceptionalQueue
from .httpsrv import HttpSrv
from .util import FAKE_MP
if PY2 and not WINDOWS:
import pickle # nosec
class MpWorker(object):
"""one single mp instance"""
@@ -25,22 +20,23 @@ class MpWorker(object):
self.args = args
self.n = n
self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
self.retpend = {}
self.retpend_mutex = threading.Lock()
self.mutex = threading.Lock()
self.workload_thr_alive = False
# we inherited signal_handler from parent,
# replace it with something harmless
if not FAKE_MP:
signal.signal(signal.SIGINT, self.signal_handler)
for sig in [signal.SIGINT, signal.SIGTERM]:
signal.signal(sig, self.signal_handler)
# starting to look like a good idea
self.asrv = AuthSrv(args, None, False)
# instantiate all services here (TODO: inheritance?)
self.httpsrv = HttpSrv(self, True)
self.httpsrv.disconnect_func = self.httpdrop
self.httpsrv = HttpSrv(self, n)
# on winxp and some other platforms,
# use thr.join() to block all signals
@@ -49,19 +45,19 @@ class MpWorker(object):
thr.start()
thr.join()
def signal_handler(self, signal, frame):
def signal_handler(self, sig, frame):
# print('k')
pass
def log(self, src, msg, c=0):
def _log_enabled(self, src, msg, c=0):
self.q_yield.put([0, "log", [src, msg, c]])
def _log_disabled(self, src, msg, c=0):
pass
def logw(self, msg, c=0):
self.log("mp{}".format(self.n), msg, c)
def httpdrop(self, addr):
self.q_yield.put([0, "httpdrop", [addr]])
def main(self):
while True:
retq_id, dest, args = self.q_pend.get()
@@ -73,24 +69,8 @@ class MpWorker(object):
sys.exit(0)
return
elif dest == "httpconn":
sck, addr = args
if PY2:
sck = pickle.loads(sck) # nosec
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
self.httpsrv.accept(sck, addr)
with self.mutex:
if not self.workload_thr_alive:
self.workload_thr_alive = True
thr = threading.Thread(
target=self.thr_workload, name="mpw-workload"
)
thr.daemon = True
thr.start()
elif dest == "listen":
self.httpsrv.listen(args[0], args[1])
elif dest == "retq":
# response from previous ipc call
@@ -114,16 +94,3 @@ class MpWorker(object):
self.q_yield.put([retq_id, dest, args])
return retq
def thr_workload(self):
"""announce workloads to MpSrv (the mp controller / loadbalancer)"""
# avoid locking in extract_filedata by tracking difference here
while True:
time.sleep(0.2)
with self.mutex:
if self.httpsrv.num_clients() == 0:
# no clients rn, termiante thread
self.workload_thr_alive = False
return
self.q_yield.put([0, "workload", [self.httpsrv.workload]])

View File

@@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals
import threading
from .authsrv import AuthSrv
from .httpsrv import HttpSrv
from .broker_util import ExceptionalQueue, try_exec
@@ -20,8 +19,7 @@ class BrokerThr(object):
self.mutex = threading.Lock()
# instantiate all services here (TODO: inheritance?)
self.httpsrv = HttpSrv(self)
self.httpsrv.disconnect_func = self.httpdrop
self.httpsrv = HttpSrv(self, None)
def shutdown(self):
# self.log("broker", "shutting down")
@@ -29,12 +27,8 @@ class BrokerThr(object):
pass
def put(self, want_retval, dest, *args):
if dest == "httpconn":
sck, addr = args
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
self.httpsrv.accept(sck, addr)
if dest == "listen":
self.httpsrv.listen(args[0], 1)
else:
# new ipc invoking managed service in hub
@@ -51,6 +45,3 @@ class BrokerThr(object):
retq = ExceptionalQueue(1)
retq.put(rv)
return retq
def httpdrop(self, addr):
self.hub.tcpsrv.num_clients.add(-1)

View File

@@ -13,15 +13,12 @@ import ctypes
from datetime import datetime
import calendar
from .__init__ import E, PY2, WINDOWS, ANYWIN
from .__init__ import E, PY2, WINDOWS, ANYWIN, unicode
from .util import * # noqa # pylint: disable=unused-wildcard-import
from .authsrv import AuthSrv
from .szip import StreamZip
from .star import StreamTar
if not PY2:
unicode = str
NO_CACHE = {"Cache-Control": "no-cache"}
NO_STORE = {"Cache-Control": "no-store; max-age=0"}
@@ -40,7 +37,6 @@ class HttpCli(object):
self.ip = conn.addr[0]
self.addr = conn.addr # type: tuple[str, int]
self.args = conn.args
self.is_mp = conn.is_mp
self.asrv = conn.asrv # type: AuthSrv
self.ico = conn.ico
self.thumbcli = conn.thumbcli
@@ -98,9 +94,13 @@ class HttpCli(object):
try:
self.mode, self.req, self.http_ver = headerlines[0].split(" ")
except:
raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
msg = " ]\n#[ ".join(headerlines)
raise Pebkac(400, "bad headers:\n#[ " + msg + " ]")
except Pebkac as ex:
self.mode = "GET"
self.req = "[junk]"
self.http_ver = "HTTP/1.1"
# self.log("pebkac at httpcli.run #1: " + repr(ex))
self.keepalive = self._check_nonfatal(ex)
self.loud_reply(unicode(ex), status=ex.code)
@@ -226,7 +226,7 @@ class HttpCli(object):
except Pebkac:
return False
def send_headers(self, length, status=200, mime=None, headers={}):
def send_headers(self, length, status=200, mime=None, headers=None):
response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
if length is not None:
@@ -236,7 +236,8 @@ class HttpCli(object):
response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
# headers{} overrides anything set previously
self.out_headers.update(headers)
if headers:
self.out_headers.update(headers)
# default to utf8 html if no content-type is set
if not mime:
@@ -253,7 +254,7 @@ class HttpCli(object):
except:
raise Pebkac(400, "client d/c while replying headers")
def reply(self, body, status=200, mime=None, headers={}):
def reply(self, body, status=200, mime=None, headers=None):
# TODO something to reply with user-supplied values safely
self.send_headers(len(body), status, mime, headers)
@@ -269,7 +270,7 @@ class HttpCli(object):
self.log(body.rstrip())
self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
def urlq(self, add={}, rm=[]):
def urlq(self, add, rm):
"""
generates url query based on uparam (b, pw, all others)
removing anything in rm, adding pairs in add
@@ -341,6 +342,9 @@ class HttpCli(object):
if "tree" in self.uparam:
return self.tx_tree()
if "stack" in self.uparam:
return self.tx_stack()
# conditional redirect to single volumes
if self.vpath == "" and not self.ouparam:
nread = len(self.rvol)
@@ -370,9 +374,6 @@ class HttpCli(object):
if "scan" in self.uparam:
return self.scanvol()
if "stack" in self.uparam:
return self.tx_stack()
return self.tx_browser()
def handle_options(self):
@@ -478,15 +479,17 @@ class HttpCli(object):
addr = self.ip.replace(":", ".")
fn = "put-{:.6f}-{}.bin".format(time.time(), addr)
path = os.path.join(fdir, fn)
if self.args.nw:
path = os.devnull
with open(fsenc(path), "wb", 512 * 1024) as f:
post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
post_sz, _, sha_b64 = hashcopy(reader, f)
vfs, vrem = vfs.get_dbv(rem)
self.conn.hsrv.broker.put(
False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
)
if not self.args.nw:
vfs, vrem = vfs.get_dbv(rem)
self.conn.hsrv.broker.put(
False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
)
return post_sz, sha_b64, remains, path
@@ -712,7 +715,7 @@ class HttpCli(object):
with open(fsenc(path), "rb+", 512 * 1024) as f:
f.seek(cstart[0])
post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
post_sz, _, sha_b64 = hashcopy(reader, f)
if sha_b64 != chash:
raise Pebkac(
@@ -792,7 +795,7 @@ class HttpCli(object):
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem)
sanitized = sanitize_fn(new_dir)
sanitized = sanitize_fn(new_dir, "", [])
if not nullwrite:
fdir = os.path.join(vfs.realpath, rem)
@@ -829,7 +832,7 @@ class HttpCli(object):
if not new_file.endswith(".md"):
new_file += ".md"
sanitized = sanitize_fn(new_file)
sanitized = sanitize_fn(new_file, "", [])
if not nullwrite:
fdir = os.path.join(vfs.realpath, rem)
@@ -862,7 +865,7 @@ class HttpCli(object):
if p_file and not nullwrite:
fdir = os.path.join(vfs.realpath, rem)
fname = sanitize_fn(
p_file, bad=[".prologue.html", ".epilogue.html"]
p_file, "", [".prologue.html", ".epilogue.html"]
)
if not os.path.isdir(fsenc(fdir)):
@@ -879,7 +882,7 @@ class HttpCli(object):
with ren_open(fname, "wb", 512 * 1024, **open_args) as f:
f, fname = f["orz"]
self.log("writing to {}/{}".format(fdir, fname))
sz, sha512_hex, _ = hashcopy(self.conn, p_data, f)
sz, sha512_hex, _ = hashcopy(p_data, f)
if sz == 0:
raise Pebkac(400, "empty files in post")
@@ -1062,7 +1065,7 @@ class HttpCli(object):
raise Pebkac(400, "expected body, got {}".format(p_field))
with open(fsenc(fp), "wb", 512 * 1024) as f:
sz, sha512, _ = hashcopy(self.conn, p_data, f)
sz, sha512, _ = hashcopy(p_data, f)
new_lastmod = os.stat(fsenc(fp)).st_mtime
new_lastmod3 = int(new_lastmod * 1000)
@@ -1252,8 +1255,7 @@ class HttpCli(object):
if use_sendfile:
remains = sendfile_kern(lower, upper, f, self.s)
else:
actor = self.conn if self.is_mp else None
remains = sendfile_py(lower, upper, f, self.s, actor)
remains = sendfile_py(lower, upper, f, self.s)
if remains > 0:
logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
@@ -1310,7 +1312,7 @@ class HttpCli(object):
fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
# for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
bgen = packer(self.log, fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
bsent = 0
for buf in bgen.gen():
if not buf:
@@ -1374,7 +1376,7 @@ class HttpCli(object):
for c, v in [[b"&", 4], [b"<", 3], [b">", 3]]:
sz_md += (len(buf) - len(buf.replace(c, b""))) * v
file_ts = max(ts_md, ts_html)
file_ts = max(ts_md, ts_html, E.t0)
file_lastmod, do_send = self._chk_lastmod(file_ts)
self.out_headers["Last-Modified"] = file_lastmod
self.out_headers.update(NO_CACHE)
@@ -1421,7 +1423,7 @@ class HttpCli(object):
return True
def tx_mounts(self):
suf = self.urlq(rm=["h"])
suf = self.urlq({}, ["h"])
rvol, wvol, avol = [
[("/" + x).rstrip("/") + "/" for x in y]
for y in [self.rvol, self.wvol, self.avol]
@@ -1471,7 +1473,7 @@ class HttpCli(object):
raise Pebkac(500, x)
def tx_stack(self):
if not self.readable or not self.writable:
if not self.avol:
raise Pebkac(403, "not admin")
if self.args.no_stack:
@@ -1561,7 +1563,9 @@ class HttpCli(object):
raise Pebkac(404)
if self.readable:
if rem.startswith(".hist/up2k."):
if rem.startswith(".hist/up2k.") or (
rem.endswith("/dir.txt") and rem.startswith(".hist/th/")
):
raise Pebkac(403)
is_dir = stat.S_ISDIR(st.st_mode)
@@ -1630,7 +1634,7 @@ class HttpCli(object):
if self.writable:
perms.append("write")
url_suf = self.urlq()
url_suf = self.urlq({}, [])
is_ls = "ls" in self.uparam
tpl = "browser"
@@ -1649,6 +1653,7 @@ class HttpCli(object):
"files": [],
"taglist": [],
"srvinf": srv_info,
"acct": self.uname,
"perms": perms,
"logues": logues,
}
@@ -1656,6 +1661,7 @@ class HttpCli(object):
"vdir": quotep(self.vpath),
"vpnodes": vpnodes,
"files": [],
"acct": self.uname,
"perms": json.dumps(perms),
"taglist": [],
"tag_order": [],

View File

@@ -34,7 +34,6 @@ class HttpConn(object):
self.args = hsrv.args
self.asrv = hsrv.asrv
self.is_mp = hsrv.is_mp
self.cert_path = hsrv.cert_path
enth = HAVE_PIL and not self.args.no_thumb
@@ -45,7 +44,6 @@ class HttpConn(object):
self.stopping = False
self.nreq = 0
self.nbyte = 0
self.workload = 0
self.u2idx = None
self.log_func = hsrv.log
self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
@@ -184,11 +182,6 @@ class HttpConn(object):
self.sr = Unrecv(self.s)
while not self.stopping:
if self.is_mp:
self.workload += 50
if self.workload >= 2 ** 31:
self.workload = 100
self.nreq += 1
cli = HttpCli(self)
if not cli.run():

View File

@@ -4,8 +4,8 @@ from __future__ import print_function, unicode_literals
import os
import sys
import time
import math
import base64
import struct
import socket
import threading
@@ -26,9 +26,15 @@ except ImportError:
)
sys.exit(1)
from .__init__ import E, MACOS
from .__init__ import E, PY2, MACOS
from .util import spack, min_ex, start_stackmon, start_log_thrs
from .httpconn import HttpConn
if PY2:
import Queue as queue
else:
import queue
class HttpSrv(object):
"""
@@ -36,19 +42,26 @@ class HttpSrv(object):
relying on MpSrv for performance (HttpSrv is just plain threads)
"""
def __init__(self, broker, is_mp=False):
def __init__(self, broker, nid):
self.broker = broker
self.is_mp = is_mp
self.nid = nid
self.args = broker.args
self.log = broker.log
self.asrv = broker.asrv
self.disconnect_func = None
self.name = "httpsrv" + ("-n{}-i{:x}".format(nid, os.getpid()) if nid else "")
self.mutex = threading.Lock()
self.stopping = False
self.clients = {}
self.workload = 0
self.workload_thr_alive = False
self.tp_nthr = 0 # actual
self.tp_ncli = 0 # fading
self.tp_time = None # latest worker collect
self.tp_q = None if self.args.no_htp else queue.LifoQueue()
self.srvs = []
self.ncli = 0 # exact
self.clients = {} # laggy
self.nclimax = 0
self.cb_ts = 0
self.cb_v = 0
@@ -65,24 +78,155 @@ class HttpSrv(object):
else:
self.cert_path = None
if self.tp_q:
self.start_threads(4)
name = "httpsrv-scaler" + ("-{}".format(nid) if nid else "")
t = threading.Thread(target=self.thr_scaler, name=name)
t.daemon = True
t.start()
if nid:
if self.args.stackmon:
start_stackmon(self.args.stackmon, nid)
if self.args.log_thrs:
start_log_thrs(self.log, self.args.log_thrs, nid)
def start_threads(self, n):
self.tp_nthr += n
if self.args.log_htp:
self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
for _ in range(n):
thr = threading.Thread(
target=self.thr_poolw,
name=self.name + "-poolw",
)
thr.daemon = True
thr.start()
def stop_threads(self, n):
self.tp_nthr -= n
if self.args.log_htp:
self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
for _ in range(n):
self.tp_q.put(None)
def thr_scaler(self):
while True:
time.sleep(2 if self.tp_ncli else 30)
with self.mutex:
self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
if self.tp_nthr > self.tp_ncli + 8:
self.stop_threads(4)
def listen(self, sck, nlisteners):
ip, port = sck.getsockname()
self.srvs.append(sck)
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
t = threading.Thread(
target=self.thr_listen,
args=(sck,),
name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
)
t.daemon = True
t.start()
def thr_listen(self, srv_sck):
"""listens on a shared tcp server"""
ip, port = srv_sck.getsockname()
fno = srv_sck.fileno()
msg = "subscribed @ {}:{} f{}".format(ip, port, fno)
self.log(self.name, msg)
while not self.stopping:
if self.args.log_conn:
self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
if self.ncli >= self.nclimax:
self.log(self.name, "at connection limit; waiting", 3)
while self.ncli >= self.nclimax:
time.sleep(0.1)
if self.args.log_conn:
self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
try:
sck, addr = srv_sck.accept()
except (OSError, socket.error) as ex:
self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
time.sleep(0.02)
continue
if self.args.log_conn:
m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
"-" * 3, ip, port % 8, port
)
self.log("%s %s" % addr, m, c="1;30")
self.accept(sck, addr)
def accept(self, sck, addr):
"""takes an incoming tcp connection and creates a thread to handle it"""
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
now = time.time()
if now - (self.tp_time or now) > 300:
self.tp_q = None
if self.tp_q:
self.tp_q.put((sck, addr))
with self.mutex:
self.ncli += 1
self.tp_time = self.tp_time or now
self.tp_ncli = max(self.tp_ncli, self.ncli + 1)
if self.tp_nthr < self.ncli + 4:
self.start_threads(8)
return
if not self.args.no_htp:
m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
self.log(self.name, m, 1)
with self.mutex:
self.ncli += 1
thr = threading.Thread(
target=self.thr_client,
args=(sck, addr),
name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
)
thr.daemon = True
thr.start()
def num_clients(self):
with self.mutex:
return len(self.clients)
def thr_poolw(self):
while True:
task = self.tp_q.get()
if not task:
break
with self.mutex:
self.tp_time = None
try:
sck, addr = task
me = threading.current_thread()
me.name = "httpconn-{}-{}".format(
addr[0].split(".", 2)[-1][-6:], addr[1]
)
self.thr_client(sck, addr)
me.name = self.name + "-poolw"
except:
self.log(self.name, "thr_client: " + min_ex(), 3)
def shutdown(self):
self.stopping = True
for srv in self.srvs:
try:
srv.close()
except:
pass
clients = list(self.clients.keys())
for cli in clients:
try:
@@ -90,7 +234,14 @@ class HttpSrv(object):
except:
pass
self.log("httpsrv-n", "ok bye")
if self.tp_q:
self.stop_threads(self.tp_nthr)
for _ in range(10):
time.sleep(0.05)
if self.tp_q.empty():
break
self.log(self.name, "ok bye")
def thr_client(self, sck, addr):
"""thread managing one tcp client"""
@@ -100,25 +251,15 @@ class HttpSrv(object):
with self.mutex:
self.clients[cli] = 0
if self.is_mp:
self.workload += 50
if not self.workload_thr_alive:
self.workload_thr_alive = True
thr = threading.Thread(
target=self.thr_workload, name="httpsrv-workload"
)
thr.daemon = True
thr.start()
fno = sck.fileno()
try:
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
cli.run()
except (OSError, socket.error) as ex:
if ex.errno not in [10038, 10054, 107, 57, 9]:
if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
self.log(
"%s %s" % addr,
"run({}): {}".format(fno, ex),
@@ -128,7 +269,7 @@ class HttpSrv(object):
finally:
sck = cli.s
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
try:
fno = sck.fileno()
@@ -152,35 +293,7 @@ class HttpSrv(object):
finally:
with self.mutex:
del self.clients[cli]
if self.disconnect_func:
self.disconnect_func(addr) # pylint: disable=not-callable
def thr_workload(self):
"""indicates the python interpreter workload caused by this HttpSrv"""
# avoid locking in extract_filedata by tracking difference here
while True:
time.sleep(0.2)
with self.mutex:
if not self.clients:
# no clients rn, termiante thread
self.workload_thr_alive = False
self.workload = 0
return
total = 0
with self.mutex:
for cli in self.clients.keys():
now = cli.workload
delta = now - self.clients[cli]
if delta < 0:
# was reset in HttpCli to prevent overflow
delta = now
total += delta
self.clients[cli] = now
self.workload = total
self.ncli -= 1
def cachebuster(self):
if time.time() - self.cb_ts < 1:
@@ -199,7 +312,7 @@ class HttpSrv(object):
except:
pass
v = base64.urlsafe_b64encode(struct.pack(">xxL", int(v)))
v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
self.cb_v = v.decode("ascii")[-4:]
self.cb_ts = time.time()
return self.cb_v

View File

@@ -7,12 +7,9 @@ import json
import shutil
import subprocess as sp
from .__init__ import PY2, WINDOWS
from .__init__ import PY2, WINDOWS, unicode
from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
if not PY2:
unicode = str
def have_ff(cmd):
if PY2:
@@ -230,37 +227,47 @@ def parse_ffprobe(txt):
class MTag(object):
def __init__(self, log_func, args):
self.log_func = log_func
self.args = args
self.usable = True
self.prefer_mt = False
mappings = args.mtm
self.prefer_mt = not args.no_mtag_ff
self.backend = "ffprobe" if args.no_mutagen else "mutagen"
or_ffprobe = " or ffprobe"
self.can_ffprobe = (
HAVE_FFPROBE
and not args.no_mtag_ff
and (not WINDOWS or sys.version_info >= (3, 8))
)
mappings = args.mtm
or_ffprobe = " or FFprobe"
if self.backend == "mutagen":
self.get = self.get_mutagen
try:
import mutagen
except:
self.log("could not load mutagen, trying ffprobe instead", c=3)
self.log("could not load Mutagen, trying FFprobe instead", c=3)
self.backend = "ffprobe"
if self.backend == "ffprobe":
self.usable = self.can_ffprobe
self.get = self.get_ffprobe
self.prefer_mt = True
# about 20x slower
self.usable = HAVE_FFPROBE
if self.usable and WINDOWS and sys.version_info < (3, 8):
self.usable = False
if not HAVE_FFPROBE:
pass
elif args.no_mtag_ff:
msg = "found FFprobe but it was disabled by --no-mtag-ff"
self.log(msg, c=3)
elif WINDOWS and sys.version_info < (3, 8):
or_ffprobe = " or python >= 3.8"
msg = "found ffprobe but your python is too old; need 3.8 or newer"
msg = "found FFprobe but your python is too old; need 3.8 or newer"
self.log(msg, c=1)
if not self.usable:
msg = "need mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
self.log(
msg.format(or_ffprobe, " " * 37, os.path.basename(sys.executable)), c=1
)
msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
pybin = os.path.basename(sys.executable)
self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
return
# https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -390,7 +397,7 @@ class MTag(object):
v2 = r2.get(k)
if v1 == v2:
print(" ", k, v1)
elif v1 != "0000": # ffprobe date=0
elif v1 != "0000": # FFprobe date=0
diffs.append(k)
print(" 1", k, v1)
print(" 2", k, v2)
@@ -411,20 +418,33 @@ class MTag(object):
md = mutagen.File(fsenc(abspath), easy=True)
x = md.info.length
except Exception as ex:
return {}
return self.get_ffprobe(abspath) if self.can_ffprobe else {}
ret = {}
try:
dur = int(md.info.length)
sz = os.path.getsize(fsenc(abspath))
ret = {".q": [0, int((sz / md.info.length) / 128)]}
for attr, k, norm in [
["codec", "ac", unicode],
["channels", "chs", int],
["sample_rate", ".hz", int],
["bitrate", ".aq", int],
["length", ".dur", int],
]:
try:
q = int(md.info.bitrate / 1024)
v = getattr(md.info, attr)
except:
q = int((os.path.getsize(fsenc(abspath)) / dur) / 128)
continue
ret[".dur"] = [0, dur]
ret[".q"] = [0, q]
except:
pass
if not v:
continue
if k == ".aq":
v /= 1000
if k == "ac" and v.startswith("mp4a.40."):
v = "aac"
ret[k] = [0, norm(v)]
return self.normalize_tags(ret, md)

View File

@@ -33,10 +33,11 @@ class QFile(object):
class StreamTar(object):
"""construct in-memory tar file from the given path"""
def __init__(self, fgen, **kwargs):
def __init__(self, log, fgen, **kwargs):
self.ci = 0
self.co = 0
self.qfile = QFile()
self.log = log
self.fgen = fgen
self.errf = None
@@ -91,7 +92,8 @@ class StreamTar(object):
errors.append([f["vp"], repr(ex)])
if errors:
self.errf = errdesc(errors)
self.errf, txt = errdesc(errors)
self.log("\n".join(([repr(self.errf)] + txt[1:])))
self.ser(self.errf)
self.tar.close()

View File

@@ -25,4 +25,4 @@ def errdesc(errors):
"vp": "archive-errors-{}.txt".format(dt),
"ap": tf_path,
"st": os.stat(tf_path),
}
}, report

View File

@@ -5,12 +5,16 @@ import re
import os
import sys
import time
import shlex
import string
import signal
import socket
import threading
from datetime import datetime, timedelta
import calendar
from .__init__ import PY2, WINDOWS, MACOS, VT100
from .util import mp
from .__init__ import E, PY2, WINDOWS, MACOS, VT100, unicode
from .util import mp, start_log_thrs, start_stackmon, min_ex
from .authsrv import AuthSrv
from .tcpsrv import TcpSrv
from .up2k import Up2k
@@ -28,14 +32,27 @@ class SvcHub(object):
put() can return a queue (if want_reply=True) which has a blocking get() with the response.
"""
def __init__(self, args):
def __init__(self, args, argv, printed):
self.args = args
self.argv = argv
self.logf = None
self.stop_req = False
self.stopping = False
self.stop_cond = threading.Condition()
self.ansi_re = re.compile("\033\\[[^m]*m")
self.log_mutex = threading.Lock()
self.next_day = 0
self.log = self._log_disabled if args.q else self._log_enabled
if args.lo:
self._setup_logfile(printed)
if args.stackmon:
start_stackmon(args.stackmon, 0)
if args.log_thrs:
start_log_thrs(self.log, args.log_thrs, 0)
# initiate all services to manage
self.asrv = AuthSrv(self.args, self.log, False)
@@ -69,22 +86,102 @@ class SvcHub(object):
self.broker = Broker(self)
def _logname(self):
dt = datetime.utcfromtimestamp(time.time())
fn = self.args.lo
for fs in "YmdHMS":
fs = "%" + fs
if fs in fn:
fn = fn.replace(fs, dt.strftime(fs))
return fn
def _setup_logfile(self, printed):
base_fn = fn = sel_fn = self._logname()
if fn != self.args.lo:
ctr = 0
# yup this is a race; if started sufficiently concurrently, two
# copyparties can grab the same logfile (considered and ignored)
while os.path.exists(sel_fn):
ctr += 1
sel_fn = "{}.{}".format(fn, ctr)
fn = sel_fn
try:
import lzma
lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
except:
import codecs
lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
lh.base_fn = base_fn
argv = [sys.executable] + self.argv
if hasattr(shlex, "quote"):
argv = [shlex.quote(x) for x in argv]
else:
argv = ['"{}"'.format(x) for x in argv]
msg = "[+] opened logfile [{}]\n".format(fn)
printed += msg
lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
self.logf = lh
print(msg, end="")
def run(self):
thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
self.tcpsrv.run()
thr = threading.Thread(target=self.sd_notify, name="sd-notify")
thr.daemon = True
thr.start()
# winxp/py2.7 support: thr.join() kills signals
try:
while True:
time.sleep(9001)
thr = threading.Thread(target=self.stop_thr, name="svchub-sig")
thr.daemon = True
thr.start()
except KeyboardInterrupt:
for sig in [signal.SIGINT, signal.SIGTERM]:
signal.signal(sig, self.signal_handler)
try:
while not self.stop_req:
time.sleep(9001)
except:
pass
self.shutdown()
def stop_thr(self):
while not self.stop_req:
with self.stop_cond:
self.stop_cond.wait(9001)
self.shutdown()
def signal_handler(self):
if self.stopping:
return
self.stop_req = True
with self.stop_cond:
self.stop_cond.notify_all()
def shutdown(self):
if self.stopping:
return
self.stopping = True
self.stop_req = True
try:
with self.log_mutex:
print("OPYTHAT")
self.tcpsrv.shutdown()
self.broker.shutdown()
self.up2k.shutdown()
if self.thumbsrv:
self.thumbsrv.shutdown()
@@ -99,9 +196,36 @@ class SvcHub(object):
print("nailed it", end="")
finally:
print("\033[0m")
if self.logf:
self.logf.close()
def _log_disabled(self, src, msg, c=0):
pass
if not self.logf:
return
with self.log_mutex:
ts = datetime.utcfromtimestamp(time.time())
ts = ts.strftime("%Y-%m%d-%H%M%S.%f")[:-3]
self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
now = time.time()
if now >= self.next_day:
self._set_next_day()
def _set_next_day(self):
if self.next_day and self.logf and self.logf.base_fn != self._logname():
self.logf.close()
self._setup_logfile("")
dt = datetime.utcfromtimestamp(time.time())
# unix timestamp of next 00:00:00 (leap-seconds safe)
day_now = dt.day
while dt.day == day_now:
dt += timedelta(hours=12)
dt = dt.replace(hour=0, minute=0, second=0)
self.next_day = calendar.timegm(dt.utctimetuple())
def _log_enabled(self, src, msg, c=0):
"""handles logging from all components"""
@@ -110,14 +234,7 @@ class SvcHub(object):
if now >= self.next_day:
dt = datetime.utcfromtimestamp(now)
print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
# unix timestamp of next 00:00:00 (leap-seconds safe)
day_now = dt.day
while dt.day == day_now:
dt += timedelta(hours=12)
dt = dt.replace(hour=0, minute=0, second=0)
self.next_day = calendar.timegm(dt.utctimetuple())
self._set_next_day()
fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
if not VT100:
@@ -144,20 +261,20 @@ class SvcHub(object):
except:
print(msg.encode("ascii", "replace").decode(), end="")
if self.logf:
self.logf.write(msg)
def check_mp_support(self):
vmin = sys.version_info[1]
if WINDOWS:
msg = "need python 3.3 or newer for multiprocessing;"
if PY2:
# py2 pickler doesn't support winsock
return msg
elif vmin < 3:
if PY2 or vmin < 3:
return msg
elif MACOS:
return "multiprocessing is wonky on mac osx;"
else:
msg = "need python 2.7 or 3.3+ for multiprocessing;"
if not PY2 and vmin < 3:
msg = "need python 3.3+ for multiprocessing;"
if PY2 or vmin < 3:
return msg
try:
@@ -189,5 +306,24 @@ class SvcHub(object):
if not err:
return True
else:
self.log("root", err)
self.log("svchub", err)
return False
def sd_notify(self):
try:
addr = os.getenv("NOTIFY_SOCKET")
if not addr:
return
addr = unicode(addr)
if addr.startswith("@"):
addr = "\0" + addr[1:]
m = "".join(x for x in addr if x in string.printable)
self.log("sd_notify", m)
sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
sck.connect(addr)
sck.sendall(b"READY=1")
except:
self.log("sd_notify", min_ex())

View File

@@ -4,15 +4,14 @@ from __future__ import print_function, unicode_literals
import os
import time
import zlib
import struct
from datetime import datetime
from .sutil import errdesc
from .util import yieldfile, sanitize_fn
from .util import yieldfile, sanitize_fn, spack, sunpack
def dostime2unix(buf):
t, d = struct.unpack("<HH", buf)
t, d = sunpack(b"<HH", buf)
ts = (t & 0x1F) * 2
tm = (t >> 5) & 0x3F
@@ -36,13 +35,13 @@ def unixtime2dos(ts):
bd = ((dy - 1980) << 9) + (dm << 5) + dd
bt = (th << 11) + (tm << 5) + ts // 2
return struct.pack("<HH", bt, bd)
return spack(b"<HH", bt, bd)
def gen_fdesc(sz, crc32, z64):
ret = b"\x50\x4b\x07\x08"
fmt = "<LQQ" if z64 else "<LLL"
ret += struct.pack(fmt, crc32, sz, sz)
fmt = b"<LQQ" if z64 else b"<LLL"
ret += spack(fmt, crc32, sz, sz)
return ret
@@ -66,7 +65,7 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
if crc32:
crc32 = struct.pack("<L", crc32)
crc32 = spack(b"<L", crc32)
else:
crc32 = b"\x00" * 4
@@ -87,14 +86,14 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
# however infozip does actual sz and it even works on winxp
# (same reasning for z64 extradata later)
vsz = 0xFFFFFFFF if z64 else sz
ret += struct.pack("<LL", vsz, vsz)
ret += spack(b"<LL", vsz, vsz)
# windows support (the "?" replace below too)
fn = sanitize_fn(fn, ok="/")
fn = sanitize_fn(fn, "/", [])
bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
z64_len = len(z64v) * 8 + 4 if z64v else 0
ret += struct.pack("<HH", len(bfn), z64_len)
ret += spack(b"<HH", len(bfn), z64_len)
if h_pos is not None:
# 2b comment, 2b diskno
@@ -106,12 +105,12 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
ret += b"\x01\x00\x00\x00\xa4\x81"
# 4b local-header-ofs
ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
ret += spack(b"<L", min(h_pos, 0xFFFFFFFF))
ret += bfn
if z64v:
ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
ret += spack(b"<HH" + b"Q" * len(z64v), 1, len(z64v) * 8, *z64v)
return ret
@@ -136,7 +135,7 @@ def gen_ecdr(items, cdir_pos, cdir_end):
need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
# 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
ret += spack(b"<HHLL", nitems, nitems, csz, cpos)
# 2b comment length
ret += b"\x00\x00"
@@ -163,7 +162,7 @@ def gen_ecdr64(items, cdir_pos, cdir_end):
# 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
cdir_sz = cdir_end - cdir_pos
ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
ret += spack(b"<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
return ret
@@ -178,13 +177,14 @@ def gen_ecdr64_loc(ecdr64_pos):
ret = b"\x50\x4b\x06\x07"
# 4b cdisk, 8b start of ecdr64, 4b ndisks
ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
ret += spack(b"<LQL", 0, ecdr64_pos, 1)
return ret
class StreamZip(object):
def __init__(self, fgen, utf8=False, pre_crc=False):
def __init__(self, log, fgen, utf8=False, pre_crc=False):
self.log = log
self.fgen = fgen
self.utf8 = utf8
self.pre_crc = pre_crc
@@ -247,8 +247,8 @@ class StreamZip(object):
errors.append([f["vp"], repr(ex)])
if errors:
errf = errdesc(errors)
print(repr(errf))
errf, txt = errdesc(errors)
self.log("\n".join(([repr(errf)] + txt[1:])))
for x in self.ser(errf):
yield x

View File

@@ -2,11 +2,9 @@
from __future__ import print_function, unicode_literals
import re
import time
import socket
import select
from .util import chkcmd, Counter
from .util import chkcmd
class TcpSrv(object):
@@ -20,7 +18,6 @@ class TcpSrv(object):
self.args = hub.args
self.log = hub.log
self.num_clients = Counter()
self.stopping = False
ip = "127.0.0.1"
@@ -66,44 +63,13 @@ class TcpSrv(object):
for srv in self.srv:
srv.listen(self.args.nc)
ip, port = srv.getsockname()
self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
fno = srv.fileno()
msg = "listening @ {}:{} f{}".format(ip, port, fno)
self.log("tcpsrv", msg)
if self.args.q:
print(msg)
while not self.stopping:
if self.args.log_conn:
self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
if self.num_clients.v >= self.args.nc:
time.sleep(0.1)
continue
if self.args.log_conn:
self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
try:
# macos throws bad-fd
ready, _, _ = select.select(self.srv, [], [])
except:
ready = []
if not self.stopping:
raise
for srv in ready:
if self.stopping:
break
sck, addr = srv.accept()
sip, sport = srv.getsockname()
if self.args.log_conn:
self.log(
"%s %s" % addr,
"|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
"-" * 3, sip, sport % 8, sport
),
c="1;30",
)
self.num_clients.add()
self.hub.broker.put(False, "httpconn", sck, addr)
self.hub.broker.put(False, "listen", srv)
def shutdown(self):
self.stopping = True

View File

@@ -9,15 +9,11 @@ import hashlib
import threading
import subprocess as sp
from .__init__ import PY2
from .__init__ import PY2, unicode
from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
if not PY2:
unicode = str
HAVE_PIL = False
HAVE_HEIF = False
HAVE_AVIF = False
@@ -53,7 +49,7 @@ except:
# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
# ffmpeg -formats
FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc mts h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
if HAVE_HEIF:
FMT_PIL += " heif heifs heic heics"
@@ -125,18 +121,19 @@ class ThumbSrv(object):
if not self.args.no_vthumb and (not HAVE_FFMPEG or not HAVE_FFPROBE):
missing = []
if not HAVE_FFMPEG:
missing.append("ffmpeg")
missing.append("FFmpeg")
if not HAVE_FFPROBE:
missing.append("ffprobe")
missing.append("FFprobe")
msg = "cannot create video thumbnails because some of the required programs are not available: "
msg += ", ".join(missing)
self.log(msg, c=3)
t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
t.daemon = True
t.start()
if self.args.th_clean:
t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
t.daemon = True
t.start()
def log(self, msg, c=0):
self.log_func("thumb", msg, c)
@@ -263,7 +260,7 @@ class ThumbSrv(object):
pass # default q = 75
if im.mode not in fmts:
print("conv {}".format(im.mode))
# print("conv {}".format(im.mode))
im = im.convert("RGB")
im.save(tpath, quality=40, method=6)

View File

@@ -7,6 +7,7 @@ import time
import threading
from datetime import datetime
from .__init__ import unicode
from .util import s3dec, Pebkac, min_ex
from .up2k import up2k_wark_from_hashlist
@@ -90,6 +91,8 @@ class U2idx(object):
mt_ctr = 0
mt_keycmp = "substr(up.w,1,16)"
mt_keycmp2 = None
ptn_lc = re.compile(r" (mt[0-9]+\.v) ([=<!>]+) \? $")
ptn_lcv = re.compile(r"[a-zA-Z]")
while True:
uq = uq.strip()
@@ -182,6 +185,21 @@ class U2idx(object):
va.append(v)
is_key = True
# lowercase tag searches
m = ptn_lc.search(q)
if not m or not ptn_lcv.search(unicode(v)):
continue
va.pop()
va.append(v.lower())
q = q[: m.start()]
field, oper = m.groups()
if oper in ["=", "=="]:
q += " {} like ? ".format(field)
else:
q += " lower({}) {} ? ".format(field, oper)
try:
return self.run_query(vols, joins + "where " + q, va)
except Exception as ex:

View File

@@ -103,13 +103,15 @@ class Up2k(object):
self.deferred_init()
else:
t = threading.Thread(
target=self.deferred_init,
name="up2k-deferred-init",
target=self.deferred_init, name="up2k-deferred-init", args=(0.5,)
)
t.daemon = True
t.start()
def deferred_init(self):
def deferred_init(self, wait=0):
if wait:
time.sleep(wait)
all_vols = self.asrv.vfs.all_vols
have_e2d = self.init_indexes(all_vols)
@@ -193,7 +195,7 @@ class Up2k(object):
return True, ret
def init_indexes(self, all_vols, scan_vols=[]):
def init_indexes(self, all_vols, scan_vols=None):
self.pp = ProgressPrinter()
vols = all_vols.values()
t0 = time.time()
@@ -302,7 +304,7 @@ class Up2k(object):
self.log(msg.format(len(vols), time.time() - t0))
if needed_mutagen:
msg = "could not read tags because no backends are available (mutagen or ffprobe)"
msg = "could not read tags because no backends are available (Mutagen or FFprobe)"
self.log(msg, c=1)
thr = None
@@ -342,7 +344,15 @@ class Up2k(object):
for k, v in flags.items()
]
if a:
self.log(" ".join(sorted(a)) + "\033[0m")
vpath = "?"
for k, v in self.asrv.vfs.all_vols.items():
if v.realpath == ptop:
vpath = k
if vpath:
vpath += "/"
self.log("/{} {}".format(vpath, " ".join(sorted(a))), "35")
reg = {}
path = os.path.join(histpath, "up2k.snap")
@@ -401,7 +411,7 @@ class Up2k(object):
if WINDOWS:
excl = [x.replace("/", "\\") for x in excl]
n_add = self._build_dir(dbw, top, set(excl), top, nohash)
n_add = self._build_dir(dbw, top, set(excl), top, nohash, [])
n_rm = self._drop_lost(dbw[0], top)
if dbw[1]:
self.log("commit {} new files".format(dbw[1]))
@@ -409,11 +419,25 @@ class Up2k(object):
return True, n_add or n_rm or do_vac
def _build_dir(self, dbw, top, excl, cdir, nohash):
def _build_dir(self, dbw, top, excl, cdir, nohash, seen):
rcdir = cdir
if not ANYWIN:
try:
# a bit expensive but worth
rcdir = os.path.realpath(cdir)
except:
pass
if rcdir in seen:
m = "bailing from symlink loop,\n prev: {}\n curr: {}\n from: {}"
self.log(m.format(seen[-1], rcdir, cdir), 3)
return 0
seen = seen + [cdir]
self.pp.msg = "a{} {}".format(self.pp.n, cdir)
histpath = self.asrv.vfs.histtab[top]
ret = 0
g = statdir(self.log, not self.args.no_scandir, False, cdir)
g = statdir(self.log_func, not self.args.no_scandir, False, cdir)
for iname, inf in sorted(g):
abspath = os.path.join(cdir, iname)
lmod = int(inf.st_mtime)
@@ -422,7 +446,7 @@ class Up2k(object):
if abspath in excl or abspath == histpath:
continue
# self.log(" dir: {}".format(abspath))
ret += self._build_dir(dbw, top, excl, abspath, nohash)
ret += self._build_dir(dbw, top, excl, abspath, nohash, seen)
else:
# self.log("file: {}".format(abspath))
rp = abspath[len(top) + 1 :]
@@ -572,7 +596,7 @@ class Up2k(object):
c2 = conn.cursor()
c3 = conn.cursor()
n_left = cur.execute("select count(w) from up").fetchone()[0]
for w, rd, fn in cur.execute("select w, rd, fn from up"):
for w, rd, fn in cur.execute("select w, rd, fn from up order by rd, fn"):
n_left -= 1
q = "select w from mt where w = ?"
if c2.execute(q, (w[:16],)).fetchone():
@@ -967,7 +991,7 @@ class Up2k(object):
if cj["ptop"] not in self.registry:
raise Pebkac(410, "location unavailable")
cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
cj["name"] = sanitize_fn(cj["name"], "", [".prologue.html", ".epilogue.html"])
cj["poke"] = time.time()
wark = self._get_wark(cj)
now = time.time()
@@ -1339,19 +1363,22 @@ class Up2k(object):
self.log("could not unsparse [{}]".format(path), 3)
def _snapshot(self):
persist_interval = 30 # persist unfinished uploads index every 30 sec
discard_interval = 21600 # drop unfinished uploads after 6 hours inactivity
prev = {}
self.snap_persist_interval = 300 # persist unfinished index every 5 min
self.snap_discard_interval = 21600 # drop unfinished after 6 hours inactivity
self.snap_prev = {}
while True:
time.sleep(persist_interval)
with self.mutex:
for k, reg in self.registry.items():
self._snap_reg(prev, k, reg, discard_interval)
time.sleep(self.snap_persist_interval)
self.do_snapshot()
def _snap_reg(self, prev, ptop, reg, discard_interval):
def do_snapshot(self):
with self.mutex:
for k, reg in self.registry.items():
self._snap_reg(k, reg)
def _snap_reg(self, ptop, reg):
now = time.time()
histpath = self.asrv.vfs.histtab[ptop]
rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
rm = [x for x in reg.values() if now - x["poke"] > self.snap_discard_interval]
if rm:
m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
vis = [self._vis_job_progress(x) for x in rm]
@@ -1373,15 +1400,15 @@ class Up2k(object):
path = os.path.join(histpath, "up2k.snap")
if not reg:
if ptop not in prev or prev[ptop] is not None:
prev[ptop] = None
if ptop not in self.snap_prev or self.snap_prev[ptop] is not None:
self.snap_prev[ptop] = None
if os.path.exists(fsenc(path)):
os.unlink(fsenc(path))
return
newest = max(x["poke"] for _, x in reg.items()) if reg else 0
etag = [len(reg), newest]
if etag == prev.get(ptop):
if etag == self.snap_prev.get(ptop):
return
try:
@@ -1397,7 +1424,7 @@ class Up2k(object):
atomic_move(path2, path)
self.log("snap: {} |{}|".format(path, len(reg.keys())))
prev[ptop] = etag
self.snap_prev[ptop] = etag
def _tagger(self):
with self.mutex:
@@ -1465,6 +1492,11 @@ class Up2k(object):
self.n_hashq += 1
# self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
def shutdown(self):
if hasattr(self, "snap_prev"):
self.log("writing snapshot")
self.do_snapshot()
def up2k_chunksize(filesize):
chunksize = 1024 * 1024
@@ -1480,7 +1512,7 @@ def up2k_chunksize(filesize):
def up2k_wark_from_hashlist(salt, filesize, hashes):
""" server-reproducible file identifier, independent of name or location """
"""server-reproducible file identifier, independent of name or location"""
ident = [salt, str(filesize)]
ident.extend(hashes)
ident = "\n".join(ident)

View File

@@ -16,6 +16,7 @@ import mimetypes
import contextlib
import subprocess as sp # nosec
from datetime import datetime
from collections import Counter
from .__init__ import PY2, WINDOWS, ANYWIN
from .stolen import surrogateescape
@@ -42,6 +43,20 @@ else:
from Queue import Queue # pylint: disable=import-error,no-name-in-module
from StringIO import StringIO as BytesIO
try:
struct.unpack(b">i", b"idgi")
spack = struct.pack
sunpack = struct.unpack
except:
def spack(f, *a, **ka):
return struct.pack(f.decode("ascii"), *a, **ka)
def sunpack(f, *a, **ka):
return struct.unpack(f.decode("ascii"), *a, **ka)
surrogateescape.register_surrogateescape()
FS_ENCODING = sys.getfilesystemencoding()
if WINDOWS and PY2:
@@ -123,20 +138,6 @@ REKOBO_KEY = {
REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
class Counter(object):
def __init__(self, v=0):
self.v = v
self.mutex = threading.Lock()
def add(self, delta=1):
with self.mutex:
self.v += delta
def set(self, absval):
with self.mutex:
self.v = absval
class Cooldown(object):
def __init__(self, maxage):
self.maxage = maxage
@@ -231,7 +232,7 @@ def nuprint(msg):
def rice_tid():
tid = threading.current_thread().ident
c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
c = sunpack(b"B" * 5, spack(b">Q", tid)[-5:])
return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
@@ -282,15 +283,69 @@ def alltrace():
return "\n".join(rret + bret)
def start_stackmon(arg_str, nid):
suffix = "-{}".format(nid) if nid else ""
fp, f = arg_str.rsplit(",", 1)
f = int(f)
t = threading.Thread(
target=stackmon,
args=(fp, f, suffix),
name="stackmon" + suffix,
)
t.daemon = True
t.start()
def stackmon(fp, ival, suffix):
ctr = 0
while True:
ctr += 1
time.sleep(ival)
st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
with open(fp + suffix, "wb") as f:
f.write(st.encode("utf-8", "replace"))
def start_log_thrs(logger, ival, nid):
ival = int(ival)
tname = lname = "log-thrs"
if nid:
tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
lname = tname[3:]
t = threading.Thread(
target=log_thrs,
args=(logger, ival, lname),
name=tname,
)
t.daemon = True
t.start()
def log_thrs(log, ival, name):
while True:
time.sleep(ival)
tv = [x.name for x in threading.enumerate()]
tv = [
x.split("-")[0]
if x.startswith("httpconn-") or x.startswith("thumb-")
else "listen"
if "-listen-" in x
else x
for x in tv
if not x.startswith("pydevd.")
]
tv = ["{}\033[36m{}".format(v, k) for k, v in sorted(Counter(tv).items())]
log(name, "\033[0m \033[33m".join(tv), 3)
def min_ex():
et, ev, tb = sys.exc_info()
tb = traceback.extract_tb(tb, 2)
ex = [
"{} @ {} <{}>: {}".format(fp.split(os.sep)[-1], ln, fun, txt)
for fp, ln, fun, txt in tb
]
ex.append("{}: {}".format(et.__name__, ev))
return "\n".join(ex)
tb = traceback.extract_tb(tb)
fmt = "{} @ {} <{}>: {}"
ex = [fmt.format(fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in tb]
ex.append("[{}] {}".format(et.__name__, ev))
return "\n".join(ex[-8:])
@contextlib.contextmanager
@@ -674,7 +729,7 @@ def undot(path):
return "/".join(ret)
def sanitize_fn(fn, ok="", bad=[]):
def sanitize_fn(fn, ok, bad):
if "/" not in ok:
fn = fn.replace("\\", "/").split("/")[-1]
@@ -904,16 +959,10 @@ def yieldfile(fn):
yield buf
def hashcopy(actor, fin, fout):
is_mp = actor.is_mp
def hashcopy(fin, fout):
hashobj = hashlib.sha512()
tlen = 0
for buf in fin:
if is_mp:
actor.workload += 1
if actor.workload > 2 ** 31:
actor.workload = 100
tlen += len(buf)
hashobj.update(buf)
fout.write(buf)
@@ -924,15 +973,10 @@ def hashcopy(actor, fin, fout):
return tlen, hashobj.hexdigest(), digest_b64
def sendfile_py(lower, upper, f, s, actor=None):
def sendfile_py(lower, upper, f, s):
remains = upper - lower
f.seek(lower)
while remains > 0:
if actor:
actor.workload += 1
if actor.workload > 2 ** 31:
actor.workload = 100
# time.sleep(0.01)
buf = f.read(min(1024 * 32, remains))
if not buf:
@@ -979,8 +1023,7 @@ def statdir(logger, scandir, lstat, top):
try:
yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
except Exception as ex:
msg = "scan-stat: \033[36m{} @ {}"
logger(msg.format(repr(ex), fsdec(fh.path)))
logger(src, "[s] {} @ {}".format(repr(ex), fsdec(fh.path)), 6)
else:
src = "listdir"
fun = os.lstat if lstat else os.stat
@@ -989,11 +1032,10 @@ def statdir(logger, scandir, lstat, top):
try:
yield [fsdec(name), fun(abspath)]
except Exception as ex:
msg = "list-stat: \033[36m{} @ {}"
logger(msg.format(repr(ex), fsdec(abspath)))
logger(src, "[s] {} @ {}".format(repr(ex), fsdec(abspath)), 6)
except Exception as ex:
logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
logger(src, "{} @ {}".format(repr(ex), top), 1)
def unescape_cookie(orig):
@@ -1035,7 +1077,7 @@ def guess_mime(url, fallback="application/octet-stream"):
if ";" not in ret:
if ret.startswith("text/") or ret.endswith("/javascript"):
ret += "; charset=UTF-8"
return ret
@@ -1070,10 +1112,7 @@ def gzip_orig_sz(fn):
with open(fsenc(fn), "rb") as f:
f.seek(-4, 2)
rv = f.read(4)
try:
return struct.unpack(b"I", rv)[0]
except:
return struct.unpack("I", rv)[0]
return sunpack(b"I", rv)[0]
def py_desc():

View File

@@ -13,7 +13,7 @@ window.baguetteBox = (function () {
captions: true,
buttons: 'auto',
noScrollbars: false,
bodyClass: 'baguetteBox-open',
bodyClass: 'bbox-open',
titleTag: false,
async: false,
preload: 2,
@@ -22,37 +22,46 @@ window.baguetteBox = (function () {
afterHide: null,
onChange: null,
},
overlay, slider, previousButton, nextButton, closeButton,
overlay, slider, btnPrev, btnNext, btnHelp, btnVmode, btnClose,
currentGallery = [],
currentIndex = 0,
isOverlayVisible = false,
touch = {}, // start-pos
touchFlag = false, // busy
regex = /.+\.(gif|jpe?g|png|webp)/i,
re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
re_v = /.+\.(webm|mp4)(\?|$)/i,
data = {}, // all galleries
imagesElements = [],
documentLastFocus = null;
documentLastFocus = null,
isFullscreen = false,
vmute = false,
vloop = false,
vnext = false,
resume_mp = false;
var overlayClickHandler = function (event) {
if (event.target.id.indexOf('baguette-img') !== -1) {
var onFSC = function (e) {
isFullscreen = !!document.fullscreenElement;
};
var overlayClickHandler = function (e) {
if (e.target.id.indexOf('baguette-img') !== -1)
hideOverlay();
}
};
var touchstartHandler = function (event) {
var touchstartHandler = function (e) {
touch.count++;
if (touch.count > 1) {
if (touch.count > 1)
touch.multitouch = true;
}
touch.startX = event.changedTouches[0].pageX;
touch.startY = event.changedTouches[0].pageY;
touch.startX = e.changedTouches[0].pageX;
touch.startY = e.changedTouches[0].pageY;
};
var touchmoveHandler = function (event) {
if (touchFlag || touch.multitouch) {
var touchmoveHandler = function (e) {
if (touchFlag || touch.multitouch)
return;
}
event.preventDefault ? event.preventDefault() : event.returnValue = false;
var touchEvent = event.touches[0] || event.changedTouches[0];
e.preventDefault ? e.preventDefault() : e.returnValue = false;
var touchEvent = e.touches[0] || e.changedTouches[0];
if (touchEvent.pageX - touch.startX > 40) {
touchFlag = true;
showPreviousImage();
@@ -65,19 +74,19 @@ window.baguetteBox = (function () {
};
var touchendHandler = function () {
touch.count--;
if (touch.count <= 0) {
if (touch.count <= 0)
touch.multitouch = false;
}
touchFlag = false;
};
var contextmenuHandler = function () {
touchendHandler();
};
var trapFocusInsideOverlay = function (event) {
if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(event.target))) {
event.stopPropagation();
initFocus();
var trapFocusInsideOverlay = function (e) {
if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(e.target))) {
e.stopPropagation();
btnClose.focus();
}
};
@@ -88,7 +97,7 @@ window.baguetteBox = (function () {
}
function bindImageClickListeners(selector, userOptions) {
var galleryNodeList = document.querySelectorAll(selector);
var galleryNodeList = QSA(selector);
var selectorData = {
galleries: [],
nodeList: galleryNodeList
@@ -96,33 +105,26 @@ window.baguetteBox = (function () {
data[selector] = selectorData;
[].forEach.call(galleryNodeList, function (galleryElement) {
if (userOptions && userOptions.filter) {
regex = userOptions.filter;
}
var tagsNodeList = [];
if (galleryElement.tagName === 'A') {
if (galleryElement.tagName === 'A')
tagsNodeList = [galleryElement];
} else {
else
tagsNodeList = galleryElement.getElementsByTagName('a');
}
tagsNodeList = [].filter.call(tagsNodeList, function (element) {
if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1) {
return regex.test(element.href);
}
if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1)
return re_i.test(element.href) || re_v.test(element.href);
});
if (tagsNodeList.length === 0) {
if (!tagsNodeList.length)
return;
}
var gallery = [];
[].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
var imageElementClickHandler = function (event) {
if (event && (event.ctrlKey || event.metaKey))
var imageElementClickHandler = function (e) {
if (ctrl(e))
return true;
event.preventDefault ? event.preventDefault() : event.returnValue = false;
e.preventDefault ? e.preventDefault() : e.returnValue = false;
prepareOverlay(gallery, userOptions);
showOverlay(imageIndex);
};
@@ -140,93 +142,186 @@ window.baguetteBox = (function () {
}
function clearCachedData() {
for (var selector in data) {
if (data.hasOwnProperty(selector)) {
for (var selector in data)
if (data.hasOwnProperty(selector))
removeFromCache(selector);
}
}
}
function removeFromCache(selector) {
if (!data.hasOwnProperty(selector)) {
if (!data.hasOwnProperty(selector))
return;
}
var galleries = data[selector].galleries;
[].forEach.call(galleries, function (gallery) {
[].forEach.call(gallery, function (imageItem) {
unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
});
if (currentGallery === gallery) {
if (currentGallery === gallery)
currentGallery = [];
}
});
delete data[selector];
}
function buildOverlay() {
overlay = ebi('baguetteBox-overlay');
if (overlay) {
slider = ebi('baguetteBox-slider');
previousButton = ebi('previous-button');
nextButton = ebi('next-button');
closeButton = ebi('close-button');
return;
overlay = ebi('bbox-overlay');
if (!overlay) {
var ctr = mknod('div');
ctr.innerHTML = (
'<div id="bbox-overlay" role="dialog">' +
'<div id="bbox-slider"></div>' +
'<button id="bbox-prev" class="bbox-btn" type="button" aria-label="Previous">&lt;</button>' +
'<button id="bbox-next" class="bbox-btn" type="button" aria-label="Next">&gt;</button>' +
'<div id="bbox-btns">' +
'<button id="bbox-help" type="button">?</button>' +
'<button id="bbox-vmode" type="button" tt="a"></button>' +
'<button id="bbox-close" type="button" aria-label="Close">X</button>' +
'</div></div>'
);
overlay = ctr.firstChild;
QS('body').appendChild(overlay);
tt.att(overlay);
}
overlay = mknod('div');
overlay.setAttribute('role', 'dialog');
overlay.id = 'baguetteBox-overlay';
document.getElementsByTagName('body')[0].appendChild(overlay);
slider = mknod('div');
slider.id = 'baguetteBox-slider';
overlay.appendChild(slider);
previousButton = mknod('button');
previousButton.setAttribute('type', 'button');
previousButton.id = 'previous-button';
previousButton.setAttribute('aria-label', 'Previous');
previousButton.innerHTML = '&lt;';
overlay.appendChild(previousButton);
nextButton = mknod('button');
nextButton.setAttribute('type', 'button');
nextButton.id = 'next-button';
nextButton.setAttribute('aria-label', 'Next');
nextButton.innerHTML = '&gt;';
overlay.appendChild(nextButton);
closeButton = mknod('button');
closeButton.setAttribute('type', 'button');
closeButton.id = 'close-button';
closeButton.setAttribute('aria-label', 'Close');
closeButton.innerHTML = '&times;';
overlay.appendChild(closeButton);
previousButton.className = nextButton.className = closeButton.className = 'baguetteBox-button';
slider = ebi('bbox-slider');
btnPrev = ebi('bbox-prev');
btnNext = ebi('bbox-next');
btnHelp = ebi('bbox-help');
btnVmode = ebi('bbox-vmode');
btnClose = ebi('bbox-close');
bindEvents();
}
function keyDownHandler(event) {
switch (event.keyCode) {
case 37: // Left
showPreviousImage();
break;
case 39: // Right
showNextImage();
break;
case 27: // Esc
hideOverlay();
break;
case 36: // Home
showFirstImage(event);
break;
case 35: // End
showLastImage(event);
break;
function halp() {
if (ebi('bbox-halp'))
return;
var list = [
['<b># hotkey</b>', '<b># operation</b>'],
['escape', 'close'],
['left, J', 'previous file'],
['right, L', 'next file'],
['home', 'first file'],
['end', 'last file'],
['space, P, K', 'video: play / pause'],
['U', 'video: seek 10sec back'],
['P', 'video: seek 10sec ahead'],
['M', 'video: toggle mute'],
['R', 'video: toggle loop'],
['C', 'video: toggle auto-next'],
['F', 'video: toggle fullscreen'],
],
d = mknod('table'),
html = ['<tbody>'];
for (var a = 0; a < list.length; a++)
html.push('<tr><td>' + list[a][0] + '</td><td>' + list[a][1] + '</td></tr>');
d.innerHTML = html.join('\n') + '</tbody>';
d.setAttribute('id', 'bbox-halp');
d.onclick = function () {
overlay.removeChild(d);
};
overlay.appendChild(d);
}
function keyDownHandler(e) {
if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
return;
var k = e.code + '', v = vid();
if (k == "ArrowLeft" || k == "KeyJ")
showPreviousImage();
else if (k == "ArrowRight" || k == "KeyL")
showNextImage();
else if (k == "Escape")
hideOverlay();
else if (k == "Home")
showFirstImage(e);
else if (k == "End")
showLastImage(e);
else if (k == "Space" || k == "KeyP" || k == "KeyK")
playpause();
else if (k == "KeyU" || k == "KeyO")
relseek(k == "KeyU" ? -10 : 10);
else if (k == "KeyM" && v) {
v.muted = vmute = !vmute;
mp_ctl();
}
else if (k == "KeyR" && v) {
vloop = !vloop;
vnext = vnext && !vloop;
setVmode();
}
else if (k == "KeyC" && v) {
vnext = !vnext;
vloop = vloop && !vnext;
setVmode();
}
else if (k == "KeyF")
try {
if (isFullscreen)
document.exitFullscreen();
else
v.requestFullscreen();
}
catch (ex) { }
}
function setVmode() {
var v = vid();
ebi('bbox-vmode').style.display = v ? '' : 'none';
if (!v)
return;
var msg = 'When video ends, ', tts = '', lbl;
if (vloop) {
lbl = 'Loop';
msg += 'repeat it';
tts = '$NHotkey: R';
}
else if (vnext) {
lbl = 'Cont';
msg += 'continue to next';
tts = '$NHotkey: C';
}
else {
lbl = 'Stop';
msg += 'just stop'
}
btnVmode.setAttribute('aria-label', msg);
btnVmode.setAttribute('tt', msg + tts);
btnVmode.textContent = lbl;
v.loop = vloop
if (vloop && v.paused)
v.play();
}
function tglVmode() {
if (vloop) {
vnext = true;
vloop = false;
}
else if (vnext)
vnext = false;
else
vloop = true;
setVmode();
if (tt.en)
tt.show.bind(this)();
}
function keyUpHandler(e) {
if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
return;
var k = e.code + '';
if (k == "Space")
ev(e);
}
var passiveSupp = false;
@@ -248,9 +343,11 @@ window.baguetteBox = (function () {
function bindEvents() {
bind(overlay, 'click', overlayClickHandler);
bind(previousButton, 'click', showPreviousImage);
bind(nextButton, 'click', showNextImage);
bind(closeButton, 'click', hideOverlay);
bind(btnPrev, 'click', showPreviousImage);
bind(btnNext, 'click', showNextImage);
bind(btnClose, 'click', hideOverlay);
bind(btnVmode, 'click', tglVmode);
bind(btnHelp, 'click', halp);
bind(slider, 'contextmenu', contextmenuHandler);
bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
@@ -260,9 +357,11 @@ window.baguetteBox = (function () {
function unbindEvents() {
unbind(overlay, 'click', overlayClickHandler);
unbind(previousButton, 'click', showPreviousImage);
unbind(nextButton, 'click', showNextImage);
unbind(closeButton, 'click', hideOverlay);
unbind(btnPrev, 'click', showPreviousImage);
unbind(btnNext, 'click', showNextImage);
unbind(btnClose, 'click', hideOverlay);
unbind(btnVmode, 'click', tglVmode);
unbind(btnHelp, 'click', halp);
unbind(slider, 'contextmenu', contextmenuHandler);
unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
@@ -271,9 +370,9 @@ window.baguetteBox = (function () {
}
function prepareOverlay(gallery, userOptions) {
if (currentGallery === gallery) {
if (currentGallery === gallery)
return;
}
currentGallery = gallery;
setOptions(userOptions);
slider.innerHTML = '';
@@ -287,8 +386,8 @@ window.baguetteBox = (function () {
fullImage.id = 'baguette-img-' + i;
imagesElements.push(fullImage);
imagesFiguresIds.push('baguetteBox-figure-' + i);
imagesCaptionsIds.push('baguetteBox-figcaption-' + i);
imagesFiguresIds.push('bbox-figure-' + i);
imagesCaptionsIds.push('bbox-figcaption-' + i);
slider.appendChild(imagesElements[i]);
}
overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
@@ -296,23 +395,21 @@ window.baguetteBox = (function () {
}
function setOptions(newOptions) {
if (!newOptions) {
if (!newOptions)
newOptions = {};
}
for (var item in defaults) {
options[item] = defaults[item];
if (typeof newOptions[item] !== 'undefined') {
if (typeof newOptions[item] !== 'undefined')
options[item] = newOptions[item];
}
}
slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
options.animation === 'slideIn' ? '' : 'none');
if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1)) {
if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1))
options.buttons = false;
}
previousButton.style.display = nextButton.style.display = (options.buttons ? '' : 'none');
btnPrev.style.display = btnNext.style.display = (options.buttons ? '' : 'none');
}
function showOverlay(chosenImageIndex) {
@@ -320,11 +417,12 @@ window.baguetteBox = (function () {
document.documentElement.style.overflowY = 'hidden';
document.body.style.overflowY = 'scroll';
}
if (overlay.style.display === 'block') {
if (overlay.style.display === 'block')
return;
}
bind(document, 'keydown', keyDownHandler);
bind(document, 'keyup', keyUpHandler);
bind(document, 'fullscreenchange', onFSC);
currentIndex = chosenImageIndex;
touch = {
count: 0,
@@ -341,50 +439,48 @@ window.baguetteBox = (function () {
// Fade in overlay
setTimeout(function () {
overlay.className = 'visible';
if (options.bodyClass && document.body.classList) {
if (options.bodyClass && document.body.classList)
document.body.classList.add(options.bodyClass);
}
if (options.afterShow) {
options.afterShow();
}
}, 50);
if (options.onChange) {
options.onChange(currentIndex, imagesElements.length);
}
documentLastFocus = document.activeElement;
initFocus();
isOverlayVisible = true;
}
function initFocus() {
if (options.buttons) {
previousButton.focus();
} else {
closeButton.focus();
}
if (options.afterShow)
options.afterShow();
}, 50);
if (options.onChange)
options.onChange(currentIndex, imagesElements.length);
documentLastFocus = document.activeElement;
btnClose.focus();
isOverlayVisible = true;
}
function hideOverlay(e) {
ev(e);
playvid(false);
if (options.noScrollbars) {
document.documentElement.style.overflowY = 'auto';
document.body.style.overflowY = 'auto';
}
if (overlay.style.display === 'none') {
if (overlay.style.display === 'none')
return;
}
unbind(document, 'keydown', keyDownHandler);
unbind(document, 'keyup', keyUpHandler);
unbind(document, 'fullscreenchange', onFSC);
// Fade out and hide the overlay
overlay.className = '';
setTimeout(function () {
overlay.style.display = 'none';
if (options.bodyClass && document.body.classList) {
if (options.bodyClass && document.body.classList)
document.body.classList.remove(options.bodyClass);
}
if (options.afterHide) {
var h = ebi('bbox-halp');
if (h)
h.parentNode.removeChild(h);
if (options.afterHide)
options.afterHide();
}
documentLastFocus && documentLastFocus.focus();
isOverlayVisible = false;
}, 500);
@@ -394,59 +490,68 @@ window.baguetteBox = (function () {
var imageContainer = imagesElements[index];
var galleryItem = currentGallery[index];
if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined') {
if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined')
return; // out-of-bounds or gallery dirty
}
if (imageContainer.getElementsByTagName('img')[0]) {
// image is loaded, cb and bail
if (callback) {
callback();
}
return;
}
if (imageContainer.querySelector('img, video'))
// was loaded, cb and bail
return callback ? callback() : null;
// maybe unloaded video
while (imageContainer.firstChild)
imageContainer.removeChild(imageContainer.firstChild);
var imageElement = galleryItem.imageElement,
imageSrc = imageElement.href,
thumbnailElement = imageElement.getElementsByTagName('img')[0],
is_vid = re_v.test(imageSrc),
thumbnailElement = imageElement.querySelector('img, video'),
imageCaption = typeof options.captions === 'function' ?
options.captions.call(currentGallery, imageElement) :
imageElement.getAttribute('data-caption') || imageElement.title;
imageSrc += imageSrc.indexOf('?') < 0 ? '?cache' : '&cache';
if (is_vid && index != currentIndex)
return; // no preload
var figure = mknod('figure');
figure.id = 'baguetteBox-figure-' + index;
figure.innerHTML = '<div class="baguetteBox-spinner">' +
'<div class="baguetteBox-double-bounce1"></div>' +
'<div class="baguetteBox-double-bounce2"></div>' +
figure.id = 'bbox-figure-' + index;
figure.innerHTML = '<div class="bbox-spinner">' +
'<div class="bbox-double-bounce1"></div>' +
'<div class="bbox-double-bounce2"></div>' +
'</div>';
if (options.captions && imageCaption) {
var figcaption = mknod('figcaption');
figcaption.id = 'baguetteBox-figcaption-' + index;
figcaption.id = 'bbox-figcaption-' + index;
figcaption.innerHTML = imageCaption;
figure.appendChild(figcaption);
}
imageContainer.appendChild(figure);
var image = mknod('img');
image.onload = function () {
var image = mknod(is_vid ? 'video' : 'img');
clmod(imageContainer, 'vid', is_vid);
image.addEventListener(is_vid ? 'loadedmetadata' : 'load', function () {
// Remove loader element
var spinner = document.querySelector('#baguette-img-' + index + ' .baguetteBox-spinner');
var spinner = QS('#baguette-img-' + index + ' .bbox-spinner');
figure.removeChild(spinner);
if (!options.async && callback) {
if (!options.async && callback)
callback();
}
};
});
image.setAttribute('src', imageSrc);
image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
if (options.titleTag && imageCaption) {
image.title = imageCaption;
if (is_vid) {
image.setAttribute('controls', 'controls');
image.onended = vidEnd;
}
image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
if (options.titleTag && imageCaption)
image.title = imageCaption;
figure.appendChild(image);
if (options.async && callback) {
if (options.async && callback)
callback();
}
}
function showNextImage(e) {
@@ -459,26 +564,20 @@ window.baguetteBox = (function () {
return show(currentIndex - 1);
}
function showFirstImage(event) {
if (event) {
event.preventDefault();
}
function showFirstImage(e) {
if (e)
e.preventDefault();
return show(0);
}
function showLastImage(event) {
if (event) {
event.preventDefault();
}
function showLastImage(e) {
if (e)
e.preventDefault();
return show(currentGallery.length - 1);
}
/**
* Move the gallery to a specific index
* @param `index` {number} - the position of the image
* @param `gallery` {array} - gallery which should be opened, if omitted assumes the currently opened one
* @return {boolean} - true on success or false if the index is invalid
*/
function show(index, gallery) {
if (!isOverlayVisible && index >= 0 && index < gallery.length) {
prepareOverlay(gallery, options);
@@ -486,18 +585,25 @@ window.baguetteBox = (function () {
return true;
}
if (index < 0) {
if (options.animation) {
if (options.animation)
bounceAnimation('left');
}
return false;
}
if (index >= imagesElements.length) {
if (options.animation) {
if (options.animation)
bounceAnimation('right');
}
return false;
}
var v = vid();
if (v) {
v.src = '';
v.load();
v.parentNode.removeChild(v);
}
currentIndex = index;
loadImage(currentIndex, function () {
preloadNext(currentIndex);
@@ -505,17 +611,49 @@ window.baguetteBox = (function () {
});
updateOffset();
if (options.onChange) {
if (options.onChange)
options.onChange(currentIndex, imagesElements.length);
}
return true;
}
/**
* Triggers the bounce animation
* @param {('left'|'right')} direction - Direction of the movement
*/
function vid() {
return imagesElements[currentIndex].querySelector('video');
}
function playvid(play) {
if (vid())
vid()[play ? 'play' : 'pause']();
}
function playpause() {
var v = vid();
if (v)
v[v.paused ? "play" : "pause"]();
}
function relseek(sec) {
if (vid())
vid().currentTime += sec;
}
function vidEnd() {
if (this == vid() && vnext)
showNextImage();
}
function mp_ctl() {
var v = vid();
if (!vmute && v && mp.au && !mp.au.paused) {
mp.fade_out();
resume_mp = true;
}
else if (resume_mp && (vmute || !v) && mp.au && mp.au.paused) {
mp.fade_in();
resume_mp = false;
}
}
function bounceAnimation(direction) {
slider.className = 'bounce-from-' + direction;
setTimeout(function () {
@@ -534,21 +672,30 @@ window.baguetteBox = (function () {
} else {
slider.style.transform = 'translate3d(' + offset + ',0,0)';
}
playvid(false);
var v = vid();
if (v) {
playvid(true);
v.muted = vmute;
v.loop = vloop;
}
mp_ctl();
setVmode();
}
function preloadNext(index) {
if (index - currentIndex >= options.preload) {
if (index - currentIndex >= options.preload)
return;
}
loadImage(index + 1, function () {
preloadNext(index + 1);
});
}
function preloadPrev(index) {
if (currentIndex - index >= options.preload) {
if (currentIndex - index >= options.preload)
return;
}
loadImage(index - 1, function () {
preloadPrev(index - 1);
});
@@ -566,7 +713,8 @@ window.baguetteBox = (function () {
unbindEvents();
clearCachedData();
unbind(document, 'keydown', keyDownHandler);
document.getElementsByTagName('body')[0].removeChild(ebi('baguetteBox-overlay'));
unbind(document, 'keyup', keyUpHandler);
document.getElementsByTagName('body')[0].removeChild(ebi('bbox-overlay'));
data = {};
currentGallery = [];
currentIndex = 0;
@@ -577,6 +725,8 @@ window.baguetteBox = (function () {
show: show,
showNext: showNextImage,
showPrevious: showPreviousImage,
relseek: relseek,
playpause: playpause,
hide: hideOverlay,
destroy: destroyPlugin
};

View File

@@ -29,10 +29,10 @@ body {
position: fixed;
max-width: 34em;
background: #222;
border: 0 solid #555;
border: 0 solid #777;
overflow: hidden;
margin-top: 1em;
padding: 0 1em;
padding: 0 1.3em;
height: 0;
opacity: .1;
transition: opacity 0.14s, height 0.14s, padding 0.14s;
@@ -40,19 +40,31 @@ body {
border-radius: .4em;
z-index: 9001;
}
#tt.b {
padding: 0 2em;
border-radius: .5em;
box-shadow: 0 .2em 1em #000;
}
#tt.show {
padding: 1em;
padding: 1em 1.3em;
border-width: .4em 0;
height: auto;
border-width: .2em 0;
opacity: 1;
}
#tt.show.b {
padding: 1.5em 2em;
border-width: .5em 0;
}
#tt code {
background: #3c3c3c;
padding: .2em .3em;
padding: .1em .3em;
border-top: 1px solid #777;
border-radius: .3em;
font-family: monospace, monospace;
line-height: 2em;
line-height: 1.7em;
}
#tt em {
color: #f6a;
}
#path,
#path * {
@@ -199,15 +211,39 @@ a, #files tbody div a:last-child {
margin: .8em 0;
}
#srv_info {
opacity: .5;
font-size: .8em;
color: #fc5;
color: #a73;
background: #333;
position: absolute;
top: .5em;
font-size: .8em;
top: .5em;
left: 2em;
padding-right: .5em;
}
#srv_info span {
color: #fff;
color: #aaa;
}
#acc_info {
position: absolute;
font-size: .81em;
top: .5em;
right: 2em;
color: #999;
}
#acc_info span:before {
color: #f4c;
border-bottom: 1px solid rgba(255,68,204,0.6);
margin-right: .6em;
}
html.read #acc_info span:before {
content: 'Read-Only access';
}
html.write #acc_info span:before {
content: 'Write-Only access';
}
html.read.write #acc_info span:before {
content: 'Read-Write access';
color: #999;
border: none;
}
#files tbody a.play {
color: #e70;
@@ -325,10 +361,18 @@ html.light #ggrid a.sel {
height: 100%;
background: #3c3c3c;
}
#wtgrid,
#wtico {
cursor: url(/.cpr/dd/4.png), pointer;
animation: cursor 500ms;
position: relative;
top: -.06em;
}
#wtgrid {
font-size: .8em;
top: -.12em;
}
#wtgrid:hover,
#wtico:hover {
animation: cursor 500ms infinite;
}
@@ -344,9 +388,10 @@ html.light #ggrid a.sel {
}
#wtoggle {
position: absolute;
white-space: nowrap;
top: -1.2em;
right: 0;
width: 1.2em;
width: 2.5em;
height: 1em;
font-size: 2em;
line-height: 1em;
@@ -378,10 +423,10 @@ html.light #ggrid a.sel {
line-height: 1em;
}
#wtoggle.np {
width: 5.5em;
width: 6.63em;
}
#wtoggle.sel {
width: 6.4em;
width: 7.57em;
}
#wtoggle.sel #wzip,
#wtoggle.np #wnp {
@@ -464,6 +509,17 @@ html.light #ggrid a.sel {
max-width: 9em;
}
}
@media (max-width: 35em) {
#ops>a[data-dest="new_md"],
#ops>a[data-dest="msg"] {
display: none;
}
#op_mkdir.act+div,
#op_mkdir.act+div+div {
display: block;
margin-top: 1em;
}
}
@@ -761,9 +817,14 @@ input.eq_gain {
display: block;
width: 1em;
border-radius: .2em;
margin: -1.3em auto 0 auto;
margin: -1.2em auto 0 auto;
top: 2em;
position: relative;
background: #444;
}
#files th span {
position: relative;
}
#files>thead>tr>th.min,
#files td.min {
display: none;
@@ -812,11 +873,13 @@ input.eq_gain {
border-bottom: 1px solid #555;
}
#thumbs,
#au_osd_cv {
#au_osd_cv,
#u2tdate {
opacity: .3;
}
#griden.on+#thumbs,
#au_os_ctl.on+#au_osd_cv {
#au_os_ctl.on+#au_osd_cv,
#u2turbo.on+#u2tdate {
opacity: 1;
}
#ghead {
@@ -888,7 +951,8 @@ html.light #ggrid a:hover {
#pvol,
#barbuf,
#barpos,
#u2conf label {
#u2conf label,
#ops {
-webkit-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
@@ -921,13 +985,16 @@ html.light {
}
html.light #tt {
background: #fff;
border-color: #888;
border-color: #888 #000 #777 #000;
box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
}
html.light #tt code {
background: #060;
color: #fff;
}
html.light #tt em {
color: #d38;
}
html.light #ops,
html.light .opbox,
html.light #srch_form {
@@ -958,10 +1025,14 @@ html.light .tgl.btn.on {
}
html.light #srv_info {
color: #c83;
background: #eee;
}
html.light #srv_info,
html.light #acc_info {
text-shadow: 1px 1px 0 #fff;
}
html.light #srv_info span {
color: #000;
color: #777;
}
html.light #treeul a+a {
background: inherit;
@@ -1128,7 +1199,7 @@ html.light #tree::-webkit-scrollbar {
#baguetteBox-overlay {
#bbox-overlay {
display: none;
opacity: 0;
position: fixed;
@@ -1138,58 +1209,66 @@ html.light #tree::-webkit-scrollbar {
left: 0;
width: 100%;
height: 100%;
z-index: 1000000;
z-index: 10;
background: rgba(0, 0, 0, 0.8);
transition: opacity .3s ease;
}
#baguetteBox-overlay.visible {
#bbox-overlay.visible {
opacity: 1;
}
#baguetteBox-overlay .full-image {
.full-image {
display: inline-block;
position: relative;
width: 100%;
height: 100%;
text-align: center;
}
#baguetteBox-overlay .full-image figure {
.full-image figure {
display: inline;
margin: 0;
height: 100%;
}
#baguetteBox-overlay .full-image img {
.full-image img,
.full-image video {
display: inline-block;
width: auto;
height: auto;
max-height: 100%;
max-width: 100%;
max-height: 100%;
max-height: calc(100% - 1.4em);
margin-bottom: 1.4em;
vertical-align: middle;
box-shadow: 0 0 8px rgba(0, 0, 0, 0.6);
}
#baguetteBox-overlay .full-image figcaption {
.full-image video {
background: #333;
}
.full-image figcaption {
display: block;
position: absolute;
bottom: 0;
position: fixed;
bottom: .1em;
width: 100%;
text-align: center;
line-height: 1.8;
white-space: normal;
color: #ccc;
}
#baguetteBox-overlay figcaption a {
#bbox-overlay figcaption a {
background: rgba(0, 0, 0, 0.6);
border-radius: .4em;
padding: .3em .6em;
}
#baguetteBox-overlay .full-image:before {
html.light #bbox-overlay figcaption a {
color: #0bf;
}
.full-image:before {
content: "";
display: inline-block;
height: 50%;
width: 1px;
margin-right: -1px;
}
#baguetteBox-slider {
position: absolute;
#bbox-slider {
position: fixed;
left: 0;
top: 0;
height: 100%;
@@ -1197,10 +1276,10 @@ html.light #tree::-webkit-scrollbar {
white-space: nowrap;
transition: left .2s ease, transform .2s ease;
}
#baguetteBox-slider.bounce-from-right {
.bounce-from-right {
animation: bounceFromRight .4s ease-out;
}
#baguetteBox-slider.bounce-from-left {
.bounce-from-left {
animation: bounceFromLeft .4s ease-out;
}
@keyframes bounceFromRight {
@@ -1213,48 +1292,63 @@ html.light #tree::-webkit-scrollbar {
50% {margin-left: 30px}
100% {margin-left: 0}
}
.baguetteBox-button#next-button,
.baguetteBox-button#previous-button {
#bbox-next,
#bbox-prev {
top: 50%;
top: calc(50% - 30px);
width: 44px;
height: 60px;
}
.baguetteBox-button {
position: absolute;
.bbox-btn {
position: fixed;
}
#bbox-overlay button {
cursor: pointer;
outline: none;
padding: 0;
margin: 0;
padding: 0 .3em;
margin: 0 .4em;
border: 0;
border-radius: 15%;
background: rgba(50, 50, 50, 0.5);
color: #ddd;
font: 1.6em sans-serif;
color: rgba(255,255,255,0.7);
transition: background-color .3s ease;
transition: color .3s ease;
font-size: 1.4em;
line-height: 1.4em;
vertical-align: top;
}
.baguetteBox-button:focus,
.baguetteBox-button:hover {
#bbox-overlay button:focus,
#bbox-overlay button:hover {
color: rgba(255,255,255,0.9);
background: rgba(50, 50, 50, 0.9);
}
#next-button {
#bbox-next {
right: 1%;
}
#bbox-prev {
left: 1%;
}
#bbox-btns {
top: .5em;
right: 2%;
position: fixed;
}
#previous-button {
left: 2%;
}
#close-button {
top: 20px;
right: 2%;
width: 30px;
height: 30px;
}
.baguetteBox-button svg {
#bbox-halp {
color: #fff;
background: #333;
position: absolute;
left: 0;
top: 0;
left: 0;
z-index: 20;
padding: .4em;
}
.baguetteBox-spinner {
#bbox-halp td {
padding: .2em .5em;
}
#bbox-halp td:first-child {
text-align: right;
}
.bbox-spinner {
width: 40px;
height: 40px;
display: inline-block;
@@ -1264,8 +1358,8 @@ html.light #tree::-webkit-scrollbar {
margin-top: -20px;
margin-left: -20px;
}
.baguetteBox-double-bounce1,
.baguetteBox-double-bounce2 {
.bbox-double-bounce1,
.bbox-double-bounce2 {
width: 100%;
height: 100%;
border-radius: 50%;
@@ -1276,7 +1370,7 @@ html.light #tree::-webkit-scrollbar {
left: 0;
animation: bounce 2s infinite ease-in-out;
}
.baguetteBox-double-bounce2 {
.bbox-double-bounce2 {
animation-delay: -1s;
}
@keyframes bounce {

View File

@@ -39,23 +39,23 @@
<div id="op_mkdir" class="opview opbox act">
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
<input type="hidden" name="act" value="mkdir" />
<input type="text" name="name" size="30">
<input type="submit" value="mkdir">
📂<input type="text" name="name" size="30">
<input type="submit" value="make directory">
</form>
</div>
<div id="op_new_md" class="opview opbox">
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
<input type="hidden" name="act" value="new_md" />
<input type="text" name="name" size="30">
<input type="submit" value="create doc">
📝<input type="text" name="name" size="30">
<input type="submit" value="new markdown doc">
</form>
</div>
<div id="op_msg" class="opview opbox act">
<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
<input type="text" name="msg" size="30">
<input type="submit" value="send msg">
📟<input type="text" name="msg" size="30">
<input type="submit" value="send msg to server log">
</form>
</div>
@@ -121,7 +121,8 @@
<div id="widget"></div>
<script>
var perms = {{ perms }},
var acct = "{{ acct }}",
perms = {{ perms }},
tag_order_cfg = {{ tag_order }},
have_up2k_idx = {{ have_up2k_idx|tojson }},
have_tags_idx = {{ have_tags_idx|tojson }},

View File

@@ -37,6 +37,7 @@ ebi('widget').innerHTML = (
' href="#" id="npirc" tt="copy irc-formatted track info">📋irc</a><a' +
' href="#" id="nptxt" tt="copy plaintext track info">📋txt</a>' +
'</span><a' +
' href="#" id="wtgrid">田</a><a' +
' href="#" id="wtico">♫</a>' +
'</div>' +
'<div id="widgeti">' +
@@ -133,6 +134,13 @@ ebi('op_cfg').innerHTML = (
(have_zip ? (
'<div><h3>folder download</h3><div id="arc_fmt"></div></div>\n'
) : '') +
'<div>\n' +
' <h3>up2k switches</h3>\n' +
' <div>\n' +
' <a id="u2turbo" class="tgl btn ttb" href="#" tt="the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them">turbo</a>\n' +
' <a id="u2tdate" class="tgl btn ttb" href="#" tt="has no effect unless the turbo button is enabled$N$Nreduces the yolo factor by a tiny amount; checks whether the file timestamps on the server matches yours$N$Nshould <em>theoretically</em> catch most unfinished/corrupted uploads, but is not a substitute for doing a verification pass with turbo disabled afterwards">date-chk</a>\n' +
' </div>\n' +
'</div>\n' +
'<div><h3>key notation</h3><div id="key_notation"></div></div>\n' +
'<div class="fill"><h3>hidden columns</h3><div id="hcols"></div></div>'
);
@@ -142,8 +150,8 @@ ebi('op_cfg').innerHTML = (
ebi('tree').innerHTML = (
'<div id="treeh">\n' +
' <a href="#" id="detree" tt="show breadcrumbs$NHotkey: B">🍞...</a>\n' +
' <a href="#" class="btn" step="2" id="twobytwo">+</a>\n' +
' <a href="#" class="btn" step="-2" id="twig">&ndash;</a>\n' +
' <a href="#" class="btn" step="2" id="twobytwo" tt="Hotkey: A">+</a>\n' +
' <a href="#" class="btn" step="-2" id="twig" tt="Hotkey: D">&ndash;</a>\n' +
' <a href="#" class="tgl btn" id="dyntree" tt="autogrow as tree expands">a</a>\n' +
'</div>\n' +
'<ul id="treeul"></ul>\n' +
@@ -160,16 +168,19 @@ ebi('tree').innerHTML = (
function opclick(e) {
ev(e);
var dest = this.getAttribute('data-dest');
swrite('opmode', dest || null);
if (ctrl(e))
return;
ev(e);
goto(dest);
swrite('opmode', dest || null);
var input = QS('.opview.act input:not([type="hidden"])')
if (input && !is_touch)
if (input && !is_touch) {
tt.skip = true;
input.focus();
}
}
@@ -597,7 +608,7 @@ var widget = (function () {
m += '[' + cv + s2ms(mp.au.currentTime) + ck + '/' + cv + s2ms(mp.au.duration) + ck + ']';
var o = document.createElement('input');
var o = mknod('input');
o.style.cssText = 'position:fixed;top:45%;left:48%;padding:1em;z-index:9';
o.value = m;
document.body.appendChild(o);
@@ -1135,7 +1146,7 @@ var audio_eq = (function () {
v = parseFloat(vs);
if (isNaN(v) || v + '' != vs)
throw 42;
throw new Error('inval band');
if (isNaN(band))
r.amp = Math.round((v + step * 0.2) * 100) / 100;
@@ -1441,17 +1452,24 @@ function play_linked() {
};
(function () {
var d = mknod('div');
d.setAttribute('id', 'acc_info');
document.body.insertBefore(d, ebi('ops'));
})();
var thegrid = (function () {
var lfiles = ebi('files'),
gfiles = document.createElement('div');
gfiles = mknod('div');
gfiles.setAttribute('id', 'gfiles');
gfiles.style.display = 'none';
gfiles.innerHTML = (
'<div id="ghead">' +
'<a href="#" class="tgl btn" id="gridsel" tt="enable file selection; ctrl-click a file to override$NHotkey: S">multiselect</a> &nbsp; zoom ' +
'<a href="#" class="btn" z="-1.2" tt="Hotkey: A">&ndash;</a> ' +
'<a href="#" class="btn" z="1.2" tt="Hotkey: D">+</a> &nbsp; sort by: ' +
'<a href="#" class="btn" z="-1.2" tt="Hotkey: shift-A">&ndash;</a> ' +
'<a href="#" class="btn" z="1.2" tt="Hotkey: shift-D">+</a> &nbsp; sort by: ' +
'<a href="#" s="href">name</a>, ' +
'<a href="#" s="sz">size</a>, ' +
'<a href="#" s="ts">date</a>, ' +
@@ -1477,8 +1495,11 @@ var thegrid = (function () {
r.setdirty();
};
ebi('griden').onclick = function (e) {
ebi('griden').onclick = ebi('wtgrid').onclick = function (e) {
ev(e);
if (!this.closest)
return;
r.en = !r.en;
bcfg_set('griden', r.en);
if (r.en) {
@@ -1545,7 +1566,7 @@ var thegrid = (function () {
setsz();
function gclick(e) {
if (e && (e.ctrlKey || e.metaKey))
if (ctrl(e))
return true;
var oth = ebi(this.getAttribute('ref')),
@@ -1699,6 +1720,26 @@ var thegrid = (function () {
})();
function tree_scrollto() {
var act = QS('#treeul a.hl'),
ul = act ? act.offsetParent : null;
if (!ul)
return;
var ctr = ebi('tree'),
em = parseFloat(getComputedStyle(act).fontSize),
top = act.offsetTop + ul.offsetTop,
min = top - 11 * em,
max = top - (ctr.offsetHeight - 10 * em);
if (ctr.scrollTop > min)
ctr.scrollTop = Math.floor(min);
else if (ctr.scrollTop < max)
ctr.scrollTop = Math.floor(max);
}
function tree_neigh(n) {
var links = QSA('#treeul li>a+a');
if (!links.length) {
@@ -1724,6 +1765,7 @@ function tree_neigh(n) {
if (act >= links.length)
act = 0;
treectl.dir_cb = tree_scrollto;
links[act].click();
}
@@ -1749,7 +1791,10 @@ document.onkeydown = function (e) {
if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
return;
var k = (e.code + ''), pos = -1, n;
if (QS('#bbox-overlay.visible'))
return;
var k = e.code + '', pos = -1, n;
if (e.shiftKey && k != 'KeyA' && k != 'KeyD')
return;
@@ -1925,7 +1970,7 @@ document.onkeydown = function (e) {
for (var b = 1; b < sconf[a].length; b++) {
var k = sconf[a][b][0],
chk = 'srch_' + k + 'c',
tvs = ebi('srch_' + k + 'v').value.split(/ /g);
tvs = ebi('srch_' + k + 'v').value.split(/ +/g);
if (!ebi(chk).checked)
continue;
@@ -1938,7 +1983,7 @@ document.onkeydown = function (e) {
q += ' and ';
if (k == 'adv') {
q += tv.replace(/ /g, " and ").replace(/([=!><]=?)/, " $1 ");
q += tv.replace(/ +/g, " and ").replace(/([=!><]=?)/, " $1 ");
continue;
}
@@ -2269,7 +2314,12 @@ var treectl = (function () {
var fun = treectl.dir_cb;
if (fun) {
treectl.dir_cb = null;
fun();
try {
fun();
}
catch (ex) {
console.log("dir_cb failed", ex);
}
}
}
@@ -2290,7 +2340,7 @@ var treectl = (function () {
}
function treego(e) {
if (e && (e.ctrlKey || e.metaKey))
if (ctrl(e))
return true;
ev(e);
@@ -2391,6 +2441,7 @@ var treectl = (function () {
if (this.hpush)
hist_push(this.top);
acct = res.acct;
apply_perms(res.perms);
despin('#files');
despin('#gfiles');
@@ -2507,9 +2558,13 @@ function despin(sel) {
function apply_perms(newperms) {
perms = newperms || [];
ebi('acc_info').innerHTML = '<span>' + (acct != '*' ?
'<a href="?pw=x">Logout ' + acct + '</a>' :
'<a href="?h">Login</a>') + '</span>';
var o = QSA('#ops>a[data-perm], #u2footfoot');
for (var a = 0; a < o.length; a++) {
var display = 'inline';
var display = '';
var needed = o[a].getAttribute('data-perm').split(' ');
for (var b = 0; b < needed.length; b++) {
if (!has(perms, needed[b])) {
@@ -2530,12 +2585,10 @@ function apply_perms(newperms) {
de = document.documentElement,
tds = QSA('#u2conf td');
/* good idea maybe
clmod(de, "read", have_read);
clmod(de, "write", have_write);
clmod(de, "nread", !have_read);
clmod(de, "nwrite", !have_write);
*/
for (var a = 0; a < tds.length; a++) {
tds[a].style.display =
@@ -2561,7 +2614,7 @@ function find_file_col(txt) {
for (var a = 0; a < tds.length; a++) {
var spans = tds[a].getElementsByTagName('span');
if (spans.length && spans[0].textContent == txt) {
min = tds[a].getAttribute('class').indexOf('min') !== -1;
min = (tds[a].getAttribute('class') || '').indexOf('min') !== -1;
i = a;
break;
}
@@ -2603,14 +2656,36 @@ function mk_files_header(taglist) {
var filecols = (function () {
var hidden = jread('filecols', []);
var hidden = jread('filecols', []),
tts = {
"c": "action buttons",
"dur": "duration",
"q": "quality / bitrate",
"Ac": "audio codec",
"Vc": "video codec",
"Res": "resolution",
"T": "filetype",
"aq": "audio quality / bitrate",
"vq": "video quality / bitrate",
"pixfmt": "subsampling / pixel structure",
"resw": "horizontal resolution",
"resh": "veritcal resolution",
"acs": "audio channels",
"hz": "sample rate"
};
var add_btns = function () {
var ths = QSA('#files th>span');
for (var a = 0, aa = ths.length; a < aa; a++) {
var th = ths[a].parentElement;
var th = ths[a].parentElement,
ttv = tts[ths[a].textContent];
th.innerHTML = '<div class="cfg"><a href="#">-</a></div>' + ths[a].outerHTML;
th.getElementsByTagName('a')[0].onclick = ev_row_tgl;
if (ttv) {
th.setAttribute("tt", ttv);
th.setAttribute("ttd", "u");
}
}
};
@@ -2633,7 +2708,10 @@ var filecols = (function () {
hcols = ebi('hcols');
for (var a = 0; a < hidden.length; a++) {
html.push('<a href="#" class="btn">' + esc(hidden[a]) + '</a>');
var ttv = tts[hidden[a]],
tta = ttv ? ' tt="' + ttv + '">' : '>';
html.push('<a href="#" class="btn"' + tta + esc(hidden[a]) + '</a>');
}
hcols.previousSibling.style.display = html.length ? 'block' : 'none';
hcols.innerHTML = html.join('\n');
@@ -2666,6 +2744,10 @@ var filecols = (function () {
for (var b = 0, bb = tds.length; b < bb; b++)
tds[b].setAttribute('class', cls);
}
if (window['tt']) {
tt.att(ebi('hcols'));
tt.att(QS('#files thead'));
}
};
set_style();
@@ -2856,7 +2938,7 @@ var arcfmt = (function () {
["tar", "tar", "plain gnutar file"],
["zip", "zip=utf8", "zip with utf8 filenames (maybe wonky on windows 7 and older)"],
["zip_dos", "zip", "zip with traditional cp437 filenames, for really old software"],
["zip_crc", "zip=crc", "cp437 with crc32 computed early for truly ancient software$N(takes longer to process before download can start)"]
["zip_crc", "zip=crc", "cp437 with crc32 computed early,$Nfor MS-DOS PKZIP v2.04g (october 1993)$N(takes longer to process before download can start)"]
];
for (var a = 0; a < fmts.length; a++) {
@@ -2888,7 +2970,7 @@ var arcfmt = (function () {
var ofs = href.lastIndexOf('?');
if (ofs < 0)
throw 'missing arg in url';
throw new Error('missing arg in url');
o.setAttribute("href", href.slice(0, ofs + 1) + arg);
o.textContent = fmt.split('_')[0];
@@ -3050,7 +3132,7 @@ function reload_browser(not_mp) {
var oo = QSA('#files>tbody>tr>td:nth-child(3)');
for (var a = 0, aa = oo.length; a < aa; a++) {
var sz = oo[a].textContent.replace(/ /g, ""),
var sz = oo[a].textContent.replace(/ +/g, ""),
hsz = sz.replace(/\B(?=(\d{3})+(?!\d))/g, " ");
oo[a].textContent = hsz;

View File

@@ -8,6 +8,47 @@ html, body {
font-family: sans-serif;
line-height: 1.5em;
}
#tt {
position: fixed;
max-width: 34em;
background: #222;
border: 0 solid #777;
overflow: hidden;
margin-top: 1em;
padding: 0 1.3em;
height: 0;
opacity: .1;
transition: opacity 0.14s, height 0.14s, padding 0.14s;
box-shadow: 0 .2em .5em #222;
border-radius: .4em;
z-index: 9001;
}
#tt.b {
padding: 0 2em;
border-radius: .5em;
box-shadow: 0 .2em 1em #000;
}
#tt.show {
padding: 1em 1.3em;
border-width: .4em 0;
height: auto;
opacity: 1;
}
#tt.show.b {
padding: 1.5em 2em;
border-width: .5em 0;
}
#tt code {
background: #3c3c3c;
padding: .1em .3em;
border-top: 1px solid #777;
border-radius: .3em;
font-family: monospace, monospace;
line-height: 1.7em;
}
#tt em {
color: #f6a;
}
#mtw {
display: none;
}

View File

@@ -14,9 +14,9 @@
<a id="lightswitch" href="#">go dark</a>
<a id="navtoggle" href="#">hide nav</a>
{%- if edit %}
<a id="save" href="?edit">save</a>
<a id="sbs" href="#">sbs</a>
<a id="nsbs" href="#">editor</a>
<a id="save" href="?edit" tt="Hotkey: ctrl-s">save</a>
<a id="sbs" href="#" tt="editor and preview side by side">sbs</a>
<a id="nsbs" href="#" tt="switch between editor and preview$NHotkey: ctrl-e">editor</a>
<div id="toolsbox">
<a id="tools" href="#">tools</a>
<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
@@ -26,8 +26,8 @@
<a id="help" href="#">help</a>
</div>
{%- else %}
<a href="?edit">edit (basic)</a>
<a href="?edit2">edit (fancy)</a>
<a href="?edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
<a href="?edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
<a href="?raw">view raw</a>
{%- endif %}
</div>

View File

@@ -176,7 +176,7 @@ function md_plug_err(ex, js) {
var lns = js.split('\n');
if (ln < lns.length) {
o = mknod('span');
o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
o.style.cssText = "color:#ac2;font-size:.9em;font-family:'scp',monospace,monospace;display:block";
o.textContent = lns[ln - 1];
}
}
@@ -530,3 +530,6 @@ dom_navtgl.onclick = function () {
if (sread('hidenav') == 1)
dom_navtgl.onclick();
if (window['tt'])
tt.init();

View File

@@ -924,10 +924,9 @@ function cfg_uni(e) {
(function () {
function keydown(ev) {
ev = ev || window.event;
var kc = ev.keyCode || ev.which;
var ctrl = ev.ctrlKey || ev.metaKey;
//console.log(ev.code, kc);
if (ctrl && (ev.code == "KeyS" || kc == 83)) {
var kc = ev.code || ev.keyCode || ev.which;
//console.log(ev.key, ev.code, ev.keyCode, ev.which);
if (ctrl(ev) && (ev.code == "KeyS" || kc == 83)) {
save();
return false;
}
@@ -936,23 +935,15 @@ function cfg_uni(e) {
if (d)
d.click();
}
if (document.activeElement == dom_src) {
if (ev.code == "Tab" || kc == 9) {
md_indent(ev.shiftKey);
return false;
}
if (ctrl && (ev.code == "KeyH" || kc == 72)) {
if (document.activeElement != dom_src)
return true;
if (ctrl(ev)) {
if (ev.code == "KeyH" || kc == 72) {
md_header(ev.shiftKey);
return false;
}
if (!ctrl && (ev.code == "Home" || kc == 36)) {
md_home(ev.shiftKey);
return false;
}
if (!ctrl && !ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
return md_newline();
}
if (ctrl && (ev.code == "KeyZ" || kc == 90)) {
if (ev.code == "KeyZ" || kc == 90) {
if (ev.shiftKey)
action_stack.redo();
else
@@ -960,33 +951,45 @@ function cfg_uni(e) {
return false;
}
if (ctrl && (ev.code == "KeyY" || kc == 89)) {
if (ev.code == "KeyY" || kc == 89) {
action_stack.redo();
return false;
}
if (!ctrl && !ev.shiftKey && kc == 8) {
return md_backspace();
}
if (ctrl && (ev.code == "KeyK")) {
if (ev.code == "KeyK") {
fmt_table();
return false;
}
if (ctrl && (ev.code == "KeyU")) {
if (ev.code == "KeyU") {
iter_uni();
return false;
}
if (ctrl && (ev.code == "KeyE")) {
if (ev.code == "KeyE") {
dom_nsbs.click();
//fmt_table();
return false;
}
var up = ev.code == "ArrowUp" || kc == 38;
var dn = ev.code == "ArrowDown" || kc == 40;
if (ctrl && (up || dn)) {
if (up || dn) {
md_p_jump(dn);
return false;
}
}
else {
if (ev.code == "Tab" || kc == 9) {
md_indent(ev.shiftKey);
return false;
}
if (ev.code == "Home" || kc == 36) {
md_home(ev.shiftKey);
return false;
}
if (!ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
return md_newline();
}
if (!ev.shiftKey && kc == 8) {
return md_backspace();
}
}
}
document.onkeydown = keydown;
ebi('save').onclick = save;

View File

@@ -35,7 +35,7 @@
</table>
</td></tr></table>
<div class="btns">
<a href="{{ avol[0] }}?stack">dump stack</a>
<a href="/?stack">dump stack</a>
</div>
{%- endif %}

View File

@@ -290,8 +290,7 @@ function U2pvis(act, btns) {
if (this.is_act(this.tab[a].in))
console.log("tab %d/%d = sz %s", a, aa, this.tab[a].bt);
console.log("a");
throw 42;
throw new Error('see console');
}
obj.innerHTML = fo.hp;
@@ -304,9 +303,8 @@ function U2pvis(act, btns) {
oldcat = fo.in,
bz_act = this.act == "bz";
if (oldcat == newcat) {
throw 42;
}
if (oldcat == newcat)
return;
fo.in = newcat;
this.ctr[oldcat]--;
@@ -319,7 +317,7 @@ function U2pvis(act, btns) {
this.addrow(nfile);
}
else if (this.is_act(oldcat)) {
while (this.head < Math.min(this.tab.length, this.tail) && (this.head == nfile || !this.is_act(this.tab[this.head].in)))
while (this.head < Math.min(this.tab.length, this.tail) && this.precard[this.tab[this.head].in])
this.head++;
if (!bz_act) {
@@ -327,9 +325,10 @@ function U2pvis(act, btns) {
tr.parentNode.removeChild(tr);
}
}
if (bz_act) {
else return;
if (bz_act)
this.bzw();
}
};
this.bzw = function () {
@@ -343,7 +342,8 @@ function U2pvis(act, btns) {
while (this.head - first > this.wsz) {
var obj = ebi('f' + (first++));
obj.parentNode.removeChild(obj);
if (obj)
obj.parentNode.removeChild(obj);
}
while (last - this.tail < this.wsz && last < this.tab.length - 2) {
var obj = ebi('f' + (++last));
@@ -376,6 +376,8 @@ function U2pvis(act, btns) {
this.changecard = function (card) {
this.act = card;
this.precard = has(["ok", "ng", "done"], this.act) ? {} : this.act == "bz" ? { "ok": 1, "ng": 1 } : { "ok": 1, "ng": 1, "bz": 1 };
this.postcard = has(["ok", "ng", "done"], this.act) ? { "bz": 1, "q": 1 } : this.act == "bz" ? { "q": 1 } : {};
this.head = -1;
this.tail = -1;
var html = [];
@@ -390,22 +392,23 @@ function U2pvis(act, btns) {
}
}
if (this.head == -1) {
var precard = has(["ok", "ng", "done"], this.act) ? {} : this.act == "bz" ? { "ok": 1, "ng": 1 } : { "ok": 1, "ng": 1, "bz": 1 },
postcard = has(["ok", "ng", "done"], this.act) ? { "bz": 1, "q": 1 } : this.act == "bz" ? { "q": 1 } : {};
for (var a = 0; a < this.tab.length; a++) {
var rt = this.tab[a].in;
if (precard[rt]) {
if (this.precard[rt]) {
this.head = a + 1;
this.tail = a;
}
else if (postcard[rt]) {
else if (this.postcard[rt]) {
this.head = a;
this.tail = a - 1;
break;
}
}
}
if (this.head < 0)
this.head = 0;
if (card == "bz") {
for (var a = this.head - 1; a >= this.head - this.wsz && a >= 0; a--) {
html.unshift(this.genrow(a, true).replace(/><td>/, "><td>a "));
@@ -452,6 +455,17 @@ function U2pvis(act, btns) {
that.changecard(newtab);
};
}
this.changecard(this.act);
}
function fsearch_explain(e) {
ev(e);
if (!has(perms, 'write'))
return alert('your access to this folder is Read-Only\n\n' + (acct == '*' ? 'you are currently not logged in' : 'you are currently logged in as ' + acct));
alert('you are currently in file-search mode\n\nswitch to upload-mode by clicking the green magnifying glass (next to the big yellow search button), and then refresh\n\nsorry');
}
@@ -482,8 +496,9 @@ function up2k_init(subtle) {
shame = 'your browser is impressively ancient';
// upload ui hidden by default, clicking the header shows it
var got_deps = false;
function init_deps() {
if (!subtle && !window.asmCrypto) {
if (!got_deps && !subtle && !window.asmCrypto) {
var fn = 'sha512.' + sha_js + '.js';
showmodal('<h1>loading ' + fn + '</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
import_js('/.cpr/deps/' + fn, unmodal);
@@ -494,6 +509,7 @@ function up2k_init(subtle) {
ebi('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance <span style="color:#' +
(sha_js == 'ac' ? 'c84">(expecting 20' : '8a5">(but dont worry too much, expect 100') + ' MiB/s)</span>';
}
got_deps = true;
}
// show uploader if the user only has write-access
@@ -548,17 +564,21 @@ function up2k_init(subtle) {
ask_up = bcfg_get('ask_up', true),
flag_en = bcfg_get('flag_en', false),
fsearch = bcfg_get('fsearch', false),
turbo = bcfg_get('u2turbo', false),
datechk = bcfg_get('u2tdate', true),
fdom_ctr = 0,
min_filebuf = 0;
var st = {
"files": [],
"todo": {
"head": [],
"hash": [],
"handshake": [],
"upload": []
},
"busy": {
"head": [],
"hash": [],
"handshake": [],
"upload": []
@@ -569,6 +589,15 @@ function up2k_init(subtle) {
}
};
function push_t(arr, t) {
var sort = arr.length && arr[arr.length - 1].n > t.n;
arr.push(t);
if (sort)
arr.sort(function (a, b) {
return a.n < b.n ? -1 : 1;
});
}
var pvis = new U2pvis("bz", '#u2cards');
var bobslice = null;
@@ -796,7 +825,10 @@ function up2k_init(subtle) {
], fobj.size, draw_each);
st.files.push(entry);
st.todo.hash.push(entry);
if (turbo)
push_t(st.todo.head, entry);
else
push_t(st.todo.hash, entry);
}
if (!draw_each) {
pvis.drawcard("q");
@@ -839,20 +871,27 @@ function up2k_init(subtle) {
if (!st.todo.handshake.length)
return true;
var cd = st.todo.handshake[0].cooldown;
var t = st.todo.handshake[0],
cd = t.cooldown;
if (cd && cd - Date.now() > 0)
return false;
// keepalive or verify
if (st.todo.handshake[0].keepalive ||
st.todo.handshake[0].t_uploaded)
if (t.keepalive ||
t.t_uploaded)
return true;
if (parallel_uploads <
st.busy.handshake.length)
return false;
if ((multitask ? parallel_uploads : 0) <
if (st.busy.handshake.length)
for (var n = t.n - 1; n >= t.n - parallel_uploads && n >= 0; n--)
if (st.files[n].t_uploading)
return false;
if ((multitask ? 1 : 0) <
st.todo.upload.length +
st.busy.upload.length)
return false;
@@ -891,9 +930,11 @@ function up2k_init(subtle) {
while (window['vis_exh']) {
var now = Date.now(),
is_busy = 0 !=
st.todo.head.length +
st.todo.hash.length +
st.todo.handshake.length +
st.todo.upload.length +
st.busy.head.length +
st.busy.hash.length +
st.busy.handshake.length +
st.busy.upload.length;
@@ -943,6 +984,12 @@ function up2k_init(subtle) {
}
}
if (st.todo.head.length &&
st.busy.head.length < parallel_uploads) {
exec_head();
mou_ikkai = true;
}
if (handshakes_permitted() &&
st.todo.handshake.length) {
exec_handshake();
@@ -1160,6 +1207,53 @@ function up2k_init(subtle) {
segm_next();
}
/////
////
/// head
//
function exec_head() {
var t = st.todo.head.shift();
st.busy.head.push(t);
var xhr = new XMLHttpRequest();
xhr.onerror = function () {
console.log('head onerror, retrying', t);
apop(st.busy.head, t);
st.todo.head.unshift(t);
tasker();
};
function orz(e) {
var ok = false;
if (xhr.status == 200) {
var srv_sz = xhr.getResponseHeader('Content-Length'),
srv_ts = xhr.getResponseHeader('Last-Modified');
ok = t.size == srv_sz;
if (ok && datechk) {
srv_ts = new Date(srv_ts) / 1000;
ok = Math.abs(srv_ts - t.lmod) < 2;
}
}
apop(st.busy.head, t);
if (!ok)
return push_t(st.todo.hash, t);
t.done = true;
st.bytes.hashed += t.size;
st.bytes.uploaded += t.size;
pvis.seth(t.n, 1, 'YOLO');
pvis.seth(t.n, 2, "turbo'd");
pvis.move(t.n, 'ok');
};
xhr.onload = function (e) {
try { orz(e); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
};
xhr.open('HEAD', t.purl + t.name, true);
xhr.send();
}
/////
////
/// handshake
@@ -1207,8 +1301,8 @@ function up2k_init(subtle) {
smsg = '';
if (!response || !response.hits || !response.hits.length) {
msg = 'not found on server';
smsg = '404';
msg = 'not found on server <a href="#" onclick="fsearch_explain()" class="fsearch_explain">(explain)</a>';
}
else {
smsg = 'found';
@@ -1268,14 +1362,24 @@ function up2k_init(subtle) {
msg = '&#x1f3b7;&#x1f41b;';
if (t.postlist.length) {
var arr = st.todo.upload,
sort = arr.length && arr[arr.length - 1].nfile > t.n;
for (var a = 0; a < t.postlist.length; a++)
st.todo.upload.push({
arr.push({
'nfile': t.n,
'npart': t.postlist[a]
});
msg = 'uploading';
done = false;
if (sort)
arr.sort(function (a, b) {
return a.nfile < b.nfile ? -1 :
/* */ a.nfile > b.nfile ? 1 :
a.npart < b.npart ? -1 : 1;
});
}
pvis.seth(t.n, 1, msg);
apop(st.busy.handshake, t);
@@ -1469,7 +1573,7 @@ function up2k_init(subtle) {
for (var a = o.length - 1; a >= 0; a--) {
o[a].parentNode.getElementsByTagName('input')[0].setAttribute('tt', o[a].getAttribute('tt'));
}
tt.init();
tt.att(QS('#u2conf'));
function bumpthread2(e) {
if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
@@ -1527,6 +1631,35 @@ function up2k_init(subtle) {
set_fsearch(!fsearch);
}
function tgl_turbo() {
turbo = !turbo;
bcfg_set('u2turbo', turbo);
draw_turbo();
}
function tgl_datechk() {
datechk = !datechk;
bcfg_set('u2tdate', datechk);
}
function draw_turbo() {
var msgu = '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
msgs = '<p class="warn">WARNING: turbo enabled, <span>&nbsp;search may give false-positives; see turbo-button tooltip</span></p>',
msg = fsearch ? msgs : msgu,
omsg = fsearch ? msgu : msgs,
html = ebi('u2foot').innerHTML,
ohtml = html;
if (turbo && html.indexOf(msg) === -1)
html = html.replace(omsg, '') + msg;
else if (!turbo)
html = html.replace(msgu, '').replace(msgs, '');
if (html !== ohtml)
ebi('u2foot').innerHTML = html;
}
draw_turbo();
function set_fsearch(new_state) {
var fixed = false;
@@ -1564,6 +1697,7 @@ function up2k_init(subtle) {
}
catch (ex) { }
draw_turbo();
onresize();
}
@@ -1608,6 +1742,8 @@ function up2k_init(subtle) {
ebi('multitask').addEventListener('click', tgl_multitask, false);
ebi('ask_up').addEventListener('click', tgl_ask_up, false);
ebi('flag_en').addEventListener('click', tgl_flag_en, false);
ebi('u2turbo').addEventListener('click', tgl_turbo, false);
ebi('u2tdate').addEventListener('click', tgl_datechk, false);
var o = ebi('fsearch');
if (o)
o.addEventListener('click', tgl_fsearch, false);

View File

@@ -215,9 +215,31 @@
color: #fff;
font-style: italic;
}
#u2foot .warn {
font-size: 1.3em;
padding: .5em .8em;
margin: 1em -.6em;
color: #f74;
background: #322;
border: 1px solid #633;
border-width: .1em 0;
text-align: center;
}
#u2foot .warn span {
color: #f86;
}
html.light #u2foot .warn {
color: #b00;
background: #fca;
border-color: #f70;
}
html.light #u2foot .warn span {
color: #930;
}
#u2foot span {
color: #999;
font-size: .9em;
font-weight: normal;
}
#u2footfoot {
margin-bottom: -1em;
@@ -235,6 +257,11 @@
float: right;
margin-bottom: -.3em;
}
.fsearch_explain {
padding-left: .7em;
font-size: 1.1em;
line-height: 0;
}

View File

@@ -7,7 +7,8 @@ if (!window['console'])
var is_touch = 'ontouchstart' in window,
ANDROID = /(android)/i.test(navigator.userAgent);
IPHONE = /iPhone|iPad|iPod/i.test(navigator.userAgent),
ANDROID = /android/i.test(navigator.userAgent);
// error handler for mobile devices
@@ -27,20 +28,24 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
window.onerror = undefined;
window['vis_exh'] = null;
var html = ['<h1>you hit a bug!</h1><p>please send me a screenshot arigathanks gozaimuch: <code>ed/irc.rizon.net</code> or <code>ed#2644</code><br />&nbsp; (and if you can, press F12 and include the "Console" tab in the screenshot too)</p><p>',
esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
var html = ['<h1>you hit a bug!</h1><p style="font-size:1.3em;margin:0">try to <a href="#" onclick="localStorage.clear();location.reload();" style="text-decoration:underline;color:#fc0">reset copyparty settings</a> if you are stuck here</p><p>please send me a screenshot arigathanks gozaimuch: <code>ed/irc.rizon.net</code> or <code>ed#2644</code><br />&nbsp; (and if you can, press F12 and include the "Console" tab in the screenshot too)</p><p>',
esc(url + ' @' + lineNo + ':' + columnNo), '<br />' + esc(String(msg)) + '</p>'];
if (error) {
var find = ['desc', 'stack', 'trace'];
for (var a = 0; a < find.length; a++)
if (String(error[find[a]]) !== 'undefined')
html.push('<h2>' + find[a] + '</h2>' +
esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
try {
if (error) {
var find = ['desc', 'stack', 'trace'];
for (var a = 0; a < find.length; a++)
if (String(error[find[a]]) !== 'undefined')
html.push('<h3>' + find[a] + '</h3>' +
esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
}
html.push('<h3>localStore</h3>' + esc(JSON.stringify(localStorage)));
}
catch (e) { }
document.body.innerHTML = html.join('\n');
var s = mknod('style');
s.innerHTML = 'body{background:#333;color:#ddd;font-family:sans-serif;font-size:0.8em;padding:0 1em 1em 1em} code{color:#bf7;background:#222;padding:.1em;margin:.2em;font-size:1.1em;font-family:monospace,monospace} *{line-height:1.5em}';
s.innerHTML = 'body{background:#333;color:#ddd;font-family:sans-serif;font-size:0.8em;padding:0 1em 1em 1em} h1{margin:.5em 1em 0 0;padding:0} h3{border-top:1px solid #999;margin:1em 0 0 0} code{color:#bf7;background:#222;padding:.1em;margin:.2em;font-size:1.1em;font-family:monospace,monospace} *{line-height:1.5em}';
document.head.appendChild(s);
throw 'fatal_err';
@@ -53,6 +58,11 @@ var ebi = document.getElementById.bind(document),
mknod = document.createElement.bind(document);
function ctrl(e) {
return e && (e.ctrlKey || e.metaKey);
}
function ev(e) {
e = e || window.event;
if (!e)
@@ -497,13 +507,20 @@ function hist_replace(url) {
var tt = (function () {
var r = {
"tt": mknod("div"),
"en": true
"en": true,
"el": null,
"skip": false
};
r.tt.setAttribute('id', 'tt');
document.body.appendChild(r.tt);
function show() {
r.show = function () {
if (r.skip) {
r.skip = false;
return;
}
var cfg = sread('tooltips');
if (cfg !== null && cfg != '1')
return;
@@ -512,21 +529,62 @@ var tt = (function () {
if (!msg)
return;
r.el = this;
var pos = this.getBoundingClientRect(),
dir = this.getAttribute('ttd') || '',
left = pos.left < window.innerWidth / 2,
top = pos.top < window.innerHeight / 2;
top = pos.top < window.innerHeight / 2,
big = this.className.indexOf(' ttb') !== -1;
if (dir.indexOf('u') + 1) top = false;
if (dir.indexOf('d') + 1) top = true;
if (dir.indexOf('l') + 1) left = false;
if (dir.indexOf('r') + 1) left = true;
clmod(r.tt, 'b', big);
r.tt.style.top = top ? pos.bottom + 'px' : 'auto';
r.tt.style.bottom = top ? 'auto' : (window.innerHeight - pos.top) + 'px';
r.tt.style.left = left ? pos.left + 'px' : 'auto';
r.tt.style.right = left ? 'auto' : (window.innerWidth - pos.right) + 'px';
r.tt.innerHTML = msg.replace(/\$N/g, "<br />");
r.el.addEventListener('mouseleave', r.hide);
clmod(r.tt, 'show', 1);
};
r.hide = function (e) {
ev(e);
clmod(r.tt, 'show');
if (r.el)
r.el.removeEventListener('mouseleave', r.hide);
};
if (is_touch && IPHONE) {
var f1 = r.show,
f2 = r.hide;
r.show = function () {
setTimeout(f1.bind(this), 301);
};
r.hide = function () {
setTimeout(f2.bind(this), 301);
};
}
function hide() {
clmod(r.tt, 'show');
r.tt.onclick = r.hide;
r.att = function (ctr) {
var _show = r.en ? r.show : null,
_hide = r.en ? r.hide : null,
o = ctr.querySelectorAll('*[tt]');
for (var a = o.length - 1; a >= 0; a--) {
o[a].onfocus = _show;
o[a].onblur = _hide;
o[a].onmouseenter = _show;
o[a].onmouseleave = _hide;
}
r.hide();
}
r.init = function () {
@@ -540,18 +598,7 @@ var tt = (function () {
};
r.en = bcfg_get('tooltips', true)
}
var _show = r.en ? show : null,
_hide = r.en ? hide : null;
var o = QSA('*[tt]');
for (var a = o.length - 1; a >= 0; a--) {
o[a].onfocus = _show;
o[a].onblur = _hide;
o[a].onmouseenter = _show;
o[a].onmouseleave = _hide;
}
hide();
r.att(document);
};
return r;

View File

@@ -166,7 +166,10 @@ dbg.asyncStore.pendingBreakpoints = {}
about:config >> devtools.debugger.prefs-schema-version = -1
# determine server version
git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done
git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > ../revs && cat ../{util,browser,up2k}.js >../vr && cat ../revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser,up2k}.js >../vg 2>/dev/null && diff -wNarU0 ../{vg,vr} | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done
# download all sfx versions
curl https://api.github.com/repos/9001/copyparty/releases?per_page=100 | jq -r '.[] | .tag_name + " " + .name' | while read v t; do fn="copyparty $v $t.py"; [ -e $fn ] || curl https://github.com/9001/copyparty/releases/download/$v/copyparty-sfx.py -Lo "$fn"; done
##

View File

@@ -20,6 +20,11 @@ echo
#
# `no-cm` saves ~90k by removing easymde/codemirror
# (the fancy markdown editor)
#
# `no-fnt` saves ~9k by removing the source-code-pro font
# (mainly used my the markdown viewer/editor)
#
# `no-dd` saves ~2k by removing the mouse cursor
# port install gnutar findutils gsed coreutils
@@ -57,14 +62,18 @@ use_gz=
do_sh=1
do_py=1
while [ ! -z "$1" ]; do
[ "$1" = clean ] && clean=1 && shift && continue
[ "$1" = re ] && repack=1 && shift && continue
[ "$1" = gz ] && use_gz=1 && shift && continue
[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
[ "$1" = no-cm ] && no_cm=1 && shift && continue
[ "$1" = no-sh ] && do_sh= && shift && continue
[ "$1" = no-py ] && do_py= && shift && continue
break
case $1 in
clean) clean=1 ; ;;
re) repack=1 ; ;;
gz) use_gz=1 ; ;;
no-ogv) no_ogv=1 ; ;;
no-fnt) no_fnt=1 ; ;;
no-dd) no_dd=1 ; ;;
no-cm) no_cm=1 ; ;;
no-sh) do_sh= ; ;;
no-py) do_py= ; ;;
esac
shift
done
tmv() {
@@ -190,6 +199,18 @@ done
sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
}
[ $no_fnt ] && {
rm -f copyparty/web/deps/scp.woff2
f=copyparty/web/md.css
sed -r '/scp\.woff2/d' <$f >t && tmv "$f"
}
[ $no_dd ] && {
rm -rf copyparty/web/dd
f=copyparty/web/browser.css
sed -r 's/(cursor: )url\([^)]+\), (pointer)/\1\2/; /[0-9]+% \{cursor:/d; /animation: cursor/d' <$f >t && tmv "$f"
}
[ $repack ] ||
find | grep -E '\.py$' |
grep -vE '__version__' |

View File

@@ -6,8 +6,8 @@ import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform,
import subprocess as sp
"""
pls don't edit this file with a text editor,
it breaks the compressed stuff at the end
to edit this file, use HxD or "vim -b"
(there is compressed stuff at the end)
run me with any version of python, i will unpack and run copyparty
@@ -380,7 +380,7 @@ def run(tmp, j2):
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except Exception as ex:
if not WINDOWS:
msg("\033[31mflock:", repr(ex))
msg("\033[31mflock:{!r}\033[0m".format(ex))
t = threading.Thread(target=utime, args=(tmp,))
t.daemon = True

View File

@@ -23,13 +23,14 @@ def hdr(query):
class Cfg(Namespace):
def __init__(self, a=[], v=[], c=None):
def __init__(self, a=None, v=None, c=None):
super(Cfg, self).__init__(
a=a,
v=v,
a=a or [],
v=v or [],
c=c,
rproxy=0,
ed=False,
nw=False,
no_zip=False,
no_scandir=False,
no_sendfile=True,

View File

@@ -16,8 +16,8 @@ from copyparty import util
class Cfg(Namespace):
def __init__(self, a=[], v=[], c=None):
ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
def __init__(self, a=None, v=None, c=None):
ex = {k: False for k in "nw e2d e2ds e2dsa e2t e2ts e2tsr".split()}
ex2 = {
"mtp": [],
"mte": "a",
@@ -27,7 +27,7 @@ class Cfg(Namespace):
"rproxy": 0,
}
ex.update(ex2)
super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
super(Cfg, self).__init__(a=a or [], v=v or [], c=c, **ex)
class TestVFS(unittest.TestCase):

View File

@@ -66,6 +66,14 @@ def get_ramdisk():
for _ in range(10):
try:
_, _ = chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
with open("/Volumes/cptd/.metadata_never_index", "w") as f:
f.write("orz")
try:
shutil.rmtree("/Volumes/cptd/.fseventsd")
except:
pass
return subdir("/Volumes/cptd")
except Exception as ex:
print(repr(ex))
@@ -119,14 +127,13 @@ class VHttpConn(object):
self.addr = ("127.0.0.1", "42069")
self.args = args
self.asrv = asrv
self.is_mp = False
self.nid = None
self.log_func = log
self.log_src = "a"
self.lf_url = None
self.hsrv = VHttpSrv()
self.nreq = 0
self.nbyte = 0
self.workload = 0
self.ico = None
self.thumbcli = None
self.t0 = time.time()