Compare commits

...

25 Commits

Author SHA1 Message Date
ed
9f578bfec6 v0.7.5 2021-02-12 07:06:38 +00:00
ed
1f170d7d28 up2k scanner messages less useless 2021-02-12 07:04:35 +00:00
ed
5ae14cf9be up2k scanner more better 2021-02-12 01:07:55 +00:00
ed
aaf9d53be9 more ssl options 2021-02-12 00:31:28 +00:00
ed
75c73f7ba7 add --http-only (might as well) 2021-02-11 22:54:40 +00:00
ed
b6dba8beee imagine going plaintext in the middle of a tls reply 2021-02-11 22:50:59 +00:00
ed
94521cdc1a add --https-only 2021-02-11 22:48:10 +00:00
ed
3365b1c355 add --ssl-ver (ssl/tls versions to allow) 2021-02-11 21:24:17 +00:00
ed
6c957c4923 v0.7.4 2021-02-04 01:01:42 +01:00
ed
833997f04c shrink sfx.py from 515k to 472k 2021-02-04 01:01:11 +01:00
ed
68d51e4037 rem 2021-02-04 01:00:41 +01:00
ed
ce274d2011 handle url-encoded posts 2021-02-03 23:18:11 +01:00
ed
280778ed43 catch macos socket errors 2021-02-03 22:32:16 +01:00
ed
0f558ecbbf upgrade bundled jinja2 2021-02-03 22:32:01 +01:00
ed
58f9e05d93 v0.7.3 2021-02-03 00:50:51 +01:00
ed
1ec981aea7 bind multiple ip/ports 2021-02-03 00:49:51 +01:00
ed
2a90286a7c dim the socket debug msgs 2021-02-03 00:25:13 +01:00
ed
12d25d09b2 limit gz/br unpacker to embedded resources 2021-02-03 00:19:14 +01:00
ed
a039fae1a4 remove extra anon-rw warning 2021-02-03 00:17:12 +01:00
ed
322b9abadc v0.7.2 2021-01-29 00:52:41 +01:00
ed
0aaf954cea up2k: increase purge timeout 2021-01-29 00:52:22 +01:00
ed
c2d22aa3d1 up2k: make confirmation optional 2021-01-29 00:49:35 +01:00
ed
6934c75bba nice 2021-01-29 00:43:57 +01:00
ed
c58cf78f86 yabe 2021-01-24 16:14:01 +01:00
ed
7f0de790ab more macports compat 2021-01-23 21:19:29 +01:00
27 changed files with 520 additions and 253 deletions

View File

@@ -56,5 +56,9 @@
// things you may wanna edit: // things you may wanna edit:
// //
"python.pythonPath": "/usr/bin/python3", "python.pythonPath": "/usr/bin/python3",
"python.formatting.blackArgs": [
"-t",
"py27"
],
//"python.linting.enabled": true, //"python.linting.enabled": true,
} }

10
.vscode/tasks.json vendored Normal file
View File

@@ -0,0 +1,10 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "pre",
"command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
"type": "shell"
}
]
}

View File

@@ -8,7 +8,7 @@
turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser turn your phone or raspi into a portable file server with resumable uploads/downloads using IE6 or any other browser
* server runs on anything with `py2.7` or `py3.2+` * server runs on anything with `py2.7` or `py3.3+`
* *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+` * *resumable* uploads need `firefox 12+` / `chrome 6+` / `safari 6+` / `IE 10+`
* code standard: `black` * code standard: `black`
@@ -68,9 +68,8 @@ summary: it works! you can use it! (but technically not even close to beta)
# dependencies # dependencies
* `jinja2` * `jinja2`
* pulls in `markupsafe` as of v2.7; use jinja 2.6 on py3.2
optional, enables thumbnails: optional, will eventually enable thumbnails:
* `Pillow` (requires py2.7 or py3.5+) * `Pillow` (requires py2.7 or py3.5+)

View File

@@ -1067,7 +1067,7 @@ def main():
dbg = null_log dbg = null_log
if WINDOWS: if WINDOWS:
os.system("") os.system("rem")
for ch in '<>:"\\|?*': for ch in '<>:"\\|?*':
# microsoft maps illegal characters to f0xx # microsoft maps illegal characters to f0xx

View File

@@ -980,7 +980,7 @@ def main():
dbg = null_log dbg = null_log
if WINDOWS: if WINDOWS:
os.system("") os.system("rem")
for ch in '<>:"\\|?*': for ch in '<>:"\\|?*':
# microsoft maps illegal characters to f0xx # microsoft maps illegal characters to f0xx

View File

@@ -8,7 +8,9 @@ __copyright__ = 2019
__license__ = "MIT" __license__ = "MIT"
__url__ = "https://github.com/9001/copyparty/" __url__ = "https://github.com/9001/copyparty/"
import re
import os import os
import sys
import time import time
import shutil import shutil
import filecmp import filecmp
@@ -19,7 +21,13 @@ from textwrap import dedent
from .__init__ import E, WINDOWS, VT100 from .__init__ import E, WINDOWS, VT100
from .__version__ import S_VERSION, S_BUILD_DT, CODENAME from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
from .svchub import SvcHub from .svchub import SvcHub
from .util import py_desc from .util import py_desc, align_tab
HAVE_SSL = True
try:
import ssl
except:
HAVE_SSL = False
class RiceFormatter(argparse.HelpFormatter): class RiceFormatter(argparse.HelpFormatter):
@@ -85,10 +93,78 @@ def ensure_cert():
# printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
def configure_ssl_ver(al):
def terse_sslver(txt):
txt = txt.lower()
for c in ["_", "v", "."]:
txt = txt.replace(c, "")
return txt.replace("tls10", "tls1")
# oh man i love openssl
# check this out
# hold my beer
ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
sslver = terse_sslver(al.ssl_ver).split(",")
flags = [k for k in ssl.__dict__ if ptn.match(k)]
# SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
if "help" in sslver:
avail = [terse_sslver(x[6:]) for x in flags]
avail = " ".join(sorted(avail) + ["all"])
print("\navailable ssl/tls versions:\n " + avail)
sys.exit(0)
al.ssl_flags_en = 0
al.ssl_flags_de = 0
for flag in sorted(flags):
ver = terse_sslver(flag[6:])
num = getattr(ssl, flag)
if ver in sslver:
al.ssl_flags_en |= num
else:
al.ssl_flags_de |= num
if sslver == ["all"]:
x = al.ssl_flags_en
al.ssl_flags_en = al.ssl_flags_de
al.ssl_flags_de = x
for k in ["ssl_flags_en", "ssl_flags_de"]:
num = getattr(al, k)
print("{}: {:8x} ({})".format(k, num, num))
# think i need that beer now
def configure_ssl_ciphers(al):
ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
if al.ssl_ver:
ctx.options &= ~al.ssl_flags_en
ctx.options |= al.ssl_flags_de
is_help = al.ciphers == "help"
if al.ciphers:
try:
ctx.set_ciphers(al.ciphers)
except:
if not is_help:
print("\n\033[1;31mfailed to set ciphers\033[0m\n")
if not hasattr(ctx, "get_ciphers"):
print("cannot read cipher list: openssl or python too old")
else:
ciphers = [x["description"] for x in ctx.get_ciphers()]
print("\n ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
if is_help:
sys.exit(0)
def main(): def main():
time.strptime("19970815", "%Y%m%d") # python#7980 time.strptime("19970815", "%Y%m%d") # python#7980
if WINDOWS: if WINDOWS:
os.system("") # enables colors os.system("rem") # enables colors
desc = py_desc().replace("[", "\033[1;30m[") desc = py_desc().replace("[", "\033[1;30m[")
@@ -96,7 +172,8 @@ def main():
print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc)) print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
ensure_locale() ensure_locale()
ensure_cert() if HAVE_SSL:
ensure_cert()
ap = argparse.ArgumentParser( ap = argparse.ArgumentParser(
formatter_class=RiceFormatter, formatter_class=RiceFormatter,
@@ -127,13 +204,23 @@ def main():
consider the config file for more flexible account/volume management, consider the config file for more flexible account/volume management,
including dynamic reload at runtime (and being more readable w) including dynamic reload at runtime (and being more readable w)
values for --urlform:
"stash" dumps the data to file and returns length + checksum
"save,get" dumps to file and returns the page like a GET
"print,get" prints the data in the log and returns GET
(leave out the ",get" to return an error instead)
see "--ciphers help" for available ssl/tls ciphers,
see "--ssl-ver help" for available ssl/tls versions,
default is what python considers safe, usually >= TLS1
""" """
), ),
) )
# fmt: off # fmt: off
ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file") ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind") ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
ap.add_argument("-p", metavar="PORT", type=int, default=3923, help="port to bind") ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients") ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores") ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account") ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
@@ -148,9 +235,37 @@ def main():
ap.add_argument("-nih", action="store_true", help="no info hostname") ap.add_argument("-nih", action="store_true", help="no info hostname")
ap.add_argument("-nid", action="store_true", help="no info disk-usage") ap.add_argument("-nid", action="store_true", help="no info disk-usage")
ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile") ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
ap2 = ap.add_argument_group('SSL/TLS options')
ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
ap2.add_argument("--ssl-ver", type=str, help="ssl/tls versions to allow")
ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
al = ap.parse_args() al = ap.parse_args()
# fmt: on # fmt: on
al.i = al.i.split(",")
try:
if "-" in al.p:
lo, hi = [int(x) for x in al.p.split("-")]
al.p = list(range(lo, hi + 1))
else:
al.p = [int(x) for x in al.p.split(",")]
except:
raise Exception("invalid value for -p")
if HAVE_SSL:
if al.ssl_ver:
configure_ssl_ver(al)
if al.ciphers:
configure_ssl_ciphers(al)
else:
print("\033[33m ssl module does not exist; cannot enable https\033[0m\n")
SvcHub(al).run() SvcHub(al).run()

View File

@@ -1,8 +1,8 @@
# coding: utf-8 # coding: utf-8
VERSION = (0, 7, 1) VERSION = (0, 7, 5)
CODENAME = "keeping track" CODENAME = "keeping track"
BUILD_DT = (2021, 1, 23) BUILD_DT = (2021, 2, 12)
S_VERSION = ".".join(map(str, VERSION)) S_VERSION = ".".join(map(str, VERSION))
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT) S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

View File

@@ -130,11 +130,10 @@ class VFS(object):
class AuthSrv(object): class AuthSrv(object):
"""verifies users against given paths""" """verifies users against given paths"""
def __init__(self, args, log_func): def __init__(self, args, log_func, warn_anonwrite=True):
self.log_func = log_func
self.args = args self.args = args
self.log_func = log_func
self.warn_anonwrite = True self.warn_anonwrite = warn_anonwrite
if WINDOWS: if WINDOWS:
self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$") self.re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")

View File

@@ -73,7 +73,7 @@ class MpWorker(object):
if PY2: if PY2:
sck = pickle.loads(sck) # nosec sck = pickle.loads(sck) # nosec
self.log("%s %s" % addr, "-" * 4 + "C-qpop") self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
self.httpsrv.accept(sck, addr) self.httpsrv.accept(sck, addr)
with self.mutex: with self.mutex:

View File

@@ -28,7 +28,7 @@ class BrokerThr(object):
def put(self, want_retval, dest, *args): def put(self, want_retval, dest, *args):
if dest == "httpconn": if dest == "httpconn":
sck, addr = args sck, addr = args
self.log("%s %s" % addr, "-" * 4 + "C-qpop") self.log("%s %s" % addr, "\033[1;30m|%sC-qpop\033[0m" % ("-" * 4,))
self.httpsrv.accept(sck, addr) self.httpsrv.accept(sck, addr)
else: else:

View File

@@ -134,6 +134,16 @@ class HttpCli(object):
uparam["raw"] = True uparam["raw"] = True
uparam["dots"] = True uparam["dots"] = True
if hasattr(self.s, "cipher"):
self.ssl_suf = "".join(
[
" \033[3{}m{}".format(c, s)
for c, s in zip([6, 3, 6], self.s.cipher())
]
)
else:
self.ssl_suf = ""
try: try:
if self.mode in ["GET", "HEAD"]: if self.mode in ["GET", "HEAD"]:
return self.handle_get() and self.keepalive return self.handle_get() and self.keepalive
@@ -211,7 +221,7 @@ class HttpCli(object):
logmsg += " [\033[36m" + rval + "\033[0m]" logmsg += " [\033[36m" + rval + "\033[0m]"
self.log(logmsg) self.log(logmsg + self.ssl_suf)
# "embedded" resources # "embedded" resources
if self.vpath.startswith(".cpr"): if self.vpath.startswith(".cpr"):
@@ -245,7 +255,7 @@ class HttpCli(object):
return self.tx_browser() return self.tx_browser()
def handle_options(self): def handle_options(self):
self.log("OPTIONS " + self.req) self.log("OPTIONS " + self.req + self.ssl_suf)
self.send_headers( self.send_headers(
None, None,
204, 204,
@@ -258,7 +268,7 @@ class HttpCli(object):
return True return True
def handle_put(self): def handle_put(self):
self.log("PUT " + self.req) self.log("PUT " + self.req + self.ssl_suf)
if self.headers.get("expect", "").lower() == "100-continue": if self.headers.get("expect", "").lower() == "100-continue":
try: try:
@@ -269,7 +279,7 @@ class HttpCli(object):
return self.handle_stash() return self.handle_stash()
def handle_post(self): def handle_post(self):
self.log("POST " + self.req) self.log("POST " + self.req + self.ssl_suf)
if self.headers.get("expect", "").lower() == "100-continue": if self.headers.get("expect", "").lower() == "100-continue":
try: try:
@@ -294,16 +304,37 @@ class HttpCli(object):
if "application/octet-stream" in ctype: if "application/octet-stream" in ctype:
return self.handle_post_binary() return self.handle_post_binary()
raise Pebkac(405, "don't know how to handle {} POST".format(ctype)) if "application/x-www-form-urlencoded" in ctype:
opt = self.args.urlform
if "stash" in opt:
return self.handle_stash()
def handle_stash(self): if "save" in opt:
post_sz, _, _, path = self.dump_to_file()
self.log("urlform: {} bytes, {}".format(post_sz, path))
elif "print" in opt:
reader, _ = self.get_body_reader()
for buf in reader:
buf = buf.decode("utf-8", "replace")
self.log("urlform:\n {}\n".format(buf))
if "get" in opt:
return self.handle_get()
raise Pebkac(405, "POST({}) is disabled".format(ctype))
raise Pebkac(405, "don't know how to handle POST({})".format(ctype))
def get_body_reader(self):
remains = int(self.headers.get("content-length", None)) remains = int(self.headers.get("content-length", None))
if remains is None: if remains is None:
reader = read_socket_unbounded(self.sr)
self.keepalive = False self.keepalive = False
return read_socket_unbounded(self.sr), remains
else: else:
reader = read_socket(self.sr, remains) return read_socket(self.sr, remains), remains
def dump_to_file(self):
reader, remains = self.get_body_reader()
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
fdir = os.path.join(vfs.realpath, rem) fdir = os.path.join(vfs.realpath, rem)
@@ -314,6 +345,10 @@ class HttpCli(object):
with open(path, "wb", 512 * 1024) as f: with open(path, "wb", 512 * 1024) as f:
post_sz, _, sha_b64 = hashcopy(self.conn, reader, f) post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
return post_sz, sha_b64, remains, path
def handle_stash(self):
post_sz, sha_b64, remains, path = self.dump_to_file()
spd = self._spd(post_sz) spd = self._spd(post_sz)
self.log("{} wrote {}/{} bytes to {}".format(spd, post_sz, remains, path)) self.log("{} wrote {}/{} bytes to {}".format(spd, post_sz, remains, path))
self.reply("{}\n{}\n".format(post_sz, sha_b64).encode("utf-8")) self.reply("{}\n{}\n".format(post_sz, sha_b64).encode("utf-8"))
@@ -517,10 +552,9 @@ class HttpCli(object):
raise Pebkac(500, "mkdir failed, check the logs") raise Pebkac(500, "mkdir failed, check the logs")
vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/") vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
esc_paths = [quotep(vpath), html_escape(vpath)]
html = self.conn.tpl_msg.render( html = self.conn.tpl_msg.render(
h2='<a href="/{}">go to /{}</a>'.format( h2='<a href="/{}">go to /{}</a>'.format(*esc_paths),
quotep(vpath), html_escape(vpath)
),
pre="aight", pre="aight",
click=True, click=True,
) )
@@ -797,6 +831,8 @@ class HttpCli(object):
editions[ext or "plain"] = [fs_path, st.st_size] editions[ext or "plain"] = [fs_path, st.st_size]
except: except:
pass pass
if not self.vpath.startswith(".cpr/"):
break
if not editions: if not editions:
raise Pebkac(404) raise Pebkac(404)
@@ -901,8 +937,11 @@ class HttpCli(object):
open_func = open open_func = open
# 512 kB is optimal for huge files, use 64k # 512 kB is optimal for huge files, use 64k
open_args = [fsenc(fs_path), "rb", 64 * 1024] open_args = [fsenc(fs_path), "rb", 64 * 1024]
if hasattr(os, "sendfile"): use_sendfile = (
use_sendfile = not self.args.no_sendfile not self.ssl_suf
and not self.args.no_sendfile
and hasattr(os, "sendfile")
)
# #
# send reply # send reply

View File

@@ -3,10 +3,15 @@ from __future__ import print_function, unicode_literals
import os import os
import sys import sys
import ssl
import time import time
import socket import socket
HAVE_SSL = True
try:
import ssl
except:
HAVE_SSL = False
try: try:
import jinja2 import jinja2
except ImportError: except ImportError:
@@ -75,9 +80,8 @@ class HttpConn(object):
def log(self, msg): def log(self, msg):
self.log_func(self.log_src, msg) self.log_func(self.log_src, msg)
def run(self): def _detect_https(self):
method = None method = None
self.sr = None
if self.cert_path: if self.cert_path:
try: try:
method = self.s.recv(4, socket.MSG_PEEK) method = self.s.recv(4, socket.MSG_PEEK)
@@ -102,16 +106,52 @@ class HttpConn(object):
self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8")) self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
return return
if method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]: return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
def run(self):
self.sr = None
if self.args.https_only:
is_https = True
elif self.args.http_only or not HAVE_SSL:
is_https = False
else:
is_https = self._detect_https()
if is_https:
if self.sr: if self.sr:
self.log("\033[1;31mTODO: cannot do https in jython\033[0m") self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
return return
self.log_src = self.log_src.replace("[36m", "[35m") self.log_src = self.log_src.replace("[36m", "[35m")
try: try:
self.s = ssl.wrap_socket( ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self.s, server_side=True, certfile=self.cert_path ctx.load_cert_chain(self.cert_path)
) if self.args.ssl_ver:
ctx.options &= ~self.args.ssl_flags_en
ctx.options |= self.args.ssl_flags_de
# print(repr(ctx.options))
if self.args.ssl_log:
try:
ctx.keylog_filename = self.args.ssl_log
except:
self.log("keylog failed; openssl or python too old")
if self.args.ciphers:
ctx.set_ciphers(self.args.ciphers)
self.s = ctx.wrap_socket(self.s, server_side=True)
if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
overlap = [y[::-1] for y in self.s.shared_ciphers()]
lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
self.log("\n".join(lines))
for k, v in [
["compression", self.s.compression()],
["ALPN proto", self.s.selected_alpn_protocol()],
["NPN proto", self.s.selected_npn_protocol()],
]:
self.log("TLS {}: {}".format(k, v or "nah"))
except Exception as ex: except Exception as ex:
em = str(ex) em = str(ex)

View File

@@ -38,7 +38,7 @@ class HttpSrv(object):
def accept(self, sck, addr): def accept(self, sck, addr):
"""takes an incoming tcp connection and creates a thread to handle it""" """takes an incoming tcp connection and creates a thread to handle it"""
self.log("%s %s" % addr, "-" * 5 + "C-cthr") self.log("%s %s" % addr, "\033[1;30m|%sC-cthr\033[0m" % ("-" * 5,))
thr = threading.Thread(target=self.thr_client, args=(sck, addr)) thr = threading.Thread(target=self.thr_client, args=(sck, addr))
thr.daemon = True thr.daemon = True
thr.start() thr.start()
@@ -66,11 +66,11 @@ class HttpSrv(object):
thr.start() thr.start()
try: try:
self.log("%s %s" % addr, "-" * 6 + "C-crun") self.log("%s %s" % addr, "\033[1;30m|%sC-crun\033[0m" % ("-" * 6,))
cli.run() cli.run()
finally: finally:
self.log("%s %s" % addr, "-" * 7 + "C-done") self.log("%s %s" % addr, "\033[1;30m|%sC-cdone\033[0m" % ("-" * 7,))
try: try:
sck.shutdown(socket.SHUT_RDWR) sck.shutdown(socket.SHUT_RDWR)
sck.close() sck.close()

View File

@@ -40,7 +40,7 @@ class SvcHub(object):
self.up2k = Up2k(self) self.up2k = Up2k(self)
if self.args.e2d and self.args.e2s: if self.args.e2d and self.args.e2s:
auth = AuthSrv(self.args, self.log) auth = AuthSrv(self.args, self.log, False)
self.up2k.build_indexes(auth.all_writable) self.up2k.build_indexes(auth.all_writable)
# decide which worker impl to use # decide which worker impl to use

View File

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
import re import re
import time import time
import socket import socket
import select
from .util import chkcmd, Counter from .util import chkcmd, Counter
@@ -23,56 +24,73 @@ class TcpSrv(object):
ip = "127.0.0.1" ip = "127.0.0.1"
eps = {ip: "local only"} eps = {ip: "local only"}
if self.args.i != ip: nonlocals = [x for x in self.args.i if x != ip]
eps = self.detect_interfaces(self.args.i) or {self.args.i: "external"} if nonlocals:
eps = self.detect_interfaces(self.args.i)
if not eps:
for x in nonlocals:
eps[x] = "external"
for ip, desc in sorted(eps.items(), key=lambda x: x[1]): for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
self.log( for port in sorted(self.args.p):
"tcpsrv", self.log(
"available @ http://{}:{}/ (\033[33m{}\033[0m)".format( "tcpsrv",
ip, self.args.p, desc "available @ http://{}:{}/ (\033[33m{}\033[0m)".format(
), ip, port, desc
) ),
)
self.srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.srv = []
self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) for ip in self.args.i:
self.srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) for port in self.args.p:
self.srv.append(self._listen(ip, port))
def _listen(self, ip, port):
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
try: try:
self.srv.bind((self.args.i, self.args.p)) srv.bind((ip, port))
return srv
except (OSError, socket.error) as ex: except (OSError, socket.error) as ex:
if ex.errno == 98: if ex.errno in [98, 48]:
raise Exception( e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
"\033[1;31mport {} is busy on interface {}\033[0m".format( elif ex.errno in [99, 49]:
self.args.p, self.args.i e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
) else:
) raise
raise Exception(e)
if ex.errno == 99:
raise Exception(
"\033[1;31minterface {} does not exist\033[0m".format(self.args.i)
)
def run(self): def run(self):
self.srv.listen(self.args.nc) for srv in self.srv:
srv.listen(self.args.nc)
self.log("tcpsrv", "listening @ {0}:{1}".format(self.args.i, self.args.p)) ip, port = srv.getsockname()
self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
while True: while True:
self.log("tcpsrv", "-" * 1 + "C-ncli") self.log("tcpsrv", "\033[1;30m|%sC-ncli\033[0m" % ("-" * 1,))
if self.num_clients.v >= self.args.nc: if self.num_clients.v >= self.args.nc:
time.sleep(0.1) time.sleep(0.1)
continue continue
self.log("tcpsrv", "-" * 2 + "C-acc1") self.log("tcpsrv", "\033[1;30m|%sC-acc1\033[0m" % ("-" * 2,))
sck, addr = self.srv.accept() ready, _, _ = select.select(self.srv, [], [])
self.log("%s %s" % addr, "-" * 3 + "C-acc2") for srv in ready:
self.num_clients.add() sck, addr = srv.accept()
self.hub.broker.put(False, "httpconn", sck, addr) sip, sport = srv.getsockname()
self.log(
"%s %s" % addr,
"\033[1;30m|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
"-" * 3, sip, sport % 8, sport
),
)
self.num_clients.add()
self.hub.broker.put(False, "httpconn", sck, addr)
def shutdown(self): def shutdown(self):
self.log("tcpsrv", "ok bye") self.log("tcpsrv", "ok bye")
def detect_interfaces(self, listen_ip): def detect_interfaces(self, listen_ips):
eps = {} eps = {}
# get all ips and their interfaces # get all ips and their interfaces
@@ -86,8 +104,9 @@ class TcpSrv(object):
for ln in ip_addr.split("\n"): for ln in ip_addr.split("\n"):
try: try:
ip, dev = r.match(ln.rstrip()).groups() ip, dev = r.match(ln.rstrip()).groups()
if listen_ip in ["0.0.0.0", ip]: for lip in listen_ips:
eps[ip] = dev if lip in ["0.0.0.0", ip]:
eps[ip] = dev
except: except:
pass pass
@@ -114,11 +133,12 @@ class TcpSrv(object):
s.close() s.close()
if default_route and listen_ip in ["0.0.0.0", default_route]: for lip in listen_ips:
desc = "\033[32mexternal" if default_route and lip in ["0.0.0.0", default_route]:
try: desc = "\033[32mexternal"
eps[default_route] += ", " + desc try:
except: eps[default_route] += ", " + desc
eps[default_route] = desc except:
eps[default_route] = desc
return eps return eps

View File

@@ -15,7 +15,7 @@ import hashlib
import threading import threading
from copy import deepcopy from copy import deepcopy
from .__init__ import WINDOWS, PY2 from .__init__ import WINDOWS
from .util import Pebkac, Queue, fsdec, fsenc, sanitize_fn, ren_open, atomic_move from .util import Pebkac, Queue, fsdec, fsenc, sanitize_fn, ren_open, atomic_move
HAVE_SQLITE3 = False HAVE_SQLITE3 = False
@@ -130,7 +130,7 @@ class Up2k(object):
if db: if db:
# can be symlink so don't `and d.startswith(top)`` # can be symlink so don't `and d.startswith(top)``
excl = set([d for d in tops if d != top]) excl = set([d for d in tops if d != top])
self._build_dir([db, 0], top, excl, top) self._build_dir([db, 0, time.time()], top, excl, top)
self._drop_lost(db, top) self._drop_lost(db, top)
db.commit() db.commit()
@@ -138,7 +138,7 @@ class Up2k(object):
try: try:
inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))] inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
except Exception as ex: except Exception as ex:
self.log("up2k", "listdir: " + repr(ex)) self.log("up2k", "listdir: {} @ [{}]".format(repr(ex), cdir))
return return
histdir = os.path.join(top, ".hist") histdir = os.path.join(top, ".hist")
@@ -147,7 +147,7 @@ class Up2k(object):
try: try:
inf = os.stat(fsenc(abspath)) inf = os.stat(fsenc(abspath))
except Exception as ex: except Exception as ex:
self.log("up2k", "stat: " + repr(ex)) self.log("up2k", "stat: {} @ [{}]".format(repr(ex), abspath))
continue continue
if stat.S_ISDIR(inf.st_mode): if stat.S_ISDIR(inf.st_mode):
@@ -182,15 +182,18 @@ class Up2k(object):
try: try:
hashes = self._hashlist_from_file(abspath) hashes = self._hashlist_from_file(abspath)
except Exception as ex: except Exception as ex:
self.log("up2k", "hash: " + repr(ex)) self.log("up2k", "hash: {} @ [{}]".format(repr(ex), abspath))
continue continue
wark = self._wark_from_hashlist(inf.st_size, hashes) wark = self._wark_from_hashlist(inf.st_size, hashes)
self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size) self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size)
dbw[1] += 1 dbw[1] += 1
if dbw[1] > 1024: td = time.time() - dbw[2]
if dbw[1] > 1024 or td > 60:
self.log("up2k", "commit {} files".format(dbw[1]))
dbw[0].commit() dbw[0].commit()
dbw[1] = 0 dbw[1] = 0
dbw[2] = time.time()
def _drop_lost(self, db, top): def _drop_lost(self, db, top):
rm = [] rm = []
@@ -201,7 +204,7 @@ class Up2k(object):
if not os.path.exists(fsenc(abspath)): if not os.path.exists(fsenc(abspath)):
rm.append(drp) rm.append(drp)
except Exception as ex: except Exception as ex:
self.log("up2k", "stat-rm: " + repr(ex)) self.log("up2k", "stat-rm: {} @ [{}]".format(repr(ex), abspath))
if not rm: if not rm:
return return
@@ -296,7 +299,12 @@ class Up2k(object):
names = [job[x] for x in ["name", "tnam"] if x in job] names = [job[x] for x in ["name", "tnam"] if x in job]
for fn in names: for fn in names:
path = os.path.join(job["ptop"], job["prel"], fn) path = os.path.join(job["ptop"], job["prel"], fn)
if not os.path.exists(path): try:
if os.path.getsize(path) > 0:
# upload completed or both present
break
except:
# missing; restart
job = None job = None
break break
else: else:
@@ -507,8 +515,15 @@ class Up2k(object):
fsz = os.path.getsize(path) fsz = os.path.getsize(path)
csz = self._get_chunksize(fsz) csz = self._get_chunksize(fsz)
ret = [] ret = []
last_print = time.time()
with open(path, "rb", 512 * 1024) as f: with open(path, "rb", 512 * 1024) as f:
while fsz > 0: while fsz > 0:
now = time.time()
td = now - last_print
if td >= 0.3:
last_print = now
print(" {} \n\033[A".format(fsz), end="")
hashobj = hashlib.sha512() hashobj = hashlib.sha512()
rem = min(csz, fsz) rem = min(csz, fsz)
fsz -= rem fsz -= rem
@@ -530,6 +545,8 @@ class Up2k(object):
self.registry[job["ptop"]][job["wark"]] = job self.registry[job["ptop"]][job["wark"]] = job
pdir = os.path.join(job["ptop"], job["prel"]) pdir = os.path.join(job["ptop"], job["prel"])
job["name"] = self._untaken(pdir, job["name"], job["t0"], job["addr"]) job["name"] = self._untaken(pdir, job["name"], job["t0"], job["addr"])
# if len(job["name"].split(".")) > 8:
# raise Exception("aaa")
tnam = job["name"] + ".PARTIAL" tnam = job["name"] + ".PARTIAL"
suffix = ".{:.6f}-{}".format(job["t0"], job["addr"]) suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
@@ -554,7 +571,7 @@ class Up2k(object):
def _snapshot(self): def _snapshot(self):
persist_interval = 30 # persist unfinished uploads index every 30 sec persist_interval = 30 # persist unfinished uploads index every 30 sec
discard_interval = 3600 # drop unfinished uploads after 1 hour inactivity discard_interval = 21600 # drop unfinished uploads after 6 hours inactivity
prev = {} prev = {}
while True: while True:
time.sleep(persist_interval) time.sleep(persist_interval)

View File

@@ -108,7 +108,7 @@ def ren_open(fname, *args, **kwargs):
with open(fname, *args, **kwargs) as f: with open(fname, *args, **kwargs) as f:
yield {"orz": [f, fname]} yield {"orz": [f, fname]}
return return
orig_name = fname orig_name = fname
bname = fname bname = fname
ext = "" ext = ""
@@ -632,10 +632,10 @@ def sendfile_kern(lower, upper, f, s):
except Exception as ex: except Exception as ex:
# print("sendfile: " + repr(ex)) # print("sendfile: " + repr(ex))
n = 0 n = 0
if n <= 0: if n <= 0:
return upper - ofs return upper - ofs
ofs += n ofs += n
# print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs)) # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
@@ -718,6 +718,22 @@ def py_desc():
) )
def align_tab(lines):
rows = []
ncols = 0
for ln in lines:
row = [x for x in ln.split(" ") if x]
ncols = max(ncols, len(row))
rows.append(row)
lens = [0] * ncols
for row in rows:
for n, col in enumerate(row):
lens[n] = max(lens[n], len(col))
return ["".join(x.ljust(y + 2) for x, y in zip(row, lens)) for row in rows]
class Pebkac(Exception): class Pebkac(Exception):
def __init__(self, code, msg=None): def __init__(self, code, msg=None):
super(Pebkac, self).__init__(msg or HTTPCODE[code]) super(Pebkac, self).__init__(msg or HTTPCODE[code])

View File

@@ -201,6 +201,7 @@ function up2k_init(have_crypto) {
var parallel_uploads = cfg_get('nthread'); var parallel_uploads = cfg_get('nthread');
var multitask = bcfg_get('multitask', true); var multitask = bcfg_get('multitask', true);
var ask_up = bcfg_get('ask_up', true);
var col_hashing = '#00bbff'; var col_hashing = '#00bbff';
var col_hashed = '#004466'; var col_hashed = '#004466';
@@ -297,7 +298,7 @@ function up2k_init(have_crypto) {
for (var a = 0; a < good_files.length; a++) for (var a = 0; a < good_files.length; a++)
msg.push(good_files[a].name); msg.push(good_files[a].name);
if (!confirm(msg.join('\n'))) if (ask_up && !confirm(msg.join('\n')))
return; return;
for (var a = 0; a < good_files.length; a++) { for (var a = 0; a < good_files.length; a++) {
@@ -839,6 +840,11 @@ function up2k_init(have_crypto) {
bcfg_set('multitask', multitask); bcfg_set('multitask', multitask);
} }
function tgl_ask_up() {
ask_up = !ask_up;
bcfg_set('ask_up', ask_up);
}
function nop(ev) { function nop(ev) {
ev.preventDefault(); ev.preventDefault();
this.click(); this.click();
@@ -855,6 +861,7 @@ function up2k_init(have_crypto) {
ebi('nthread').addEventListener('input', bumpthread, false); ebi('nthread').addEventListener('input', bumpthread, false);
ebi('multitask').addEventListener('click', tgl_multitask, false); ebi('multitask').addEventListener('click', tgl_multitask, false);
ebi('ask_up').addEventListener('click', tgl_ask_up, false);
var nodes = ebi('u2conf').getElementsByTagName('a'); var nodes = ebi('u2conf').getElementsByTagName('a');
for (var a = nodes.length - 1; a >= 0; a--) for (var a = nodes.length - 1; a >= 0; a--)

View File

@@ -194,6 +194,12 @@
#u2conf input+a { #u2conf input+a {
background: #d80; background: #d80;
} }
#u2conf input[type="checkbox"]+label {
color: #f5a;
}
#u2conf input[type="checkbox"]:checked+label {
color: #fc5;
}
#u2foot { #u2foot {
color: #fff; color: #fff;
font-style: italic; font-style: italic;

View File

@@ -3,7 +3,8 @@
href="#" data-dest="up2k">up2k</a><i></i><a href="#" data-dest="up2k">up2k</a><i></i><a
href="#" data-dest="bup">bup</a><i></i><a href="#" data-dest="bup">bup</a><i></i><a
href="#" data-dest="mkdir">mkdir</a><i></i><a href="#" data-dest="mkdir">mkdir</a><i></i><a
href="#" data-dest="new_md">new.md</a></div> href="#" data-dest="new_md">new.md</a><i></i><a
href="#" data-dest="msg">msg</a></div>
<div id="op_bup" class="opview opbox act"> <div id="op_bup" class="opview opbox act">
<div id="u2err"></div> <div id="u2err"></div>
@@ -30,6 +31,13 @@
</form> </form>
</div> </div>
<div id="op_msg" class="opview opbox">
<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="/{{ vdir }}">
<input type="text" name="msg" size="30">
<input type="submit" value="send">
</form>
</div>
<div id="op_up2k" class="opview"> <div id="op_up2k" class="opview">
<form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form> <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
@@ -43,10 +51,14 @@
<input class="txtbox" id="nthread" value="2" /> <input class="txtbox" id="nthread" value="2" />
<a href="#" id="nthread_add">+</a> <a href="#" id="nthread_add">+</a>
</td> </td>
<td rowspan="2"> <td rowspan="2" style="padding-left:1.5em">
<input type="checkbox" id="multitask" /> <input type="checkbox" id="multitask" />
<label for="multitask">hash while<br />uploading</label> <label for="multitask">hash while<br />uploading</label>
</td> </td>
<td rowspan="2">
<input type="checkbox" id="ask_up" />
<label for="ask_up">ask for<br />confirmation</label>
</td>
</tr> </tr>
</table> </table>

View File

@@ -8,7 +8,7 @@ exit 1
## (supports linux/macos, probably windows+msys2) ## (supports linux/macos, probably windows+msys2)
gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f -- "$f"; done gzip -d < .hist/up2k.snap | jq -r '.[].tnam' | while IFS= read -r f; do rm -f -- "$f"; done
gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' & rm -f -- "$f"; done gzip -d < .hist/up2k.snap | jq -r '.[].name' | while IFS= read -r f; do wc -c -- "$f" | grep -qiE '^[^0-9a-z]*0' && rm -f -- "$f"; done
## ##

View File

@@ -3,12 +3,15 @@ set -e
echo echo
# osx support # osx support
command -v gtar >/dev/null && # port install gnutar findutils gsed coreutils
command -v gfind >/dev/null && { gtar=$(command -v gtar || command -v gnutar) || true
tar() { gtar "$@"; } [ ! -z "$gtar" ] && command -v gfind >/dev/null && {
tar() { $gtar "$@"; }
sed() { gsed "$@"; } sed() { gsed "$@"; }
find() { gfind "$@"; } find() { gfind "$@"; }
sort() { gsort "$@"; } sort() { gsort "$@"; }
command -v grealpath >/dev/null &&
realpath() { grealpath "$@"; }
} }
which md5sum 2>/dev/null >/dev/null && which md5sum 2>/dev/null >/dev/null &&

View File

@@ -62,28 +62,32 @@ cd sfx
)/pe-copyparty" )/pe-copyparty"
echo "repack of files in $old" echo "repack of files in $old"
cp -pR "$old/"*{jinja2,copyparty} . cp -pR "$old/"*{dep-j2,copyparty} .
mv {x.,}jinja2 2>/dev/null || true
} }
[ $repack ] || { [ $repack ] || {
echo collecting jinja2 echo collecting jinja2
f="../build/Jinja2-2.6.tar.gz" f="../build/Jinja2-2.11.3.tar.gz"
[ -e "$f" ] || [ -e "$f" ] ||
(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz; (url=https://files.pythonhosted.org/packages/4f/e7/65300e6b32e69768ded990494809106f87da1d436418d5f1367ed3966fd7/Jinja2-2.11.3.tar.gz;
wget -O$f "$url" || curl -L "$url" >$f) wget -O$f "$url" || curl -L "$url" >$f)
tar -zxf $f tar -zxf $f
mv Jinja2-*/jinja2 . mv Jinja2-*/src/jinja2 .
rm -rf Jinja2-* jinja2/testsuite jinja2/_markupsafe/tests.py jinja2/_stringdefs.py rm -rf Jinja2-*
f=jinja2/lexer.py echo collecting markupsafe
sed -r '/.*föö.*/ raise SyntaxError/' <$f >t f="../build/MarkupSafe-1.1.1.tar.gz"
tmv $f [ -e "$f" ] ||
(url=https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz;
f=jinja2/_markupsafe/_constants.py wget -O$f "$url" || curl -L "$url" >$f)
awk '!/: [0-9]+,?$/ || /(amp|gt|lt|quot|apos|nbsp).:/' <$f >t
tmv $f tar -zxf $f
mv MarkupSafe-*/src/markupsafe .
rm -rf MarkupSafe-* markupsafe/_speedups.c
mkdir dep-j2/
mv {markupsafe,jinja2} dep-j2/
# msys2 tar is bad, make the best of it # msys2 tar is bad, make the best of it
echo collecting source echo collecting source
@@ -165,6 +169,15 @@ done
sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f" sed -r '/edit2">edit \(fancy/d' <$f >t && tmv "$f"
} }
find | grep -E '\.py$' |
grep -vE '__version__' |
tr '\n' '\0' |
xargs -0 python ../scripts/uncomment.py
f=dep-j2/jinja2/constants.py
awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
tmv "$f"
# up2k goes from 28k to 22k laff # up2k goes from 28k to 22k laff
echo entabbening echo entabbening
find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do find | grep -E '\.(js|css|html|py)$' | while IFS= read -r f; do
@@ -177,7 +190,7 @@ args=(--owner=1000 --group=1000)
[ "$OSTYPE" = msys ] && [ "$OSTYPE" = msys ] &&
args=() args=()
tar -cf tar "${args[@]}" --numeric-owner copyparty jinja2 tar -cf tar "${args[@]}" --numeric-owner copyparty dep-j2
echo compressing tar echo compressing tar
# detect best level; bzip2 -7 is usually better than -9 # detect best level; bzip2 -7 is usually better than -9

View File

@@ -2,12 +2,16 @@
set -e set -e
echo echo
command -v gtar >/dev/null && # osx support
command -v gfind >/dev/null && { # port install gnutar findutils gsed coreutils
tar() { gtar "$@"; } gtar=$(command -v gtar || command -v gnutar) || true
[ ! -z "$gtar" ] && command -v gfind >/dev/null && {
tar() { $gtar "$@"; }
sed() { gsed "$@"; } sed() { gsed "$@"; }
find() { gfind "$@"; } find() { gfind "$@"; }
sort() { gsort "$@"; } sort() { gsort "$@"; }
command -v grealpath >/dev/null &&
realpath() { grealpath "$@"; }
} }
which md5sum 2>/dev/null >/dev/null && which md5sum 2>/dev/null >/dev/null &&

View File

@@ -2,7 +2,7 @@
# coding: utf-8 # coding: utf-8
from __future__ import print_function, unicode_literals from __future__ import print_function, unicode_literals
import re, os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile import os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
import subprocess as sp import subprocess as sp
""" """
@@ -202,93 +202,6 @@ def u8(gen):
yield s yield s
def get_py_win(ret):
tops = []
p = str(os.getenv("LocalAppdata"))
if p:
tops.append(os.path.join(p, "Programs", "Python"))
progfiles = {}
for p in ["ProgramFiles", "ProgramFiles(x86)"]:
p = str(os.getenv(p))
if p:
progfiles[p] = 1
# 32bit apps get x86 for both
if p.endswith(" (x86)"):
progfiles[p[:-6]] = 1
tops += list(progfiles.keys())
for sysroot in [me, sys.executable]:
sysroot = sysroot[:3].upper()
if sysroot[1] == ":" and sysroot not in tops:
tops.append(sysroot)
# $WIRESHARK_SLOGAN
for top in tops:
try:
for name1 in u8(sorted(os.listdir(top), reverse=True)):
if name1.lower().startswith("python"):
path1 = os.path.join(top, name1)
try:
for name2 in u8(os.listdir(path1)):
if name2.lower() == "python.exe":
path2 = os.path.join(path1, name2)
ret[path2.lower()] = path2
except:
pass
except:
pass
def get_py_nix(ret):
ptn = re.compile(r"^(python|pypy)[0-9\.-]*$")
for bindir in os.getenv("PATH").split(":"):
if not bindir:
next
try:
for fn in u8(os.listdir(bindir)):
if ptn.match(fn):
fn = os.path.join(bindir, fn)
ret[fn.lower()] = fn
except:
pass
def read_py(binp):
cmd = [
binp,
"-c",
"import sys; sys.stdout.write(' '.join(str(x) for x in sys.version_info)); import jinja2",
]
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
ver, _ = p.communicate()
ver = ver.decode("utf-8").split(" ")[:3]
ver = [int(x) if x.isdigit() else 0 for x in ver]
return ver, p.returncode == 0
def get_pys():
ver, chk = read_py(sys.executable)
if chk or PY2:
return [[chk, ver, sys.executable]]
hits = {sys.executable.lower(): sys.executable}
if platform.system() == "Windows":
get_py_win(hits)
else:
get_py_nix(hits)
ret = []
for binp in hits.values():
ver, chk = read_py(binp)
ret.append([chk, ver, binp])
msg("\t".join(str(x) for x in ret[-1]))
return ret
def yieldfile(fn): def yieldfile(fn):
with open(fn, "rb") as f: with open(fn, "rb") as f:
for block in iter(lambda: f.read(64 * 1024), b""): for block in iter(lambda: f.read(64 * 1024), b""):
@@ -440,12 +353,11 @@ def confirm():
pass pass
def run(tmp, py): def run(tmp, j2ver):
global cpp global cpp
msg("OK") msg("jinja2:", j2ver or "bundled")
msg("will use:", py) msg("sfxdir:", tmp)
msg("bound to:", tmp)
# "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit # "systemd-tmpfiles-clean.timer"?? HOW do you even come up with this shit
try: try:
@@ -457,24 +369,20 @@ def run(tmp, py):
except: except:
pass pass
fp_py = os.path.join(tmp, "py") ld = [tmp, os.path.join(tmp, "dep-j2")]
try: if j2ver:
with open(fp_py, "wb") as f: del ld[-1]
f.write(py.encode("utf-8") + b"\n")
except:
pass
# avoid loading ./copyparty.py cmd = (
cmd = [ "import sys, runpy; "
py, + "".join(['sys.path.insert(0, r"' + x + '"); ' for x in ld])
"-c", + 'runpy.run_module("copyparty", run_name="__main__")'
'import sys, runpy; sys.path.insert(0, r"' )
+ tmp cmd = [sys.executable, "-c", cmd] + list(sys.argv[1:])
+ '"); runpy.run_module("copyparty", run_name="__main__")',
] + list(sys.argv[1:])
cmd = [str(x) for x in cmd]
msg("\n", cmd, "\n") msg("\n", cmd, "\n")
cpp = sp.Popen(str(x) for x in cmd) cpp = sp.Popen(cmd)
try: try:
cpp.wait() cpp.wait()
except: except:
@@ -494,7 +402,6 @@ def bye(sig, frame):
def main(): def main():
sysver = str(sys.version).replace("\n", "\n" + " " * 18) sysver = str(sys.version).replace("\n", "\n" + " " * 18)
pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP)) pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
os.system("")
msg() msg()
msg(" this is: copyparty", VER) msg(" this is: copyparty", VER)
msg(" packed at:", pktime, "UTC,", STAMP) msg(" packed at:", pktime, "UTC,", STAMP)
@@ -526,33 +433,13 @@ def main():
signal.signal(signal.SIGTERM, bye) signal.signal(signal.SIGTERM, bye)
tmp = unpack() tmp = unpack()
fp_py = os.path.join(tmp, "py")
if os.path.exists(fp_py):
with open(fp_py, "rb") as f:
py = f.read().decode("utf-8").rstrip()
return run(tmp, py) try:
from jinja2 import __version__ as j2ver
except:
j2ver = None
pys = get_pys() return run(tmp, j2ver)
pys.sort(reverse=True)
j2, ver, py = pys[0]
if j2:
try:
os.rename(os.path.join(tmp, "jinja2"), os.path.join(tmp, "x.jinja2"))
except:
pass
return run(tmp, py)
msg("\n could not find jinja2; will use py2 + the bundled version\n")
for _, ver, py in pys:
if ver > [2, 7] and ver < [3, 0]:
return run(tmp, py)
m = "\033[1;31m\n\n\ncould not find a python with jinja2 installed; please do one of these:\n\n pip install --user jinja2\n\n install python2\n\n\033[0m"
msg(m)
confirm()
sys.exit(1)
if __name__ == "__main__": if __name__ == "__main__":

77
scripts/uncomment.py Normal file
View File

@@ -0,0 +1,77 @@
#!/usr/bin/env python
# coding: utf-8
from __future__ import print_function, unicode_literals
import io
import sys
import tokenize
def uncomment(fpath):
""" modified https://stackoverflow.com/a/62074206 """
with open(fpath, "rb") as f:
orig = f.read().decode("utf-8")
out = ""
for ln in orig.split("\n"):
if not ln.startswith("#"):
break
out += ln + "\n"
io_obj = io.StringIO(orig)
prev_toktype = tokenize.INDENT
last_lineno = -1
last_col = 0
for tok in tokenize.generate_tokens(io_obj.readline):
# print(repr(tok))
token_type = tok[0]
token_string = tok[1]
start_line, start_col = tok[2]
end_line, end_col = tok[3]
if start_line > last_lineno:
last_col = 0
if start_col > last_col:
out += " " * (start_col - last_col)
is_legalese = (
"copyright" in token_string.lower() or "license" in token_string.lower()
)
if token_type == tokenize.STRING:
if (
prev_toktype != tokenize.INDENT
and prev_toktype != tokenize.NEWLINE
and start_col > 0
or is_legalese
):
out += token_string
else:
out += '"a"'
elif token_type != tokenize.COMMENT or is_legalese:
out += token_string
prev_toktype = token_type
last_lineno = end_line
last_col = end_col
# out = "\n".join(x for x in out.splitlines() if x.strip())
with open(fpath, "wb") as f:
f.write(out.encode("utf-8"))
def main():
print("uncommenting", end="")
for f in sys.argv[1:]:
print(".", end="")
uncomment(f)
print("k")
if __name__ == "__main__":
main()

View File

@@ -110,7 +110,6 @@ args = {
"Programming Language :: Python :: 2", "Programming Language :: Python :: 2",
"Programming Language :: Python :: 2.7", "Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3", "Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.2",
"Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.3",
"Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.5",