paulmataruso/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-11-03 21:43:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

u2c: early exclude + fix py2 perf

Browse Source 
* don't traverse into excluded folders
* 3.5x faster on python2.7
This commit is contained in:
ed
2024-09-23 17:20:04 +00:00
parent 9daeed923f
commit b9c5c7bbde
1 changed files with 19 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
bin/u2c.py
View File

@@ -1,8 +1,8 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from __future__ import print_function, unicode_literals from __future__ import print_function, unicode_literals
S_VERSION = "2.0" S_VERSION = "2.1"
S_BUILD_DT = "2024-09-22" S_BUILD_DT = "2024-09-23"
""" """
u2c.py: upload to copyparty u2c.py: upload to copyparty
@@ -11,7 +11,6 @@ https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py
- dependencies: no - dependencies: no
- supports python 2.6, 2.7, and 3.3 through 3.12 - supports python 2.6, 2.7, and 3.3 through 3.12
(for higher performance on 2.6 and 2.7, use u2c v1.x)
- if something breaks just try again and it'll autoresume - if something breaks just try again and it'll autoresume
""" """
@@ -43,6 +42,8 @@ except:
PY2 = sys.version_info < (3,) PY2 = sys.version_info < (3,)
PY27 = sys.version_info > (2, 7) and PY2
PY37 = sys.version_info > (3, 7)
if PY2: if PY2:
import httplib as http_client import httplib as http_client
from Queue import Queue from Queue import Queue
@@ -170,8 +171,10 @@ class HCli(object):
} }
def _connect(self): def _connect(self):
sbs = "blocksize" args = {}
args = {sbs: 1048576} if PY37:
args["blocksize"] = 1048576
if not self.tls: if not self.tls:
C = http_client.HTTPConnection C = http_client.HTTPConnection
else: else:
@@ -179,13 +182,7 @@ class HCli(object):
if self.ctx: if self.ctx:
args = {"context": self.ctx} args = {"context": self.ctx}
for _ in range(2): return C(self.addr, self.port, timeout=999, **args)
try:
return C(self.addr, self.port, timeout=999, **args)
except:
if sbs not in args:
raise
del args[sbs]
def req(self, meth, vpath, hdrs, body=None, ctype=None): def req(self, meth, vpath, hdrs, body=None, ctype=None):
hdrs.update(self.base_hdrs) hdrs.update(self.base_hdrs)
@@ -201,7 +198,11 @@ class HCli(object):
c = self.conns.pop() if self.conns else self._connect() c = self.conns.pop() if self.conns else self._connect()
try: try:
c.request(meth, vpath, body, hdrs) c.request(meth, vpath, body, hdrs)
rsp = c.getresponse() if PY27:
rsp = c.getresponse(buffering=True)
else:
rsp = c.getresponse()
data = rsp.read() data = rsp.read()
self.conns.append(c) self.conns.append(c)
return rsp.status, data.decode("utf-8") return rsp.status, data.decode("utf-8")
@@ -545,7 +546,7 @@ else:
statdir = _lsd statdir = _lsd
def walkdir(err, top, seen): def walkdir(err, top, excl, seen):
"""recursive statdir""" """recursive statdir"""
atop = os.path.abspath(os.path.realpath(top)) atop = os.path.abspath(os.path.realpath(top))
if atop in seen: if atop in seen:
@@ -554,10 +555,12 @@ def walkdir(err, top, seen):
seen = seen[:] + [atop] seen = seen[:] + [atop]
for ap, inf in sorted(statdir(err, top)): for ap, inf in sorted(statdir(err, top)):
if excl.match(ap):
continue
yield ap, inf yield ap, inf
if stat.S_ISDIR(inf.st_mode): if stat.S_ISDIR(inf.st_mode):
try: try:
for x in walkdir(err, ap, seen): for x in walkdir(err, ap, excl, seen):
yield x yield x
except Exception as ex: except Exception as ex:
err.append((ap, str(ex))) err.append((ap, str(ex)))
@@ -597,9 +600,7 @@ def walkdirs(err, tops, excl):
yield stop, dn, os.stat(stop) yield stop, dn, os.stat(stop)
if isdir: if isdir:
for ap, inf in walkdir(err, top, []): for ap, inf in walkdir(err, top, ptn, []):
if ptn.match(ap):
continue
yield stop, ap[len(stop) :].lstrip(sep), inf yield stop, ap[len(stop) :].lstrip(sep), inf
else: else:
d, n = top.rsplit(sep, 1) d, n = top.rsplit(sep, 1)