diff --git a/ZeroNetOldFork.zip b/ZeroNetOldFork.zip new file mode 100644 index 000000000..4bfafcbf5 Binary files /dev/null and b/ZeroNetOldFork.zip differ diff --git a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py index fab7bb1ff..ae674c008 100644 --- a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py +++ b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py @@ -3,7 +3,7 @@ import struct import socket -import lib.bencode_open as bencode_open +import bencode from lib.subtl.subtl import UdpTrackerClient import socks import sockshandler @@ -133,7 +133,9 @@ def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protoc # Decode peers try: - peer_data = bencode_open.loads(response)[b"peers"] + peer_data = bencode.decode(response)["peers"] + if type(peer_data) is not bytes: + peer_data = peer_data.encode() response = None peer_count = int(len(peer_data) / 6) peers = [] diff --git a/plugins/AnnounceLocal/AnnounceLocalPlugin.py b/plugins/AnnounceLocal/AnnounceLocalPlugin.py index b92259665..0919762a3 100644 --- a/plugins/AnnounceLocal/AnnounceLocalPlugin.py +++ b/plugins/AnnounceLocal/AnnounceLocalPlugin.py @@ -120,11 +120,12 @@ def actionSiteListResponse(self, sender, params): @PluginManager.registerTo("FileServer") class FileServerPlugin(object): def __init__(self, *args, **kwargs): - super(FileServerPlugin, self).__init__(*args, **kwargs) + res = super(FileServerPlugin, self).__init__(*args, **kwargs) if config.broadcast_port and config.tor != "always" and not config.disable_udp: self.local_announcer = LocalAnnouncer(self, config.broadcast_port) else: self.local_announcer = None + return res def start(self, *args, **kwargs): if self.local_announcer: diff --git a/plugins/AnnounceZero/AnnounceZeroPlugin.py b/plugins/AnnounceZero/AnnounceZeroPlugin.py index 623cd4b5e..dcaa04f04 100644 --- a/plugins/AnnounceZero/AnnounceZeroPlugin.py +++ b/plugins/AnnounceZero/AnnounceZeroPlugin.py @@ -21,15 +21,6 @@ def importHostClasses(): # Process result got back from tracker def processPeerRes(tracker_address, site, peers): added = 0 - - # Onion - found_onion = 0 - for packed_address in peers["onion"]: - found_onion += 1 - peer_onion, peer_port = helper.unpackOnionAddress(packed_address) - if site.addPeer(peer_onion, peer_port, source="tracker"): - added += 1 - # Ip4 found_ipv4 = 0 peers_normal = itertools.chain(peers.get("ip4", []), peers.get("ipv4", []), peers.get("ipv6", [])) @@ -38,6 +29,13 @@ def processPeerRes(tracker_address, site, peers): peer_ip, peer_port = helper.unpackAddress(packed_address) if site.addPeer(peer_ip, peer_port, source="tracker"): added += 1 + # Onion + found_onion = 0 + for packed_address in peers["onion"]: + found_onion += 1 + peer_onion, peer_port = helper.unpackOnionAddress(packed_address) + if site.addPeer(peer_onion, peer_port, source="tracker"): + added += 1 if added: site.worker_manager.onPeers() @@ -133,8 +131,8 @@ def announceTrackerZero(self, tracker_address, mode="start", num_want=10): tracker_peer.remove() # Close connection, we don't need it in next 5 minute self.site.log.debug( - "Tracker announce result: zero://%s (sites: %s, new peers: %s, add: %s, mode: %s) in %.3fs" % - (tracker_address, site_index, peers_added, add_types, mode, time.time() - s) + "Tracker announce result: zero://%s (sites: %s, new peers: %s, add: %s) in %.3fs" % + (tracker_address, site_index, peers_added, add_types, time.time() - s) ) return True diff --git a/plugins/Bigfile/BigfilePlugin.py b/plugins/Bigfile/BigfilePlugin.py index 78a27b05d..e3974ef61 100644 --- a/plugins/Bigfile/BigfilePlugin.py +++ b/plugins/Bigfile/BigfilePlugin.py @@ -33,7 +33,6 @@ def importPluginnedClasses(): from Content.ContentManager import VerifyError from Config import config - if "upload_nonces" not in locals(): upload_nonces = {} @@ -61,44 +60,13 @@ def actionBigfileUpload(self): }) self.readMultipartHeaders(self.env['wsgi.input']) # Skip http headers - result = self.handleBigfileUpload(upload_info, self.env['wsgi.input'].read) - return json.dumps(result) - - def actionBigfileUploadWebsocket(self): - ws = self.env.get("wsgi.websocket") - - if not ws: - self.start_response("400 Bad Request", []) - return [b"Not a websocket request!"] - - nonce = self.get.get("upload_nonce") - if nonce not in upload_nonces: - return self.error403("Upload nonce error.") - upload_info = upload_nonces[nonce] - del upload_nonces[nonce] - - ws.send("poll") - - buffer = b"" - def read(size): - nonlocal buffer - while len(buffer) < size: - buffer += ws.receive() - ws.send("poll") - part, buffer = buffer[:size], buffer[size:] - return part - - result = self.handleBigfileUpload(upload_info, read) - ws.send(json.dumps(result)) - - def handleBigfileUpload(self, upload_info, read): site = upload_info["site"] inner_path = upload_info["inner_path"] with site.storage.open(inner_path, "wb", create_dirs=True) as out_file: merkle_root, piece_size, piecemap_info = site.content_manager.hashBigfile( - read, upload_info["size"], upload_info["piece_size"], out_file + self.env['wsgi.input'], upload_info["size"], upload_info["piece_size"], out_file ) if len(piecemap_info["sha512_pieces"]) == 1: # Small file, don't split @@ -137,12 +105,12 @@ def handleBigfileUpload(self, upload_info, read): site.content_manager.contents.loadItem(file_info["content_inner_path"]) # reload cache - return { + return json.dumps({ "merkle_root": merkle_root, "piece_num": len(piecemap_info["sha512_pieces"]), "piece_size": piece_size, "inner_path": inner_path - } + }) def readMultipartHeaders(self, wsgi_input): found = False @@ -169,7 +137,7 @@ def actionFile(self, file_path, *args, **kwargs): @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): - def actionBigfileUploadInit(self, to, inner_path, size, protocol="xhr"): + def actionBigfileUploadInit(self, to, inner_path, size): valid_signers = self.site.content_manager.getValidSigners(inner_path) auth_address = self.user.getAuthAddress(self.site.address) if not self.site.settings["own"] and auth_address not in valid_signers: @@ -193,29 +161,12 @@ def actionBigfileUploadInit(self, to, inner_path, size, protocol="xhr"): "piece_size": piece_size, "piecemap": inner_path + ".piecemap.msgpack" } - - if protocol == "xhr": - return { - "url": "/ZeroNet-Internal/BigfileUpload?upload_nonce=" + nonce, - "piece_size": piece_size, - "inner_path": inner_path, - "file_relative_path": file_relative_path - } - elif protocol == "websocket": - server_url = self.request.getWsServerUrl() - if server_url: - proto, host = server_url.split("://") - origin = proto.replace("http", "ws") + "://" + host - else: - origin = "{origin}" - return { - "url": origin + "/ZeroNet-Internal/BigfileUploadWebsocket?upload_nonce=" + nonce, - "piece_size": piece_size, - "inner_path": inner_path, - "file_relative_path": file_relative_path - } - else: - return {"error": "Unknown protocol"} + return { + "url": "/ZeroNet-Internal/BigfileUpload?upload_nonce=" + nonce, + "piece_size": piece_size, + "inner_path": inner_path, + "file_relative_path": file_relative_path + } @flag.no_multiuser def actionSiteSetAutodownloadBigfileLimit(self, to, limit): @@ -258,14 +209,14 @@ def getFileInfo(self, inner_path, *args, **kwargs): file_info = super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs) return file_info - def readFile(self, read_func, size, buff_size=1024 * 64): + def readFile(self, file_in, size, buff_size=1024 * 64): part_num = 0 recv_left = size while 1: part_num += 1 read_size = min(buff_size, recv_left) - part = read_func(read_size) + part = file_in.read(read_size) if not part: break @@ -278,7 +229,7 @@ def readFile(self, read_func, size, buff_size=1024 * 64): if recv_left <= 0: break - def hashBigfile(self, read_func, size, piece_size=1024 * 1024, file_out=None): + def hashBigfile(self, file_in, size, piece_size=1024 * 1024, file_out=None): self.site.settings["has_bigfile"] = True recv = 0 @@ -291,7 +242,7 @@ def hashBigfile(self, read_func, size, piece_size=1024 * 1024, file_out=None): mt.hash_function = CryptHash.sha512t part = "" - for part in self.readFile(read_func, size): + for part in self.readFile(file_in, size): if file_out: file_out.write(part) @@ -357,7 +308,7 @@ def hashFile(self, dir_inner_path, file_relative_path, optional=False): return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional) self.log.info("- [HASHING] %s" % file_relative_path) - merkle_root, piece_size, piecemap_info = self.hashBigfile(self.site.storage.open(inner_path, "rb").read, file_size) + merkle_root, piece_size, piecemap_info = self.hashBigfile(self.site.storage.open(inner_path, "rb"), file_size) if not hash: hash = merkle_root @@ -389,11 +340,7 @@ def getPiecemap(self, inner_path): return piecemap def verifyPiece(self, inner_path, pos, piece): - try: - piecemap = self.getPiecemap(inner_path) - except Exception as err: - raise VerifyError("Unable to download piecemap: %s" % Debug.formatException(err)) - + piecemap = self.getPiecemap(inner_path) piece_i = int(pos / piecemap["piece_size"]) if CryptHash.sha512sum(piece, format="digest") != piecemap["sha512_pieces"][piece_i]: raise VerifyError("Invalid hash") @@ -459,7 +406,9 @@ def __init__(self, *args, **kwargs): def createSparseFile(self, inner_path, size, sha512=None): file_path = self.getPath(inner_path) - self.ensureDir(os.path.dirname(inner_path)) + file_dir = os.path.dirname(file_path) + if not os.path.isdir(file_dir): + os.makedirs(file_dir) f = open(file_path, 'wb') f.truncate(min(1024 * 1024 * 5, size)) # Only pre-allocate up to 5MB @@ -483,7 +432,9 @@ def write(self, inner_path, content): file_path = self.getPath(inner_path) # Create dir if not exist - self.ensureDir(os.path.dirname(inner_path)) + file_dir = os.path.dirname(file_path) + if not os.path.isdir(file_dir): + os.makedirs(file_dir) if not os.path.isfile(file_path): file_info = self.site.content_manager.getFileInfo(inner_path) @@ -597,9 +548,6 @@ def seek(self, pos, whence=0): whence = 0 return self.f.seek(pos, whence) - def seekable(self): - return self.f.seekable() - def tell(self): return self.f.tell() diff --git a/plugins/Bigfile/Test/TestBigfile.py b/plugins/Bigfile/Test/TestBigfile.py index 402646a6e..9f67838e9 100644 --- a/plugins/Bigfile/Test/TestBigfile.py +++ b/plugins/Bigfile/Test/TestBigfile.py @@ -134,7 +134,7 @@ def testRangedFileDownload(self, file_server, site, site_temp): peer_client = site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"] assert not bad_files @@ -172,7 +172,7 @@ def testOpenBigfile(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) # Open virtual file assert not site_temp.storage.isFile(inner_path) @@ -255,7 +255,7 @@ def testFileGet(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) # Download second block with site_temp.storage.openBigfile(inner_path) as f: @@ -380,7 +380,7 @@ def testDownloadStats(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) # Open virtual file assert not site_temp.storage.isFile(inner_path) @@ -417,7 +417,7 @@ def testPrebuffer(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) # Open virtual file assert not site_temp.storage.isFile(inner_path) @@ -453,7 +453,7 @@ def testDownloadAllPieces(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) # Open virtual file assert not site_temp.storage.isFile(inner_path) @@ -482,7 +482,7 @@ def testFileSize(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) # Open virtual file assert not site_temp.storage.isFile(inner_path) @@ -507,7 +507,7 @@ def testFileRename(self, file_server, site, site_temp): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) with Spy.Spy(FileRequest, "route") as requests: site_temp.needFile("%s|%s-%s" % (inner_path, 0, 1 * self.piece_size)) @@ -529,7 +529,7 @@ def testFileRename(self, file_server, site, site_temp): with Spy.Spy(FileRequest, "route") as requests: site.publish() time.sleep(0.1) - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) # Wait for download + site_temp.download(blind_includes=True).join(timeout=5) # Wait for download assert len([req[1] for req in requests if req[1] == "streamFile"]) == 0 @@ -563,7 +563,7 @@ def testNullFileRead(self, file_server, site, site_temp, size): site_temp.addPeer(file_server.ip, 1544) # Download site - site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) + site_temp.download(blind_includes=True).join(timeout=5) if "piecemap" in site.content_manager.getFileInfo(inner_path): # Bigfile site_temp.needFile(inner_path + "|all") diff --git a/plugins/Chart/ChartCollector.py b/plugins/Chart/ChartCollector.py index 215c603cf..776343afc 100644 --- a/plugins/Chart/ChartCollector.py +++ b/plugins/Chart/ChartCollector.py @@ -119,8 +119,6 @@ def collectDatas(self, collectors, last_values, site=None): value = collector(peers) else: value = collector() - except ValueError: - value = None except Exception as err: self.log.info("Collector %s error: %s" % (key, err)) value = None @@ -148,14 +146,15 @@ def collectGlobal(self, collectors, last_values): s = time.time() cur = self.db.getCursor() - cur.executemany("INSERT INTO data (type_id, value, date_added) VALUES (?, ?, ?)", values) + cur.cursor.executemany("INSERT INTO data (type_id, value, date_added) VALUES (?, ?, ?)", values) + cur.close() self.log.debug("Global collectors inserted in %.3fs" % (time.time() - s)) def collectSites(self, sites, collectors, last_values): now = int(time.time()) s = time.time() values = [] - for address, site in list(sites.items()): + for address, site in sites.items(): site_datas = self.collectDatas(collectors, last_values["site:%s" % address], site) for key, value in site_datas.items(): values.append((self.db.getTypeId(key), self.db.getSiteId(address), value, now)) @@ -164,7 +163,8 @@ def collectSites(self, sites, collectors, last_values): s = time.time() cur = self.db.getCursor() - cur.executemany("INSERT INTO data (type_id, site_id, value, date_added) VALUES (?, ?, ?, ?)", values) + cur.cursor.executemany("INSERT INTO data (type_id, site_id, value, date_added) VALUES (?, ?, ?, ?)", values) + cur.close() self.log.debug("Site collectors inserted in %.3fs" % (time.time() - s)) def collector(self): diff --git a/plugins/Chart/ChartDb.py b/plugins/Chart/ChartDb.py index 66a22082b..9dd4d3dbf 100644 --- a/plugins/Chart/ChartDb.py +++ b/plugins/Chart/ChartDb.py @@ -48,15 +48,15 @@ def getSchema(self): def getTypeId(self, name): if name not in self.types: - res = self.execute("INSERT INTO type ?", {"name": name}) - self.types[name] = res.lastrowid + self.execute("INSERT INTO type ?", {"name": name}) + self.types[name] = self.cur.cursor.lastrowid return self.types[name] def getSiteId(self, address): if address not in self.sites: - res = self.execute("INSERT INTO site ?", {"address": address}) - self.sites[address] = res.lastrowid + self.execute("INSERT INTO site ?", {"address": address}) + self.sites[address] = self.cur.cursor.lastrowid return self.sites[address] diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py index f2f84b497..f46321d42 100644 --- a/plugins/ContentFilter/ContentFilterPlugin.py +++ b/plugins/ContentFilter/ContentFilterPlugin.py @@ -1,6 +1,7 @@ import time import re import html +import hashlib import os from Plugin import PluginManager @@ -24,24 +25,6 @@ def load(self, *args, **kwargs): super(SiteManagerPlugin, self).load(*args, **kwargs) filter_storage = ContentFilterStorage(site_manager=self) - def add(self, address, *args, **kwargs): - should_ignore_block = kwargs.get("ignore_block") or kwargs.get("settings") - if should_ignore_block: - block_details = None - elif filter_storage.isSiteblocked(address): - block_details = filter_storage.getSiteblockDetails(address) - else: - address_hashed = filter_storage.getSiteAddressHashed(address) - if filter_storage.isSiteblocked(address_hashed): - block_details = filter_storage.getSiteblockDetails(address_hashed) - else: - block_details = None - - if block_details: - raise Exception("Site blocked: %s" % html.escape(block_details.get("reason", "unknown reason"))) - else: - return super(SiteManagerPlugin, self).add(address, *args, **kwargs) - @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): @@ -89,17 +72,6 @@ def actionMuteList(self, to): self.response(to, filter_storage.file_content["mutes"]) # Siteblock - @flag.no_multiuser - @flag.admin - def actionSiteblockIgnoreAddSite(self, to, site_address): - if site_address in filter_storage.site_manager.sites: - return {"error": "Site already added"} - else: - if filter_storage.site_manager.need(site_address, ignore_block=True): - return "ok" - else: - return {"error": "Invalid address"} - @flag.no_multiuser @flag.admin def actionSiteblockAdd(self, to, site_address, reason=None): @@ -118,18 +90,6 @@ def actionSiteblockRemove(self, to, site_address): def actionSiteblockList(self, to): self.response(to, filter_storage.file_content["siteblocks"]) - @flag.admin - def actionSiteblockGet(self, to, site_address): - if filter_storage.isSiteblocked(site_address): - res = filter_storage.getSiteblockDetails(site_address) - else: - site_address_hashed = filter_storage.getSiteAddressHashed(site_address) - if filter_storage.isSiteblocked(site_address_hashed): - res = filter_storage.getSiteblockDetails(site_address_hashed) - else: - res = {"error": "Site block not found"} - self.response(to, res) - # Include @flag.no_multiuser def actionFilterIncludeAdd(self, to, inner_path, description=None, address=None): @@ -231,15 +191,15 @@ def actionWrapper(self, path, extra_headers=None): if self.server.site_manager.get(address): # Site already exists return super(UiRequestPlugin, self).actionWrapper(path, extra_headers) - if self.isDomain(address): - address = self.resolveDomain(address) + if self.server.site_manager.isDomain(address): + address = self.server.site_manager.resolveDomain(address) if address: - address_hashed = filter_storage.getSiteAddressHashed(address) + address_sha256 = "0x" + hashlib.sha256(address.encode("utf8")).hexdigest() else: - address_hashed = None + address_sha256 = None - if filter_storage.isSiteblocked(address) or filter_storage.isSiteblocked(address_hashed): + if filter_storage.isSiteblocked(address) or filter_storage.isSiteblocked(address_sha256): site = self.server.site_manager.get(config.homepage) if not extra_headers: extra_headers = {} diff --git a/plugins/ContentFilter/ContentFilterStorage.py b/plugins/ContentFilter/ContentFilterStorage.py index 289ec2a98..3df0b4354 100644 --- a/plugins/ContentFilter/ContentFilterStorage.py +++ b/plugins/ContentFilter/ContentFilterStorage.py @@ -3,14 +3,12 @@ import logging import collections import time -import hashlib from Debug import Debug from Plugin import PluginManager from Config import config from util import helper - class ContentFilterStorage(object): def __init__(self, site_manager): self.log = logging.getLogger("ContentFilterStorage") @@ -116,33 +114,11 @@ def isMuted(self, auth_address): else: return False - def getSiteAddressHashed(self, address): - return "0x" + hashlib.sha256(address.encode("ascii")).hexdigest() - def isSiteblocked(self, address): if address in self.file_content["siteblocks"] or address in self.include_filters["siteblocks"]: return True - return False - - def getSiteblockDetails(self, address): - details = self.file_content["siteblocks"].get(address) - if not details: - address_sha256 = self.getSiteAddressHashed(address) - details = self.file_content["siteblocks"].get(address_sha256) - - if not details: - includes = self.file_content.get("includes", {}).values() - for include in includes: - include_site = self.site_manager.get(include["address"]) - if not include_site: - continue - content = include_site.storage.loadJson(include["inner_path"]) - details = content.get("siteblocks", {}).get(address) - if details: - details["include"] = include - break - - return details + else: + return False # Search and remove or readd files of an user def changeDbs(self, auth_address, action): diff --git a/plugins/ContentFilter/media/blocklisted.html b/plugins/ContentFilter/media/blocklisted.html index c9d201a92..9a287b722 100644 --- a/plugins/ContentFilter/media/blocklisted.html +++ b/plugins/ContentFilter/media/blocklisted.html @@ -62,7 +62,25 @@

This site is on your blocklist:

} async updateSiteblockDetails(address) { - var block = await this.cmdp("siteblockGet", address) + var address_sha256 = await sha256hex(address) + var blocks = await this.cmdp("siteblockList") + if (blocks[address] || blocks[address_sha256]) { + block = blocks[address] + } else { + var includes = await this.cmdp("filterIncludeList", {all_sites: true, filters: true}) + for (let include of includes) { + if (include["siteblocks"][address]) { + var block = include["siteblocks"][address] + block["include"] = include + } + if (include["siteblocks"][address_sha256]) { + var block = include["siteblocks"][address_sha256] + block["include"] = include + } + } + } + + this.blocks = blocks var reason = block["reason"] if (!reason) reason = "Unknown reason" var date = new Date(block["date_added"] * 1000) @@ -77,7 +95,7 @@

This site is on your blocklist:

document.getElementById("visit").style.opacity = "1" document.getElementById("visit").onclick = () => { if (block["include"]) - this.cmd("siteblockIgnoreAddSite", address, () => { this.cmd("wrapperReload") }) + this.cmd("siteAdd", address, () => { this.cmd("wrapperReload") }) else this.cmd("siteblockRemove", address, () => { this.cmd("wrapperReload") }) } diff --git a/plugins/Cors/CorsPlugin.py b/plugins/Cors/CorsPlugin.py index c9437538f..a26c5c2bf 100644 --- a/plugins/Cors/CorsPlugin.py +++ b/plugins/Cors/CorsPlugin.py @@ -2,7 +2,6 @@ import html import copy import os -import gevent from Plugin import PluginManager from Translate import Translate @@ -75,53 +74,28 @@ def actionOptionalFileInfo(self, to, inner_path, *args, **kwargs): return self.corsFuncWrapper("actionOptionalFileInfo", to, inner_path, *args, **kwargs) def actionCorsPermission(self, to, address): - if isinstance(address, list): - addresses = address + site = self.server.sites.get(address) + if site: + site_name = site.content_manager.contents.get("content.json", {}).get("title") + button_title = _["Grant"] else: - addresses = [address] - - button_title = _["Grant"] - site_names = [] - site_addresses = [] - for address in addresses: - site = self.server.sites.get(address) - if site: - site_name = site.content_manager.contents.get("content.json", {}).get("title", address) - else: - site_name = address - # If at least one site is not downloaded yet, show "Grant & Add" instead - button_title = _["Grant & Add"] - - if not (site and "Cors:" + address in self.permissions): - # No site or no permission - site_names.append(site_name) - site_addresses.append(address) - - if len(site_names) == 0: + site_name = address + button_title = _["Grant & Add"] + + if site and "Cors:" + address in self.permissions: return "ignored" self.cmd( "confirm", - [_["This site requests read permission to: %s"] % ", ".join(map(html.escape, site_names)), button_title], - lambda res: self.cbCorsPermission(to, site_addresses) + [_["This site requests read permission to: %s"] % html.escape(site_name), button_title], + lambda res: self.cbCorsPermission(to, address) ) - def cbCorsPermission(self, to, addresses): - # Add permissions - for address in addresses: - permission = "Cors:" + address - if permission not in self.site.settings["permissions"]: - self.site.settings["permissions"].append(permission) - - self.site.saveSettings() - self.site.updateWebsocket(permission_added=permission) - - self.response(to, "ok") - - for address in addresses: - site = self.server.sites.get(address) - if not site: - gevent.spawn(self.server.site_manager.need, address) + def cbCorsPermission(self, to, address): + self.actionPermissionAdd(to, "Cors:" + address) + site = self.server.sites.get(address) + if not site: + self.server.site_manager.need(address) @PluginManager.registerTo("UiRequest") diff --git a/plugins/CryptMessage/CryptMessage.py b/plugins/CryptMessage/CryptMessage.py index 8349809cc..6f2cbdc2c 100644 --- a/plugins/CryptMessage/CryptMessage.py +++ b/plugins/CryptMessage/CryptMessage.py @@ -1,58 +1,59 @@ import hashlib import base64 -import struct -from lib import sslcrypto -from Crypt import Crypt +import lib.pybitcointools as btctools -curve = sslcrypto.ecc.get_curve("secp256k1") +ecc_cache = {} -def eciesEncrypt(data, pubkey, ciphername="aes-256-cbc"): - ciphertext, key_e = curve.encrypt( - data, - base64.b64decode(pubkey), - algo=ciphername, - derivation="sha512", - return_aes_key=True - ) - return key_e, ciphertext +def eciesEncrypt(data, pubkey, ephemcurve=None, ciphername='aes-256-cbc'): + from lib import pyelliptic + pubkey_openssl = toOpensslPublickey(base64.b64decode(pubkey)) + curve, pubkey_x, pubkey_y, i = pyelliptic.ECC._decode_pubkey(pubkey_openssl) + if ephemcurve is None: + ephemcurve = curve + ephem = pyelliptic.ECC(curve=ephemcurve) + key = hashlib.sha512(ephem.raw_get_ecdh_key(pubkey_x, pubkey_y)).digest() + key_e, key_m = key[:32], key[32:] + pubkey = ephem.get_pubkey() + iv = pyelliptic.OpenSSL.rand(pyelliptic.OpenSSL.get_cipher(ciphername).get_blocksize()) + ctx = pyelliptic.Cipher(key_e, iv, 1, ciphername) + ciphertext = iv + pubkey + ctx.ciphering(data) + mac = pyelliptic.hmac_sha256(key_m, ciphertext) + return key_e, ciphertext + mac +def eciesDecrypt(encrypted_data, privatekey): + ecc_key = getEcc(privatekey) + return ecc_key.decrypt(base64.b64decode(encrypted_data)) -@Crypt.thread_pool_crypt.wrap -def eciesDecryptMulti(encrypted_datas, privatekey): - texts = [] # Decoded texts - for encrypted_data in encrypted_datas: - try: - text = eciesDecrypt(encrypted_data, privatekey).decode("utf8") - texts.append(text) - except Exception: - texts.append(None) - return texts +def split(encrypted): + iv = encrypted[0:16] + ciphertext = encrypted[16 + 70:-32] + return iv, ciphertext -def eciesDecrypt(ciphertext, privatekey): - return curve.decrypt(base64.b64decode(ciphertext), curve.wif_to_private(privatekey.encode()), derivation="sha512") +def getEcc(privatekey=None): + from lib import pyelliptic + global ecc_cache + if privatekey not in ecc_cache: + if privatekey: + publickey_bin = btctools.encode_pubkey(btctools.privtopub(privatekey), "bin") + publickey_openssl = toOpensslPublickey(publickey_bin) + privatekey_openssl = toOpensslPrivatekey(privatekey) + ecc_cache[privatekey] = pyelliptic.ECC(curve='secp256k1', privkey=privatekey_openssl, pubkey=publickey_openssl) + else: + ecc_cache[None] = pyelliptic.ECC() + return ecc_cache[privatekey] -def decodePubkey(pubkey): - i = 0 - curve = struct.unpack('!H', pubkey[i:i + 2])[0] - i += 2 - tmplen = struct.unpack('!H', pubkey[i:i + 2])[0] - i += 2 - pubkey_x = pubkey[i:i + tmplen] - i += tmplen - tmplen = struct.unpack('!H', pubkey[i:i + 2])[0] - i += 2 - pubkey_y = pubkey[i:i + tmplen] - i += tmplen - return curve, pubkey_x, pubkey_y, i +def toOpensslPrivatekey(privatekey): + privatekey_bin = btctools.encode_privkey(privatekey, "bin") + return b'\x02\xca\x00\x20' + privatekey_bin -def split(encrypted): - iv = encrypted[0:16] - curve, pubkey_x, pubkey_y, i = decodePubkey(encrypted[16:]) - ciphertext = encrypted[16 + i:-32] - return iv, ciphertext +def toOpensslPublickey(publickey): + publickey_bin = btctools.encode_pubkey(publickey, "bin") + publickey_bin = publickey_bin[1:] + publickey_openssl = b'\x02\xca\x00 ' + publickey_bin[:32] + b'\x00 ' + publickey_bin[32:] + return publickey_openssl diff --git a/plugins/CryptMessage/CryptMessagePlugin.py b/plugins/CryptMessage/CryptMessagePlugin.py index 7c24f7307..64c8ac81f 100644 --- a/plugins/CryptMessage/CryptMessagePlugin.py +++ b/plugins/CryptMessage/CryptMessagePlugin.py @@ -1,26 +1,26 @@ import base64 import os -import gevent - from Plugin import PluginManager from Crypt import CryptBitcoin, CryptHash -from Config import config -import sslcrypto +import lib.pybitcointools as btctools from . import CryptMessage -curve = sslcrypto.ecc.get_curve("secp256k1") - @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): + def eciesDecrypt(self, encrypted, privatekey): + back = CryptMessage.getEcc(privatekey).decrypt(encrypted) + return back.decode("utf8") + # - Actions - # Returns user's public key unique to site # Return: Public key def actionUserPublickey(self, to, index=0): - self.response(to, self.user.getEncryptPublickey(self.site.address, index)) + publickey = self.user.getEncryptPublickey(self.site.address, index) + self.response(to, publickey) # Encrypt a text using the publickey or user's sites unique publickey # Return: Encrypted text using base64 encoding @@ -44,7 +44,13 @@ def actionEciesDecrypt(self, to, param, privatekey=0): else: encrypted_texts = [param] - texts = CryptMessage.eciesDecryptMulti(encrypted_texts, privatekey) + texts = [] # Decoded texts + for encrypted_text in encrypted_texts: + try: + text = CryptMessage.eciesDecrypt(encrypted_text, privatekey).decode("utf8") + texts.append(text) + except Exception as err: + texts.append(None) if type(param) == list: self.response(to, texts) @@ -53,16 +59,23 @@ def actionEciesDecrypt(self, to, param, privatekey=0): # Encrypt a text using AES # Return: Iv, AES key, Encrypted text - def actionAesEncrypt(self, to, text, key=None): + def actionAesEncrypt(self, to, text, key=None, iv=None): + from lib import pyelliptic + if key: key = base64.b64decode(key) else: - key = sslcrypto.aes.new_key() + key = os.urandom(32) + + if iv: # Generate new AES key if not definied + iv = base64.b64decode(iv) + else: + iv = pyelliptic.Cipher.gen_IV('aes-256-cbc') if text: - encrypted, iv = sslcrypto.aes.encrypt(text.encode("utf8"), key) + encrypted = pyelliptic.Cipher(key, iv, 1, ciphername='aes-256-cbc').ciphering(text.encode("utf8")) else: - encrypted, iv = b"", b"" + encrypted = b"" res = [base64.b64encode(item).decode("utf8") for item in [key, iv, encrypted]] self.response(to, res) @@ -70,6 +83,8 @@ def actionAesEncrypt(self, to, text, key=None): # Decrypt a text using AES # Return: Decrypted text def actionAesDecrypt(self, to, *args): + from lib import pyelliptic + if len(args) == 3: # Single decrypt encrypted_texts = [(args[0], args[1])] keys = [args[2]] @@ -82,8 +97,9 @@ def actionAesDecrypt(self, to, *args): iv = base64.b64decode(iv) text = None for key in keys: + ctx = pyelliptic.Cipher(base64.b64decode(key), iv, 0, ciphername='aes-256-cbc') try: - decrypted = sslcrypto.aes.decrypt(encrypted_text, iv, base64.b64decode(key)) + decrypted = ctx.ciphering(encrypted_text) if decrypted and decrypted.decode("utf8"): # Valid text decoded text = decrypted.decode("utf8") except Exception as err: @@ -110,11 +126,12 @@ def actionEcdsaVerify(self, to, data, address, signature): # Gets the publickey of a given privatekey def actionEccPrivToPub(self, to, privatekey): - self.response(to, curve.private_to_public(curve.wif_to_private(privatekey.encode()))) + self.response(to, btctools.privtopub(privatekey)) # Gets the address of a given publickey def actionEccPubToAddr(self, to, publickey): - self.response(to, curve.public_to_address(bytes.fromhex(publickey))) + address = btctools.pubtoaddr(btctools.decode_pubkey(publickey)) + self.response(to, address) @PluginManager.registerTo("User") @@ -149,77 +166,7 @@ def getEncryptPublickey(self, address, param_index=0): index = param_index if "encrypt_publickey_%s" % index not in site_data: - privatekey = self.getEncryptPrivatekey(address, param_index).encode() - publickey = curve.private_to_public(curve.wif_to_private(privatekey) + b"\x01") + privatekey = self.getEncryptPrivatekey(address, param_index) + publickey = btctools.encode_pubkey(btctools.privtopub(privatekey), "bin_compressed") site_data["encrypt_publickey_%s" % index] = base64.b64encode(publickey).decode("utf8") return site_data["encrypt_publickey_%s" % index] - - -@PluginManager.registerTo("Actions") -class ActionsPlugin: - publickey = "A3HatibU4S6eZfIQhVs2u7GLN5G9wXa9WwlkyYIfwYaj" - privatekey = "5JBiKFYBm94EUdbxtnuLi6cvNcPzcKymCUHBDf2B6aq19vvG3rL" - utf8_text = '\xc1rv\xedzt\xfbr\xf5t\xfck\xf6rf\xfar\xf3g\xe9p' - - def getBenchmarkTests(self, online=False): - if hasattr(super(), "getBenchmarkTests"): - tests = super().getBenchmarkTests(online) - else: - tests = [] - - aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode("utf8"), self.publickey) # Warm-up - tests.extend([ - {"func": self.testCryptEciesEncrypt, "kwargs": {}, "num": 100, "time_standard": 1.2}, - {"func": self.testCryptEciesDecrypt, "kwargs": {}, "num": 500, "time_standard": 1.3}, - {"func": self.testCryptEciesDecryptMulti, "kwargs": {}, "num": 5, "time_standard": 0.68}, - {"func": self.testCryptAesEncrypt, "kwargs": {}, "num": 10000, "time_standard": 0.27}, - {"func": self.testCryptAesDecrypt, "kwargs": {}, "num": 10000, "time_standard": 0.25} - ]) - return tests - - def testCryptEciesEncrypt(self, num_run=1): - for i in range(num_run): - aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode("utf8"), self.publickey) - assert len(aes_key) == 32 - yield "." - - def testCryptEciesDecrypt(self, num_run=1): - aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode("utf8"), self.publickey) - for i in range(num_run): - assert len(aes_key) == 32 - decrypted = CryptMessage.eciesDecrypt(base64.b64encode(encrypted), self.privatekey) - assert decrypted == self.utf8_text.encode("utf8"), "%s != %s" % (decrypted, self.utf8_text.encode("utf8")) - yield "." - - def testCryptEciesDecryptMulti(self, num_run=1): - yield "x 100 (%s threads) " % config.threads_crypt - aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode("utf8"), self.publickey) - - threads = [] - for i in range(num_run): - assert len(aes_key) == 32 - threads.append(gevent.spawn( - CryptMessage.eciesDecryptMulti, [base64.b64encode(encrypted)] * 100, self.privatekey - )) - - for thread in threads: - res = thread.get() - assert res[0] == self.utf8_text, "%s != %s" % (res[0], self.utf8_text) - assert res[0] == res[-1], "%s != %s" % (res[0], res[-1]) - yield "." - gevent.joinall(threads) - - def testCryptAesEncrypt(self, num_run=1): - for i in range(num_run): - key = os.urandom(32) - encrypted = sslcrypto.aes.encrypt(self.utf8_text.encode("utf8"), key) - yield "." - - def testCryptAesDecrypt(self, num_run=1): - key = os.urandom(32) - encrypted_text, iv = sslcrypto.aes.encrypt(self.utf8_text.encode("utf8"), key) - - for i in range(num_run): - decrypted = sslcrypto.aes.decrypt(encrypted_text, iv, key).decode("utf8") - assert decrypted == self.utf8_text - yield "." diff --git a/plugins/CryptMessage/Test/TestCrypt.py b/plugins/CryptMessage/Test/TestCrypt.py index 25a077d87..05cc6e445 100644 --- a/plugins/CryptMessage/Test/TestCrypt.py +++ b/plugins/CryptMessage/Test/TestCrypt.py @@ -18,10 +18,13 @@ def testEncryptEcies(self, text, text_repeat): assert len(aes_key) == 32 # assert len(encrypted) == 134 + int(len(text) / 16) * 16 # Not always true - assert CryptMessage.eciesDecrypt(base64.b64encode(encrypted), self.privatekey) == text_repeated + ecc = CryptMessage.getEcc(self.privatekey) + assert ecc.decrypt(encrypted) == text_repeated def testDecryptEcies(self, user): - assert CryptMessage.eciesDecrypt(self.ecies_encrypted_text, self.privatekey) == b"hello" + encrypted = base64.b64decode(self.ecies_encrypted_text) + ecc = CryptMessage.getEcc(self.privatekey) + assert ecc.decrypt(encrypted) == b"hello" def testPublickey(self, ui_websocket): pub = ui_websocket.testAction("UserPublickey", 0) @@ -54,12 +57,12 @@ def testEcies(self, ui_websocket): assert decrypted != "hello" # Decrypt using correct privatekey - decrypted = ui_websocket.testAction("EciesDecrypt", encrypted) + decrypted = ui_websocket.testAction("EciesDecrypt", encrypted) assert decrypted == "hello" # Decrypt incorrect text decrypted = ui_websocket.testAction("EciesDecrypt", "baad") - assert decrypted is None + assert decrypted == None # Decrypt batch decrypted = ui_websocket.testAction("EciesDecrypt", [encrypted, "baad", encrypted]) @@ -87,21 +90,6 @@ def testEciesAes(self, ui_websocket): ui_websocket.actionAesDecrypt(0, base64.b64encode(aes_iv), base64.b64encode(aes_encrypted), aes_key) assert ui_websocket.ws.getResult() == "hello" - def testEciesAesLongpubkey(self, ui_websocket): - privatekey = "5HwVS1bTFnveNk9EeGaRenWS1QFzLFb5kuncNbiY3RiHZrVR6ok" - - ecies_encrypted, aes_key = ["lWiXfEikIjw1ac3J/RaY/gLKACALRUfksc9rXYRFyKDSaxhwcSFBYCgAdIyYlY294g/6VgAf/68PYBVMD3xKH1n7Zbo+ge8b4i/XTKmCZRJvy0eutMKWckYCMVcxgIYNa/ZL1BY1kvvH7omgzg1wBraoLfdbNmVtQgdAZ9XS8PwRy6OB2Q==", "Rvlf7zsMuBFHZIGHcbT1rb4If+YTmsWDv6kGwcvSeMM="] - - # Decrypt using Ecies - ui_websocket.actionEciesDecrypt(0, ecies_encrypted, privatekey) - assert ui_websocket.ws.getResult() == "hello" - - # Decrypt using AES - aes_iv, aes_encrypted = CryptMessage.split(base64.b64decode(ecies_encrypted)) - - ui_websocket.actionAesDecrypt(0, base64.b64encode(aes_iv), base64.b64encode(aes_encrypted), aes_key) - assert ui_websocket.ws.getResult() == "hello" - def testAes(self, ui_websocket): ui_websocket.actionAesEncrypt(0, "hello") key, iv, encrypted = ui_websocket.ws.getResult() diff --git a/plugins/FilePack/FilePackPlugin.py b/plugins/FilePack/FilePackPlugin.py index a095c6d45..840961b7c 100644 --- a/plugins/FilePack/FilePackPlugin.py +++ b/plugins/FilePack/FilePackPlugin.py @@ -21,6 +21,9 @@ def openArchive(archive_path, file_obj=None): if archive_path.endswith("tar.gz"): import tarfile archive_cache[archive_path] = tarfile.open(archive_path, fileobj=file_obj, mode="r:gz") + elif archive_path.endswith("tar.bz2"): + import tarfile + archive_cache[archive_path] = tarfile.open(archive_path, fileobj=file_obj, mode="r:bz2") else: import zipfile archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path) @@ -45,7 +48,7 @@ def actionSiteMedia(self, path, **kwargs): file_obj = None path_parts = self.parsePath(path) file_path = "%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"]) - match = re.match("^(.*\.(?:tar.gz|zip))/(.*)", file_path) + match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path) archive_path, path_within = match.groups() if archive_path not in archive_cache: site = self.server.site_manager.get(path_parts["address"]) @@ -99,7 +102,7 @@ def streamFile(self, file): class SiteStoragePlugin(object): def isFile(self, inner_path): if ".zip/" in inner_path or ".tar.gz/" in inner_path: - match = re.match("^(.*\.(?:tar.gz|zip))/(.*)", inner_path) + match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", inner_path) archive_inner_path, path_within = match.groups() return super(SiteStoragePlugin, self).isFile(archive_inner_path) else: @@ -127,7 +130,7 @@ def openArchive(self, inner_path): def walk(self, inner_path, *args, **kwags): if ".zip" in inner_path or ".tar.gz" in inner_path: - match = re.match("^(.*\.(?:tar.gz|zip))(.*)", inner_path) + match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path) archive_inner_path, path_within = match.groups() archive = self.openArchive(archive_inner_path) path_within = path_within.lstrip("/") @@ -151,7 +154,7 @@ def walk(self, inner_path, *args, **kwags): def list(self, inner_path, *args, **kwags): if ".zip" in inner_path or ".tar.gz" in inner_path: - match = re.match("^(.*\.(?:tar.gz|zip))(.*)", inner_path) + match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path) archive_inner_path, path_within = match.groups() archive = self.openArchive(archive_inner_path) path_within = path_within.lstrip("/") @@ -176,9 +179,9 @@ def list(self, inner_path, *args, **kwags): else: return super(SiteStoragePlugin, self).list(inner_path, *args, **kwags) - def read(self, inner_path, mode="rb", **kwargs): + def read(self, inner_path, mode="rb"): if ".zip/" in inner_path or ".tar.gz/" in inner_path: - match = re.match("^(.*\.(?:tar.gz|zip))(.*)", inner_path) + match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path) archive_inner_path, path_within = match.groups() archive = self.openArchive(archive_inner_path) path_within = path_within.lstrip("/") @@ -189,5 +192,5 @@ def read(self, inner_path, mode="rb", **kwargs): return archive.extractfile(path_within).read() else: - return super(SiteStoragePlugin, self).read(inner_path, mode, **kwargs) + return super(SiteStoragePlugin, self).read(inner_path, mode) diff --git a/plugins/MergerSite/MergerSitePlugin.py b/plugins/MergerSite/MergerSitePlugin.py index 2dccc6dec..ae2b14846 100644 --- a/plugins/MergerSite/MergerSitePlugin.py +++ b/plugins/MergerSite/MergerSitePlugin.py @@ -79,11 +79,8 @@ def actionMergerSiteAdd(self, to, addresses): def cbMergerSiteAdd(self, to, addresses): added = 0 for address in addresses: - try: - site_manager.need(address) - added += 1 - except Exception as err: - self.cmd("notification", ["error", _["Adding %s failed: %s"] % (address, err)]) + added += 1 + site_manager.need(address) if added: self.cmd("notification", ["done", _["Added %s new site"] % added, 5000]) RateLimit.called(self.site.address + "-MergerSiteAdd") @@ -298,6 +295,9 @@ def getDbFiles(self): # Also notice merger sites on a merged site file change def onUpdated(self, inner_path, file=None): + if inner_path == "content.json": + site_manager.updateMergerSites() + super(SiteStoragePlugin, self).onUpdated(inner_path, file) merged_type = merged_db.get(self.site.address) @@ -342,9 +342,9 @@ class SiteManagerPlugin(object): def updateMergerSites(self): global merger_db, merged_db, merged_to_merger, site_manager s = time.time() - merger_db_new = {} - merged_db_new = {} - merged_to_merger_new = {} + merger_db = {} + merged_db = {} + merged_to_merger = {} site_manager = self if not self.sites: return @@ -356,7 +356,7 @@ def updateMergerSites(self): self.log.error("Error loading site %s: %s" % (site.address, Debug.formatException(err))) continue if merged_type: - merged_db_new[site.address] = merged_type + merged_db[site.address] = merged_type # Update merger sites for permission in site.settings["permissions"]: @@ -370,30 +370,24 @@ def updateMergerSites(self): site.settings["permissions"].remove(permission) continue merger_type = permission.replace("Merger:", "") - if site.address not in merger_db_new: - merger_db_new[site.address] = [] - merger_db_new[site.address].append(merger_type) + if site.address not in merger_db: + merger_db[site.address] = [] + merger_db[site.address].append(merger_type) site_manager.sites[site.address] = site # Update merged to merger if merged_type: for merger_site in self.sites.values(): if "Merger:" + merged_type in merger_site.settings["permissions"]: - if site.address not in merged_to_merger_new: - merged_to_merger_new[site.address] = [] - merged_to_merger_new[site.address].append(merger_site) - - # Update globals - merger_db = merger_db_new - merged_db = merged_db_new - merged_to_merger = merged_to_merger_new - + if site.address not in merged_to_merger: + merged_to_merger[site.address] = [] + merged_to_merger[site.address].append(merger_site) self.log.debug("Updated merger sites in %.3fs" % (time.time() - s)) def load(self, *args, **kwags): super(SiteManagerPlugin, self).load(*args, **kwags) self.updateMergerSites() - def saveDelayed(self, *args, **kwags): - super(SiteManagerPlugin, self).saveDelayed(*args, **kwags) + def save(self, *args, **kwags): + super(SiteManagerPlugin, self).save(*args, **kwags) self.updateMergerSites() diff --git a/plugins/OptionalManager/ContentDbPlugin.py b/plugins/OptionalManager/ContentDbPlugin.py index f0f8a877d..e7945d93e 100644 --- a/plugins/OptionalManager/ContentDbPlugin.py +++ b/plugins/OptionalManager/ContentDbPlugin.py @@ -24,8 +24,8 @@ def __init__(self, *args, **kwargs): self.time_peer_numbers_updated = 0 self.my_optional_files = {} # Last 50 site_address/inner_path called by fileWrite (auto-pinning these files) self.optional_files = collections.defaultdict(dict) - self.optional_files_loaded = False - self.timer_check_optional = helper.timer(60 * 5, self.checkOptionalLimit) + self.optional_files_loading = False + helper.timer(60 * 5, self.checkOptionalLimit) super(ContentDbPlugin, self).__init__(*args, **kwargs) def getSchema(self): @@ -60,6 +60,9 @@ def initSite(self, site): super(ContentDbPlugin, self).initSite(site) if self.need_filling: self.fillTableFileOptional(site) + if not self.optional_files_loading: + gevent.spawn_later(1, self.loadFilesOptional) + self.optional_files_loading = True def checkTables(self): changed_tables = super(ContentDbPlugin, self).checkTables() @@ -88,7 +91,7 @@ def loadFilesOptional(self): site_ids_reverse = {val: key for key, val in self.site_ids.items()} for site_id, stats in site_sizes.items(): site_address = site_ids_reverse.get(site_id) - if not site_address or site_address not in self.sites: + if not site_address: self.log.error("Not found site_id: %s" % site_id) continue site = self.sites[site_address] @@ -97,7 +100,7 @@ def loadFilesOptional(self): total += stats["size_optional"] total_downloaded += stats["optional_downloaded"] - self.log.info( + self.log.debug( "Loaded %s optional files: %.2fMB, downloaded: %.2fMB in %.3fs" % (num, float(total) / 1024 / 1024, float(total_downloaded) / 1024 / 1024, time.time() - s) ) @@ -105,7 +108,7 @@ def loadFilesOptional(self): if self.need_filling and self.getOptionalLimitBytes() >= 0 and self.getOptionalLimitBytes() < total_downloaded: limit_bytes = self.getOptionalLimitBytes() limit_new = round((float(total_downloaded) / 1024 / 1024 / 1024) * 1.1, 2) # Current limit + 10% - self.log.info( + self.log.debug( "First startup after update and limit is smaller than downloaded files size (%.2fGB), increasing it from %.2fGB to %.2fGB" % (float(total_downloaded) / 1024 / 1024 / 1024, float(limit_bytes) / 1024 / 1024 / 1024, limit_new) ) @@ -139,14 +142,14 @@ def fillTableFileOptional(self, site): if not user: user = UserManager.user_manager.create() auth_address = user.getAuthAddress(site.address) - res = self.execute( + self.execute( "UPDATE file_optional SET is_pinned = 1 WHERE site_id = :site_id AND inner_path LIKE :inner_path", {"site_id": site_id, "inner_path": "%%/%s/%%" % auth_address} ) self.log.debug( "Filled file_optional table for %s in %.3fs (loaded: %s, is_pinned: %s)" % - (site.address, time.time() - s, num, res.rowcount) + (site.address, time.time() - s, num, self.cur.cursor.rowcount) ) self.filled[site.address] = True @@ -402,13 +405,3 @@ def checkOptionalLimit(self, limit=None): for file_id in deleted_file_ids: cur.execute("UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?", {"file_id": file_id}) cur.close() - - -@PluginManager.registerTo("SiteManager") -class SiteManagerPlugin(object): - def load(self, *args, **kwargs): - back = super(SiteManagerPlugin, self).load(*args, **kwargs) - if self.sites and not content_db.optional_files_loaded and content_db.conn: - content_db.optional_files_loaded = True - content_db.loadFilesOptional() - return back \ No newline at end of file diff --git a/plugins/OptionalManager/OptionalManagerPlugin.py b/plugins/OptionalManager/OptionalManagerPlugin.py index f01fab652..909caa316 100644 --- a/plugins/OptionalManager/OptionalManagerPlugin.py +++ b/plugins/OptionalManager/OptionalManagerPlugin.py @@ -17,46 +17,32 @@ def importPluginnedClasses(): def processAccessLog(): - global access_log if access_log: content_db = ContentDbPlugin.content_db - if not content_db.conn: - return False - - s = time.time() - access_log_prev = access_log - access_log = collections.defaultdict(dict) now = int(time.time()) num = 0 - for site_id in access_log_prev: + for site_id in access_log: content_db.execute( "UPDATE file_optional SET time_accessed = %s WHERE ?" % now, - {"site_id": site_id, "inner_path": list(access_log_prev[site_id].keys())} + {"site_id": site_id, "inner_path": list(access_log[site_id].keys())} ) - num += len(access_log_prev[site_id]) - - content_db.log.debug("Inserted %s web request stat in %.3fs" % (num, time.time() - s)) + num += len(access_log[site_id]) + access_log.clear() def processRequestLog(): - global request_log if request_log: content_db = ContentDbPlugin.content_db - if not content_db.conn: - return False - - s = time.time() - request_log_prev = request_log - request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}} + cur = content_db.getCursor() num = 0 - for site_id in request_log_prev: - for inner_path, uploaded in request_log_prev[site_id].items(): + for site_id in request_log: + for inner_path, uploaded in request_log[site_id].items(): content_db.execute( "UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded, {"site_id": site_id, "inner_path": inner_path} ) num += 1 - content_db.log.debug("Inserted %s file request stat in %.3fs" % (num, time.time() - s)) + request_log.clear() if "access_log" not in locals().keys(): # To keep between module reloads @@ -86,12 +72,12 @@ def optionalDownloaded(self, inner_path, hash_id, size=None, own=False): return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own) def optionalRemoved(self, inner_path, hash_id, size=None): - res = self.contents.db.execute( + self.contents.db.execute( "UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 1", {"site_id": self.contents.db.site_ids[self.site.address], "inner_path": inner_path} ) - if res.rowcount > 0: + if self.contents.db.cur.cursor.rowcount > 0: back = super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size) # Re-add to hashfield if we have other file with the same hash_id if self.isDownloaded(hash_id=hash_id, force_check_db=True): diff --git a/plugins/OptionalManager/UiWebsocketPlugin.py b/plugins/OptionalManager/UiWebsocketPlugin.py index 0acc53cf7..103bbe848 100644 --- a/plugins/OptionalManager/UiWebsocketPlugin.py +++ b/plugins/OptionalManager/UiWebsocketPlugin.py @@ -139,7 +139,7 @@ def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC wheres = {} wheres_raw = [] if "bigfile" in filter: - wheres["size >"] = 1024 * 1024 * 1 + wheres["size >"] = 1024 * 1024 * 10 if "downloaded" in filter: wheres_raw.append("(is_downloaded = 1 OR is_pinned = 1)") if "pinned" in filter: @@ -166,14 +166,11 @@ def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC row["address"] = address if row["size"] > 1024 * 1024: - has_bigfile_info = self.addBigfileInfo(row) + has_info = self.addBigfileInfo(row) else: - has_bigfile_info = False + has_info = False - if not has_bigfile_info and "bigfile" in filter: - continue - - if not has_bigfile_info: + if not has_info: if row["is_downloaded"]: row["bytes_downloaded"] = row["size"] row["downloaded_percent"] = 100 diff --git a/plugins/PeerDb/PeerDbPlugin.py b/plugins/PeerDb/PeerDbPlugin.py index a66b81cfb..b4c8787b6 100644 --- a/plugins/PeerDb/PeerDbPlugin.py +++ b/plugins/PeerDb/PeerDbPlugin.py @@ -59,7 +59,7 @@ def loadPeers(self, site): def iteratePeers(self, site): site_id = self.site_ids.get(site.address) - for key, peer in list(site.peers.items()): + for key, peer in site.peers.items(): address, port = key.rsplit(":", 1) if peer.has_hashfield: hashfield = sqlite3.Binary(peer.hashfield.tobytes()) @@ -70,7 +70,7 @@ def iteratePeers(self, site): def savePeers(self, site, spawn=False): if spawn: # Save peers every hour (+random some secs to not update very site at same time) - site.greenlet_manager.spawnLater(60 * 60 + random.randint(0, 60), self.savePeers, site, spawn=True) + gevent.spawn_later(60 * 60 + random.randint(0, 60), self.savePeers, site, spawn=True) if not site.peers: site.log.debug("Peers not saved: No peers found") return @@ -79,7 +79,7 @@ def savePeers(self, site, spawn=False): cur = self.getCursor() try: cur.execute("DELETE FROM peer WHERE site_id = :site_id", {"site_id": site_id}) - cur.executemany( + cur.cursor.executemany( "INSERT INTO peer (site_id, address, port, hashfield, reputation, time_added, time_found) VALUES (?, ?, ?, ?, ?, ?, ?)", self.iteratePeers(site) ) @@ -89,8 +89,8 @@ def savePeers(self, site, spawn=False): def initSite(self, site): super(ContentDbPlugin, self).initSite(site) - site.greenlet_manager.spawnLater(0.5, self.loadPeers, site) - site.greenlet_manager.spawnLater(60*60, self.savePeers, site, spawn=True) + gevent.spawn_later(0.5, self.loadPeers, site) + gevent.spawn_later(60*60, self.savePeers, site, spawn=True) def saveAllPeers(self): for site in list(self.sites.values()): diff --git a/plugins/Sidebar/ConsolePlugin.py b/plugins/Sidebar/ConsolePlugin.py index 15f6a1bac..30d00fee4 100644 --- a/plugins/Sidebar/ConsolePlugin.py +++ b/plugins/Sidebar/ConsolePlugin.py @@ -58,16 +58,9 @@ def actionConsoleLogRead(self, to, filter=None, read_size=32 * 1024, limit=500): assert SafeRe.isSafePattern(filter) filter_re = re.compile(".*" + filter) - last_match = False for line in log_file: - if not line.startswith("[") and last_match: # Multi-line log entry - lines.append(line.replace(" ", " ")) - continue - if filter and not filter_re.match(line): - last_match = False continue - last_match = True lines.append(line) num_found = len(lines) diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index 4ecca75a1..c4b3a4bb2 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -11,7 +11,6 @@ import gevent -import util from Config import config from Plugin import PluginManager from Debug import Debug @@ -117,7 +116,7 @@ def sidebarRenderPeerStats(self, body, site): peer_ips = [peer.key for peer in site.getConnectablePeers(20, allow_private=False)] peer_ips.sort(key=lambda peer_ip: ".onion:" in peer_ip) copy_link = "http://127.0.0.1:43110/%s/?zeronet_peers=%s" % ( - site.content_manager.contents.get("content.json", {}).get("domain", site.address), + site.content_manager.contents["content.json"].get("domain", site.address), ",".join(peer_ips) ) @@ -173,10 +172,8 @@ def sidebarRenderFileStats(self, body, site):