From e3abc1e3c64d31aae1a3ed87807de86e1906bf38 Mon Sep 17 00:00:00 2001 From: faza Date: Tue, 17 Nov 2020 01:55:11 +0530 Subject: [PATCH 1/5] Optimize object collection --- bin/helper.js | 45 ++++---- bin/index.mjs | 2 +- bin/lib/dgit.js | 61 ---------- bin/lib/git.js | 34 ++---- package-lock.json | 279 ++++++++++++++++++---------------------------- package.json | 8 +- 6 files changed, 150 insertions(+), 279 deletions(-) delete mode 100644 bin/lib/dgit.js diff --git a/bin/helper.js b/bin/helper.js index 8cc1c7e..04b3992 100644 --- a/bin/helper.js +++ b/bin/helper.js @@ -4,7 +4,6 @@ import ora from "ora"; import path from "path"; import shell from "shelljs"; import GitHelper from "./lib/git.js"; -import DGitHelper from "./lib/dgit.js"; import LineHelper from "./lib/line.js"; import Arweave from "arweave"; import { @@ -64,7 +63,6 @@ export default class Helper { this.debug = debug("gitopia"); this.line = new LineHelper(); this.git = new GitHelper(this); - this.dgit = new DGitHelper(this); } // OK @@ -253,7 +251,7 @@ export default class Helper { let spinner; let txHash; let mapping = {}; - const puts = []; + let dataItems = []; const pins = []; try { @@ -306,38 +304,45 @@ export default class Helper { this._die(); } - // update ref - puts.push( - makeUpdateRefDataItem(this.ArData, this.wallet, this.url, dst, srcOid) - ); - - const bar1 = newProgressBar(); - // collect git objects console.error("Collecting git objects [this may take a while]"); + const bar1 = newProgressBar(); bar1.start(objects.length, 0); try { for (const oid of objects) { const object = await this.git.load(oid); - bar1.increment(); - - puts.push( - makeDataItem(this.ArData, this.wallet, this.url, oid, object) + const dataItem = await makeDataItem( + this.ArData, + this.wallet, + this.url, + oid, + object ); + dataItems.push(dataItem); + + bar1.increment(); } - // spinner.succeed("Git objects collected"); + // update ref + dataItems.push( + await makeUpdateRefDataItem( + this.ArData, + this.wallet, + this.url, + dst, + srcOid + ) + ); + + bar1.stop(); + console.error("Git objects collected successfully"); } catch (err) { - // spinner.fail("Failed to collect git objects: " + err.message); + console.error("Failed to collect git objects: " + err.message); this._die(); } - const dataItems = await Promise.all(puts); - bar1.stop(); - console.error("Git objects collected successfully"); - // upload git objects try { console.error( diff --git a/bin/index.mjs b/bin/index.mjs index 396a3cf..63f9f70 100755 --- a/bin/index.mjs +++ b/bin/index.mjs @@ -1,4 +1,4 @@ -#!/usr/bin/env node +#!/usr/bin/env -S node --max-old-space-size=16384 import axios from "axios"; import Helper, { VERSION } from "./helper.js"; diff --git a/bin/lib/dgit.js b/bin/lib/dgit.js deleted file mode 100644 index bf8b351..0000000 --- a/bin/lib/dgit.js +++ /dev/null @@ -1,61 +0,0 @@ -import CID from "cids"; -import IPLDGit from "ipld-git"; -// tslint:disable-next-line:no-submodule-imports -import pkg from "ipld-git/src/util/util.js"; -const { cidToSha, shaToCid } = pkg; -import URL from "url-parse"; - -export default class DGitHelper { - // OK - constructor(helper) { - this.helper = helper; - } - - // OK - async deserialize(buffer) { - return new ((resolve, reject) => { - IPLDGit.util.deserialize(buffer, (err, node) => { - if (err) { - reject(err); - } else { - resolve(node); - } - }); - })(); - } - - // OK - async serialize(node) { - return new ((resolve, reject) => { - IPLDGit.util.serialize(node, async (err, buffer) => { - if (err) { - reject(err); - } else { - resolve(buffer); - } - }); - })(); - } - - // OK - async put(object) {} - - // OK - async get(cid) {} - - // OK - async pin(cid) {} - - // OK - async cid(object) {} - - // OK - shaToCid(oid) { - return new CID(shaToCid(Buffer.from(oid, "hex"))).toBaseEncodedString(); - } - - // OK - cidToSha(cid) { - return cidToSha(new CID(cid)).toString("hex"); - } -} diff --git a/bin/lib/git.js b/bin/lib/git.js index 41c14bf..1fe2f52 100644 --- a/bin/lib/git.js +++ b/bin/lib/git.js @@ -1,16 +1,12 @@ import debug from "debug"; import fs from "fs-extra"; import npath from "path"; -import shell from "shelljs"; -// tslint:disable-next-line:no-submodule-imports -import gitP from "simple-git/promise.js"; -import pkg from "smart-buffer"; -const { SmartBuffer } = pkg; -import zlib from "zlib"; +import pkg from "isomorphic-git"; +const { readObject } = pkg; import { fetchGitObjects } from "./graphql.js"; -const git = gitP(); +const cache = {}; export default class GitHelper { // OK @@ -58,21 +54,15 @@ export default class GitHelper { // OK async load(oid) { - const type = shell - .exec(`git cat-file -t ${oid}`, { silent: true }) - .stdout.trim(); - const size = shell - .exec(`git cat-file -s ${oid}`, { silent: true }) - .stdout.trim(); - const data = await git.binaryCatFile([type, oid]); - - const raw = new SmartBuffer(); - raw.writeString(`${type} `); - raw.writeString(size); - raw.writeUInt8(0); - raw.writeBuffer(data); - - return zlib.deflateSync(raw.toBuffer()); + const { object } = await readObject({ + fs, + gitdir: this.helper.path, + oid, + cache, + format: "deflated", + }); + + return object; } // OK diff --git a/package-lock.json b/package-lock.json index 2b27e6c..a228480 100644 --- a/package-lock.json +++ b/package-lock.json @@ -97,13 +97,10 @@ "safer-buffer": "^2.1.0" } }, - "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "requires": { - "lodash": "^4.17.14" - } + "async-lock": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.2.4.tgz", + "integrity": "sha512-UBQJC2pbeyGutIfYmErGc9RaJYnpZ1FHaxuKwb0ahvGiiCkPUf3p67Io+YLPmmv3RHY+mF6JEtNW8FlHsraAaA==" }, "axios": { "version": "0.21.0", @@ -118,14 +115,6 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, - "base-x": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.8.tgz", - "integrity": "sha512-Rl/1AWP4J/zRrk54hhlxH4drNxPJXYUaKffODVI53/dAsV4t9fBxyxYKAVPU1XBHxYwOWP9h9H0hM2MVw4YfJA==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, "base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", @@ -136,11 +125,6 @@ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==" }, - "blakejs": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.1.0.tgz", - "integrity": "sha1-ad+S75U6qIylGjLfarHFShVfx6U=" - }, "bn.js": { "version": "4.11.9", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", @@ -179,21 +163,10 @@ "supports-color": "^5.3.0" } }, - "cids": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/cids/-/cids-0.5.7.tgz", - "integrity": "sha512-SlAz4p8XMEW3mhwiYbzfjn+5+Y//+kIuHqzRUytK0a3uGBnsjJb76xHliehv0HcVMCjRKv2vZnPTwd4QX+IcMA==", - "requires": { - "class-is": "^1.1.0", - "multibase": "~0.6.0", - "multicodec": "~0.2.7", - "multihashes": "~0.4.14" - } - }, - "class-is": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/class-is/-/class-is-1.1.0.tgz", - "integrity": "sha512-rhjH9AG1fvabIDoGRVH587413LPjTZgmDF9fOFCbFJQV4yuocX1mHxxvXI4g3cGwbVY9wAYIoKlg1N79frJKQw==" + "clean-git-ref": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/clean-git-ref/-/clean-git-ref-2.0.1.tgz", + "integrity": "sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw==" }, "cli-cursor": { "version": "2.1.0", @@ -270,6 +243,15 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "crc-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz", + "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==", + "requires": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + } + }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -283,6 +265,14 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, "defaults": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", @@ -314,6 +304,11 @@ } } }, + "diff3": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/diff3/-/diff3-0.0.3.tgz", + "integrity": "sha1-1OXDpM305f4SEatC5pP8tDIVgPw=" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -357,6 +352,11 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "exit-on-epipe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz", + "integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==" + }, "find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", @@ -429,6 +429,11 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==" + }, "immediate": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.2.3.tgz", @@ -453,49 +458,33 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==" }, - "ipld-git": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/ipld-git/-/ipld-git-0.2.3.tgz", - "integrity": "sha512-AbIlgbK0vgLqQ/U+kJmnLnK2+uxaMS6PdP/PBK5heBQII3Lb2IpYoMqdJja5wYISTYxGzJeh9PB2U/de3B0ucw==", - "requires": { - "async": "^2.6.0", - "cids": "~0.5.2", - "multicodec": "~0.4.0", - "multihashes": "~0.4.12", - "multihashing-async": "~0.5.1", - "smart-buffer": "^4.0.0", - "traverse": "~0.6.6" - }, - "dependencies": { - "multicodec": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/multicodec/-/multicodec-0.4.0.tgz", - "integrity": "sha512-npzvuOHRJD172WiolvyF8mAS5JfWWJlVh22OcGZ4I5ZKyFVI4aFQYB5AcMtmMTWRsjiePlOHH/dnaZi8ZidWuA==", - "requires": { - "varint": "^5.0.0" - } - } - } - }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, - "is-promise": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-1.0.1.tgz", - "integrity": "sha1-MVc3YcBX4zwukaq56W2gjO++duU=" - }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, - "js-sha3": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", - "integrity": "sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==" + "isomorphic-git": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/isomorphic-git/-/isomorphic-git-1.8.0.tgz", + "integrity": "sha512-TWJvQh+++eFrEG0IFS/jLhMwsBoCOX1/Dsw9q8no59Mp1K0jEjSHXFWv2P04PwkxcIpePkXVBI5YFcFT2nkuQg==", + "requires": { + "async-lock": "^1.1.0", + "clean-git-ref": "^2.0.1", + "crc-32": "^1.2.0", + "diff3": "0.0.3", + "ignore": "^5.1.4", + "minimisted": "^2.0.0", + "pako": "^1.0.10", + "pify": "^4.0.1", + "readable-stream": "^3.4.0", + "sha.js": "^2.4.9", + "simple-get": "^3.0.2" + } }, "jsonfile": { "version": "5.0.0", @@ -621,11 +610,6 @@ "p-locate": "^4.1.0" } }, - "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, "log-symbols": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", @@ -649,6 +633,11 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, "minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", @@ -662,65 +651,23 @@ "brace-expansion": "^1.1.7" } }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, - "multibase": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/multibase/-/multibase-0.6.1.tgz", - "integrity": "sha512-pFfAwyTjbbQgNc3G7D48JkJxWtoJoBMaR4xQUOuB8RnCgRqaYmWNFeJTTvrJ2w51bjLq2zTby6Rqj9TQ9elSUw==", - "requires": { - "base-x": "^3.0.8", - "buffer": "^5.5.0" - } - }, - "multicodec": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/multicodec/-/multicodec-0.2.7.tgz", - "integrity": "sha512-96xc9zs7bsclMW0Po9ERnRFqcsWHY8OZ8JW/I8DeHG58YYJZy3cBGI00Ze7hz9Ix56DNHMTSxEj9cgoZByruMg==", - "requires": { - "varint": "^5.0.0" - } - }, - "multihashes": { - "version": "0.4.21", - "resolved": "https://registry.npmjs.org/multihashes/-/multihashes-0.4.21.tgz", - "integrity": "sha512-uVSvmeCWf36pU2nB4/1kzYZjsXD9vofZKpgudqkceYY5g2aZZXJ5r9lxuzoRLl1OAp28XljXsEJ/X/85ZsKmKw==", - "requires": { - "buffer": "^5.5.0", - "multibase": "^0.7.0", - "varint": "^5.0.0" - }, - "dependencies": { - "multibase": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/multibase/-/multibase-0.7.0.tgz", - "integrity": "sha512-TW8q03O0f6PNFTQDvh3xxH03c8CjGaaYrjkl9UQPG6rz53TQzzxJVCIWVjzcbN/Q5Y53Zd0IBQBMVktVgNx4Fg==", - "requires": { - "base-x": "^3.0.8", - "buffer": "^5.5.0" - } - } - } - }, - "multihashing-async": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/multihashing-async/-/multihashing-async-0.5.2.tgz", - "integrity": "sha512-mmyG6M/FKxrpBh9xQDUvuJ7BbqT93ZeEeH5X6LeMYKoYshYLr9BDdCsvDtZvn+Egf+/Xi+aOznrWL4vp3s+p0Q==", + "minimisted": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/minimisted/-/minimisted-2.0.1.tgz", + "integrity": "sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA==", "requires": { - "blakejs": "^1.1.0", - "js-sha3": "~0.8.0", - "multihashes": "~0.4.13", - "murmurhash3js": "^3.0.1", - "nodeify": "^1.0.1" + "minimist": "^1.2.5" } }, - "murmurhash3js": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/murmurhash3js/-/murmurhash3js-3.0.1.tgz", - "integrity": "sha1-Ppg+W0fCoG9DpxMXTn5DXKBEuZg=" + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "napi-macros": { "version": "2.0.0", @@ -732,15 +679,6 @@ "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" }, - "nodeify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/nodeify/-/nodeify-1.0.1.tgz", - "integrity": "sha1-ZKtpp7268DzhB7TwM1yHwLnpGx0=", - "requires": { - "is-promise": "~1.0.0", - "promise": "~1.3.0" - } - }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -796,6 +734,11 @@ "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -811,13 +754,15 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, - "promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/promise/-/promise-1.3.0.tgz", - "integrity": "sha1-5cyaTIJ45GZP/twBx9qEhCsEAXU=", - "requires": { - "is-promise": "~1" - } + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, + "printj": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz", + "integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==" }, "prr": { "version": "1.0.1", @@ -894,6 +839,15 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, + "sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "shelljs": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", @@ -909,19 +863,21 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, - "simple-git": { - "version": "1.132.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-1.132.0.tgz", - "integrity": "sha512-xauHm1YqCTom1sC9eOjfq3/9RKiUA9iPnxBbrY2DdL8l4ADMu0jjM5l5lphQP5YWNqAL2aXC/OeuQ76vHtW5fg==", + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" + }, + "simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", "requires": { - "debug": "^4.0.1" + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" } }, - "smart-buffer": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz", - "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw==" - }, "smartweave": { "version": "github:TheTechTrap/SmartWeave#0fa009545ea906b824c1ef567185b6df9a9d7575", "from": "github:TheTechTrap/SmartWeave#v0.4.0", @@ -989,11 +945,6 @@ "has-flag": "^3.0.0" } }, - "traverse": { - "version": "0.6.6", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz", - "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=" - }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", @@ -1021,11 +972,6 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, - "varint": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.0.tgz", - "integrity": "sha1-2Ca4n3SQcy+rwMDtaT7Uddyynr8=" - }, "wcwidth": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", @@ -1126,11 +1072,6 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } - }, - "zlib": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz", - "integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA=" } } } diff --git a/package.json b/package.json index 1ccca6e..c207897 100644 --- a/package.json +++ b/package.json @@ -12,20 +12,16 @@ "arweave-bundles": "^1.0.0", "axios": "^0.21.0", "bignumber.js": "^9.0.1", - "cids": "^0.5.7", "cli-progress": "^3.8.2", "debug": "^4.1.1", "fs-extra": "^7.0.1", - "ipld-git": "^0.2.3", + "isomorphic-git": "^1.8.0", "jsonfile": "^5.0.0", "level": "^5.0.0", "ora": "^3.1.0", "shelljs": "^0.8.3", - "simple-git": "^1.107.0", - "smart-buffer": "^4.0.2", "smartweave": "github:TheTechTrap/SmartWeave#v0.4.0", - "url-parse": "^1.4.4", - "zlib": "^1.0.5" + "url-parse": "^1.4.4" }, "devDependencies": {}, "scripts": { From 87d9477a95d0d49d0c343e6e4a79c70a08ed899f Mon Sep 17 00:00:00 2001 From: faza Date: Thu, 19 Nov 2020 15:33:24 +0530 Subject: [PATCH 2/5] feat: Chunk objects bundle --- bin/helper.js | 83 +++++++++++++++++++++++++++++++++------------ bin/lib/arweave.js | 84 +++++++++++++++++++++++++++++----------------- 2 files changed, 114 insertions(+), 53 deletions(-) diff --git a/bin/helper.js b/bin/helper.js index 04b3992..6f27cad 100644 --- a/bin/helper.js +++ b/bin/helper.js @@ -8,9 +8,11 @@ import LineHelper from "./lib/line.js"; import Arweave from "arweave"; import { makeDataItem, - makeUpdateRefDataItem, + makeUpdateRefTx, parseArgitRemoteURI, - postBundledTransaction, + makeBundledDataTx, + postTransaction, + sendPSTFee, } from "./lib/arweave.js"; import { getAllRefs } from "./lib/graphql.js"; import { newProgressBar } from "./lib/util.js"; @@ -20,6 +22,8 @@ import ArweaveBundles from "arweave-bundles"; export const VERSION = "0.1.7"; +const CHUNK_SIZE = 128 * 1024 * 1024; // 128Mb + const _timeout = async (duration) => { return new Promise((resolve, reject) => { setTimeout(() => { @@ -249,10 +253,7 @@ export default class Helper { return (async (resolve, reject) => { let spinner; - let txHash; - let mapping = {}; - let dataItems = []; - const pins = []; + let bundledDataTxs = []; try { const refs = await this._fetchRefs(); @@ -310,9 +311,14 @@ export default class Helper { const bar1 = newProgressBar(); bar1.start(objects.length, 0); + let currentChunk = []; + let currentChunkSize = 0; + try { - for (const oid of objects) { + for (let i = 0; i < objects.length; i++) { + const oid = objects[i]; const object = await this.git.load(oid); + const dataItem = await makeDataItem( this.ArData, this.wallet, @@ -320,23 +326,34 @@ export default class Helper { oid, object ); - dataItems.push(dataItem); + + currentChunkSize += Buffer.byteLength( + JSON.stringify(dataItem), + "utf8" + ); + currentChunk.push(dataItem); + + if (currentChunkSize >= CHUNK_SIZE || i === objects.length - 1) { + const bundledDataTx = await makeBundledDataTx( + this._arweave, + this.ArData, + this.wallet, + this.url, + currentChunk + ); + + // Reset chunk + currentChunkSize = 0; + currentChunk = []; + + bundledDataTxs.push(bundledDataTx); + } bar1.increment(); } - // update ref - dataItems.push( - await makeUpdateRefDataItem( - this.ArData, - this.wallet, - this.url, - dst, - srcOid - ) - ); - bar1.stop(); + console.error("Git objects collected successfully"); } catch (err) { console.error("Failed to collect git objects: " + err.message); @@ -348,13 +365,35 @@ export default class Helper { console.error( "Uploading git objects to Gitopia [this may take a while]" ); - await postBundledTransaction( + + // Git object bundles + await Promise.all( + bundledDataTxs.map( + async (bundleDataTx) => + await postTransaction(this._arweave, bundleDataTx) + ) + ); + + // update ref + const updateRefTx = await makeUpdateRefTx( this._arweave, - this.ArData, this.wallet, this.url, - dataItems + dst, + srcOid, + bundledDataTxs ); + await postTransaction(this._arweave, updateRefTx); + + // PST Fee + await sendPSTFee( + this._arweave, + this.wallet, + this.url, + bundledDataTxs, + updateRefTx.id + ); + console.error("Git objects uploaded to Gitopia successfully"); } catch (err) { spinner.fail( diff --git a/bin/lib/arweave.js b/bin/lib/arweave.js index 9f31d7a..9067b65 100644 --- a/bin/lib/arweave.js +++ b/bin/lib/arweave.js @@ -18,27 +18,15 @@ export function parseArgitRemoteURI(remoteURI) { return { repoOwnerAddress, repoName }; } -export async function makeUpdateRefDataItem( - arData, +export async function makeUpdateRefTx( + arweave, wallet, remoteURI, ref, - oid + oid, + bundledDataTxs ) { const { repoName } = parseArgitRemoteURI(remoteURI); - const tags = [ - { name: "Repo", value: repoName }, - { name: "Version", value: "0.0.2" }, - { name: "Ref", value: ref }, - { name: "Type", value: "update-ref" }, - { name: "App-Name", value: "Gitopia" }, - { - name: "Unix-Time", - value: Math.round(new Date().getTime() / 1000).toString(), - }, - { name: "Content-Type", value: "application/json" }, - ]; - const numCommits = shell .exec(`git rev-list --count ${ref}`, { silent: true }) .stdout.trim(); @@ -48,8 +36,33 @@ export async function makeUpdateRefDataItem( }; const data = JSON.stringify(obj); - const item = await arData.createData({ data, tags }, wallet); - return await arData.sign(item, wallet); + const tx = await arweave.createTransaction({ data }, wallet); + + tx.addTag("Repo", repoName); + tx.addTag("Version", "0.0.2"); + tx.addTag("Ref", ref); + tx.addTag("Type", "update-ref"); + tx.addTag("App-Name", "Gitopia"); + tx.addTag("Unix-Time", Math.round(new Date().getTime() / 1000).toString()); + tx.addTag("Content-Type", "application/json"); + tx.addTag("Helper", VERSION); + + // Push triggered from gitopia mirror action + if (process.env.GITHUB_SHA) { + tx.addTag("Origin", "gitopia-mirror-action"); + } else { + tx.addTag("Origin", "git-remote-gitopia"); + } + + const bundledDataTxIds = []; + for (let i = 0; i < bundledDataTxs.length; i++) { + bundledDataTxIds.push(bundledDataTxs[i].id); + } + + tx.addTag("Reference-Txs", JSON.stringify(bundledDataTxIds)); + + await arweave.transactions.sign(tx, wallet); + return tx; } export const makeDataItem = async ( @@ -77,7 +90,7 @@ export const makeDataItem = async ( return await arData.sign(item, wallet); }; -export const postBundledTransaction = async ( +export const makeBundledDataTx = async ( arweave, arData, wallet, @@ -95,17 +108,13 @@ export const postBundledTransaction = async ( tx.addTag("Bundle-Format", "json"); tx.addTag("Bundle-Version", "1.0.0"); tx.addTag("Content-Type", "application/json"); - tx.addTag("Helper", VERSION); tx.addTag("Unix-Time", Math.round(new Date().getTime() / 1000).toString()); - // Push triggered from gitopia mirror action - if (process.env.GITHUB_SHA) { - tx.addTag("Origin", "gitopia-mirror-action"); - } else { - tx.addTag("Origin", "git-remote-gitopia"); - } - await arweave.transactions.sign(tx, wallet); + return tx; +}; + +export const postTransaction = async (arweave, tx) => { const uploader = await arweave.transactions.getUploader(tx); const bar = newProgressBar(); @@ -117,8 +126,16 @@ export const postBundledTransaction = async ( } bar.stop(); +}; - // Send fee to PST holders +export const sendPSTFee = async ( + arweave, + wallet, + remoteURI, + transactions, + referenceId +) => { + const { repoName } = parseArgitRemoteURI(remoteURI); const contractState = await smartweave.default.readContract( arweave, contractId @@ -128,8 +145,13 @@ export const postBundledTransaction = async ( ); // PST Fee - const txFee = new BigNumber(tx.reward); - const pstFee = txFee.multipliedBy(0.1); + let totalTxFee = new BigNumber(0); + for (let i = 0; i < transactions.length; i++) { + const txFee = new BigNumber(transactions[i].reward); + totalTxFee = totalTxFee.plus(txFee); + } + + const pstFee = totalTxFee.multipliedBy(0.1); const quantity = pstFee.isGreaterThan( BigNumber(arweave.ar.arToWinston("0.01")) @@ -141,7 +163,7 @@ export const postBundledTransaction = async ( { target: holder, quantity }, wallet ); - pstTx.addTag("Bundle-Txid", tx.id); + pstTx.addTag("Reference-Id", referenceId); pstTx.addTag("Repo", repoName); pstTx.addTag("Version", "0.0.2"); pstTx.addTag("App-Name", "Gitopia"); From 120559d9438c9cfd3295d48d83bf26165e761bb8 Mon Sep 17 00:00:00 2001 From: faza Date: Mon, 7 Dec 2020 19:14:49 +0530 Subject: [PATCH 3/5] Wait for tx propogation --- bin/helper.js | 65 ++++++++++++++++++----------- bin/lib/arweave.js | 102 ++++++++++++++++++++++++++++++++------------- bin/lib/graphql.js | 18 +------- 3 files changed, 115 insertions(+), 70 deletions(-) diff --git a/bin/helper.js b/bin/helper.js index 6f27cad..08269d4 100644 --- a/bin/helper.js +++ b/bin/helper.js @@ -12,6 +12,7 @@ import { parseArgitRemoteURI, makeBundledDataTx, postTransaction, + waitTxPropogation, sendPSTFee, } from "./lib/arweave.js"; import { getAllRefs } from "./lib/graphql.js"; @@ -253,7 +254,7 @@ export default class Helper { return (async (resolve, reject) => { let spinner; - let bundledDataTxs = []; + let bundledDatas = []; try { const refs = await this._fetchRefs(); @@ -334,19 +335,13 @@ export default class Helper { currentChunk.push(dataItem); if (currentChunkSize >= CHUNK_SIZE || i === objects.length - 1) { - const bundledDataTx = await makeBundledDataTx( - this._arweave, - this.ArData, - this.wallet, - this.url, - currentChunk - ); + const bundledData = await this.ArData.bundleData(currentChunk); // Reset chunk currentChunkSize = 0; currentChunk = []; - bundledDataTxs.push(bundledDataTx); + bundledDatas.push(bundledData); } bar1.increment(); @@ -366,31 +361,53 @@ export default class Helper { "Uploading git objects to Gitopia [this may take a while]" ); + const bundledDataTxInfo = []; + // Git object bundles - await Promise.all( - bundledDataTxs.map( - async (bundleDataTx) => - await postTransaction(this._arweave, bundleDataTx) - ) - ); + for (let i = 0; i < bundledDatas.length; i++) { + let bundledDataTx = null; + + do { + bundledDataTx = await makeBundledDataTx( + this._arweave, + this.wallet, + this.url, + bundledDatas[i] + ); + bundledDataTxInfo.push({ + id: bundledDataTx.id, + reward: bundledDataTx.reward, + }); + + await postTransaction(this._arweave, bundledDataTx); + } while ( + (await waitTxPropogation(this._arweave, bundledDataTx)) !== 202 + ); + } // update ref - const updateRefTx = await makeUpdateRefTx( - this._arweave, - this.wallet, - this.url, - dst, - srcOid, - bundledDataTxs + let updateRefTx = null; + do { + updateRefTx = await makeUpdateRefTx( + this._arweave, + this.wallet, + this.url, + dst, + srcOid, + bundledDataTxInfo + ); + + await postTransaction(this._arweave, updateRefTx); + } while ( + (await waitTxPropogation(this._arweave, updateRefTx)) !== 202 ); - await postTransaction(this._arweave, updateRefTx); // PST Fee await sendPSTFee( this._arweave, this.wallet, this.url, - bundledDataTxs, + bundledDataTxInfo, updateRefTx.id ); diff --git a/bin/lib/arweave.js b/bin/lib/arweave.js index 9067b65..3b86d80 100644 --- a/bin/lib/arweave.js +++ b/bin/lib/arweave.js @@ -10,6 +10,11 @@ import { newProgressBar } from "./util.js"; const argitRemoteURIRegex = '^gitopia:\/\/([a-zA-Z0-9-_]{43})\/([A-Za-z0-9_.-]*)' const contractId = "1ljLAR55OhtenU0iDWkLGT6jF4ApxeQd5P0gXNyNJXg"; +const sleep = async (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const getStatus = async (arweave, txid) => + (await arweave.transactions.getStatus(txid)).status; + export function parseArgitRemoteURI(remoteURI) { const matchGroups = remoteURI.match(argitRemoteURIRegex); const repoOwnerAddress = matchGroups[1]; @@ -24,7 +29,7 @@ export async function makeUpdateRefTx( remoteURI, ref, oid, - bundledDataTxs + bundledDataTxInfo ) { const { repoName } = parseArgitRemoteURI(remoteURI); const numCommits = shell @@ -54,11 +59,9 @@ export async function makeUpdateRefTx( tx.addTag("Origin", "git-remote-gitopia"); } - const bundledDataTxIds = []; - for (let i = 0; i < bundledDataTxs.length; i++) { - bundledDataTxIds.push(bundledDataTxs[i].id); - } - + const bundledDataTxIds = bundledDataTxInfo.map( + (bundledDataTx) => bundledDataTx.id + ); tx.addTag("Reference-Txs", JSON.stringify(bundledDataTxIds)); await arweave.transactions.sign(tx, wallet); @@ -90,15 +93,8 @@ export const makeDataItem = async ( return await arData.sign(item, wallet); }; -export const makeBundledDataTx = async ( - arweave, - arData, - wallet, - remoteURI, - dataItems -) => { +export const makeBundledDataTx = async (arweave, wallet, remoteURI, bundle) => { const { repoName } = parseArgitRemoteURI(remoteURI); - const bundle = await arData.bundleData(dataItems); const data = JSON.stringify(bundle); const tx = await arweave.createTransaction({ data }, wallet); tx.addTag("Repo", repoName); @@ -128,11 +124,53 @@ export const postTransaction = async (arweave, tx) => { bar.stop(); }; +export const waitTxPropogation = async (arweave, tx) => { + let status = await getStatus(arweave, tx.id); + + let wait = 6; + while (status === 404 && wait--) { + await sleep(5000); + try { + status = await getStatus(arweave, tx.id); + } catch (err) { + wait++; + status = 404; + } + } + + if (status === 400 || status === 404 || status === 410) { + return status; + } + + if (status === 202) { + return 202; + } + + // we'll give it 2 minutes for propogation + if (status === 404) { + let tries = 3; + do { + await sleep(40000); //40 secs + try { + status = await getStatus(arweave, tx.id); + } catch (err) { + tries++; + status = 404; + } + if (status === 200) { + return 200; + } + } while (--tries); + } + + return 404; +}; + export const sendPSTFee = async ( arweave, wallet, remoteURI, - transactions, + transactionsInfo, referenceId ) => { const { repoName } = parseArgitRemoteURI(remoteURI); @@ -146,8 +184,8 @@ export const sendPSTFee = async ( // PST Fee let totalTxFee = new BigNumber(0); - for (let i = 0; i < transactions.length; i++) { - const txFee = new BigNumber(transactions[i].reward); + for (let i = 0; i < transactionsInfo.length; i++) { + const txFee = new BigNumber(transactionsInfo[i].reward); totalTxFee = totalTxFee.plus(txFee); } @@ -159,16 +197,22 @@ export const sendPSTFee = async ( ? pstFee.toFixed(0) : arweave.ar.arToWinston("0.01"); - const pstTx = await arweave.createTransaction( - { target: holder, quantity }, - wallet - ); - pstTx.addTag("Reference-Id", referenceId); - pstTx.addTag("Repo", repoName); - pstTx.addTag("Version", "0.0.2"); - pstTx.addTag("App-Name", "Gitopia"); - pstTx.addTag("Unix-Time", Math.round(new Date().getTime() / 1000).toString()); - - await arweave.transactions.sign(pstTx, wallet); - await arweave.transactions.post(pstTx); + let pstTx = null; + do { + pstTx = await arweave.createTransaction( + { target: holder, quantity }, + wallet + ); + pstTx.addTag("Reference-Id", referenceId); + pstTx.addTag("Repo", repoName); + pstTx.addTag("Version", "0.0.2"); + pstTx.addTag("App-Name", "Gitopia"); + pstTx.addTag( + "Unix-Time", + Math.round(new Date().getTime() / 1000).toString() + ); + + await arweave.transactions.sign(pstTx, wallet); + await arweave.transactions.post(pstTx); + } while ((await waitTxPropogation(arweave, pstTx)) !== 202); }; diff --git a/bin/lib/graphql.js b/bin/lib/graphql.js index 50b3282..e133c72 100644 --- a/bin/lib/graphql.js +++ b/bin/lib/graphql.js @@ -29,18 +29,11 @@ export const getOidByRef = async (arweave, remoteURI, ref) => { { name: "Ref", values: ["${ref}"] } { name: "App-Name", values: ["Gitopia"] } ] - first: 10 + first: 1 ) { edges { node { id - tags { - name - value - } - block { - height - } } } } @@ -56,15 +49,6 @@ export const getOidByRef = async (arweave, remoteURI, ref) => { }; } - edges.sort((a, b) => { - if (b.node.block.height - a.node.block.height < 50) { - const bUnixTime = Number(getTagValue("Unix-Time", b.node.tags)); - const aUnixTime = Number(getTagValue("Unix-Time", a.node.tags)); - return bUnixTime - aUnixTime; - } - return 0; - }); - const id = edges[0].node.id; const response = await arweave.transactions.getData(id, { decode: true, From a808250e0b91e9c3090ceac36e1006d6bbf4cd8a Mon Sep 17 00:00:00 2001 From: faza Date: Mon, 7 Dec 2020 19:17:39 +0530 Subject: [PATCH 4/5] Bump version to 0.1.8 --- bin/helper.js | 2 +- package-lock.json | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/helper.js b/bin/helper.js index 08269d4..a945734 100644 --- a/bin/helper.js +++ b/bin/helper.js @@ -21,7 +21,7 @@ import { newProgressBar } from "./lib/util.js"; import * as deepHash from "arweave/node/lib/deepHash.js"; import ArweaveBundles from "arweave-bundles"; -export const VERSION = "0.1.7"; +export const VERSION = "0.1.8"; const CHUNK_SIZE = 128 * 1024 * 1024; // 128Mb diff --git a/package-lock.json b/package-lock.json index a228480..16feaf6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@gitopia/git-remote-gitopia", - "version": "0.1.7", + "version": "0.1.8", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c207897..fa89a03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gitopia/git-remote-gitopia", - "version": "0.1.7", + "version": "0.1.8", "description": "git-remote-helper for Gitopia", "main": "bin/index.mjs", "bin": { From 4f39ffe5e9235dccc762a6fc9608d4b29b446eae Mon Sep 17 00:00:00 2001 From: faza Date: Mon, 1 Feb 2021 12:56:42 +0530 Subject: [PATCH 5/5] Refactor arweave module --- bin/helper.js | 63 ++++++++++++++++------------------------------ bin/lib/arweave.js | 55 ++++++++++++++++++++-------------------- bin/lib/git.js | 6 +---- bin/lib/graphql.js | 24 ++++++++++-------- 4 files changed, 65 insertions(+), 83 deletions(-) diff --git a/bin/helper.js b/bin/helper.js index a945734..7b3a009 100644 --- a/bin/helper.js +++ b/bin/helper.js @@ -7,12 +7,12 @@ import GitHelper from "./lib/git.js"; import LineHelper from "./lib/line.js"; import Arweave from "arweave"; import { + arweave, makeDataItem, makeUpdateRefTx, parseArgitRemoteURI, makeBundledDataTx, postTransaction, - waitTxPropogation, sendPSTFee, } from "./lib/arweave.js"; import { getAllRefs } from "./lib/graphql.js"; @@ -23,7 +23,7 @@ import ArweaveBundles from "arweave-bundles"; export const VERSION = "0.1.8"; -const CHUNK_SIZE = 128 * 1024 * 1024; // 128Mb +const CHUNK_SIZE = 100 * 1024; // 128Mb const _timeout = async (duration) => { return new Promise((resolve, reject) => { @@ -75,11 +75,7 @@ export default class Helper { // create dirs // fs.ensureDirSync(path.join(this.path, "refs", "remotes", this.name)); // fs.ensureDirSync(path.join(this.path, "dgit", "refs")); - this._arweave = Arweave.init({ - host: "arweave.net", - port: 443, - protocol: "https", - }); + const deps = { utils: Arweave.utils, crypto: Arweave.crypto, @@ -207,7 +203,7 @@ export default class Helper { const spinner = ora("Fetching remote refs from Gitopia").start(); try { - const refs = await getAllRefs(this._arweave, this.url); + const refs = await getAllRefs(this.url); spinner.succeed("Remote refs fetched from Gitopia"); return refs; @@ -288,7 +284,7 @@ export default class Helper { try { spinner = ora(`Checking permissions over ${this.address}`).start(); // check push permission for repo - const address = await this._arweave.wallets.jwkToAddress(this.wallet); + const address = await arweave.wallets.jwkToAddress(this.wallet); const { repoOwnerAddress } = parseArgitRemoteURI(this.url); if (address === repoOwnerAddress) { @@ -365,46 +361,31 @@ export default class Helper { // Git object bundles for (let i = 0; i < bundledDatas.length; i++) { - let bundledDataTx = null; - - do { - bundledDataTx = await makeBundledDataTx( - this._arweave, - this.wallet, - this.url, - bundledDatas[i] - ); - bundledDataTxInfo.push({ - id: bundledDataTx.id, - reward: bundledDataTx.reward, - }); - - await postTransaction(this._arweave, bundledDataTx); - } while ( - (await waitTxPropogation(this._arweave, bundledDataTx)) !== 202 - ); - } - - // update ref - let updateRefTx = null; - do { - updateRefTx = await makeUpdateRefTx( - this._arweave, + let bundledDataTx = await makeBundledDataTx( this.wallet, this.url, - dst, - srcOid, - bundledDataTxInfo + bundledDatas[i] ); + bundledDataTxInfo.push({ + id: bundledDataTx.id, + reward: bundledDataTx.reward, + }); - await postTransaction(this._arweave, updateRefTx); - } while ( - (await waitTxPropogation(this._arweave, updateRefTx)) !== 202 + await postTransaction(bundledDataTx); + } + + // update ref + const updateRefTx = await makeUpdateRefTx( + this.wallet, + this.url, + dst, + srcOid, + bundledDataTxInfo ); + await postTransaction(updateRefTx); // PST Fee await sendPSTFee( - this._arweave, this.wallet, this.url, bundledDataTxInfo, diff --git a/bin/lib/arweave.js b/bin/lib/arweave.js index 3b86d80..858a8bc 100644 --- a/bin/lib/arweave.js +++ b/bin/lib/arweave.js @@ -1,3 +1,4 @@ +import Arweave from "arweave"; import * as smartweave from "smartweave"; import shell from "shelljs"; import pkg from "bignumber.js"; @@ -6,13 +7,21 @@ const { BigNumber } = pkg; import { VERSION } from "../helper.js"; import { newProgressBar } from "./util.js"; +export const arweave = Arweave.init({ + host: "arweave.net", // Arweave Gateway + //host: 'arweave.dev', // Arweave Dev Gateway + port: 443, + protocol: "https", + timeout: 600000, +}); + // prettier-ignore const argitRemoteURIRegex = '^gitopia:\/\/([a-zA-Z0-9-_]{43})\/([A-Za-z0-9_.-]*)' const contractId = "1ljLAR55OhtenU0iDWkLGT6jF4ApxeQd5P0gXNyNJXg"; const sleep = async (ms) => new Promise((resolve) => setTimeout(resolve, ms)); -const getStatus = async (arweave, txid) => +const getStatus = async (txid) => (await arweave.transactions.getStatus(txid)).status; export function parseArgitRemoteURI(remoteURI) { @@ -24,7 +33,6 @@ export function parseArgitRemoteURI(remoteURI) { } export async function makeUpdateRefTx( - arweave, wallet, remoteURI, ref, @@ -93,7 +101,7 @@ export const makeDataItem = async ( return await arData.sign(item, wallet); }; -export const makeBundledDataTx = async (arweave, wallet, remoteURI, bundle) => { +export const makeBundledDataTx = async (wallet, remoteURI, bundle) => { const { repoName } = parseArgitRemoteURI(remoteURI); const data = JSON.stringify(bundle); const tx = await arweave.createTransaction({ data }, wallet); @@ -110,7 +118,7 @@ export const makeBundledDataTx = async (arweave, wallet, remoteURI, bundle) => { return tx; }; -export const postTransaction = async (arweave, tx) => { +export const postTransaction = async (tx) => { const uploader = await arweave.transactions.getUploader(tx); const bar = newProgressBar(); @@ -124,14 +132,14 @@ export const postTransaction = async (arweave, tx) => { bar.stop(); }; -export const waitTxPropogation = async (arweave, tx) => { - let status = await getStatus(arweave, tx.id); +export const waitTxPropogation = async (tx) => { + let status = await getStatus(tx.id); let wait = 6; while (status === 404 && wait--) { await sleep(5000); try { - status = await getStatus(arweave, tx.id); + status = await getStatus(tx.id); } catch (err) { wait++; status = 404; @@ -152,7 +160,7 @@ export const waitTxPropogation = async (arweave, tx) => { do { await sleep(40000); //40 secs try { - status = await getStatus(arweave, tx.id); + status = await getStatus(tx.id); } catch (err) { tries++; status = 404; @@ -167,7 +175,6 @@ export const waitTxPropogation = async (arweave, tx) => { }; export const sendPSTFee = async ( - arweave, wallet, remoteURI, transactionsInfo, @@ -197,22 +204,16 @@ export const sendPSTFee = async ( ? pstFee.toFixed(0) : arweave.ar.arToWinston("0.01"); - let pstTx = null; - do { - pstTx = await arweave.createTransaction( - { target: holder, quantity }, - wallet - ); - pstTx.addTag("Reference-Id", referenceId); - pstTx.addTag("Repo", repoName); - pstTx.addTag("Version", "0.0.2"); - pstTx.addTag("App-Name", "Gitopia"); - pstTx.addTag( - "Unix-Time", - Math.round(new Date().getTime() / 1000).toString() - ); - - await arweave.transactions.sign(pstTx, wallet); - await arweave.transactions.post(pstTx); - } while ((await waitTxPropogation(arweave, pstTx)) !== 202); + const pstTx = await arweave.createTransaction( + { target: holder, quantity }, + wallet + ); + pstTx.addTag("Reference-Id", referenceId); + pstTx.addTag("Repo", repoName); + pstTx.addTag("Version", "0.0.2"); + pstTx.addTag("App-Name", "Gitopia"); + pstTx.addTag("Unix-Time", Math.round(new Date().getTime() / 1000).toString()); + + await arweave.transactions.sign(pstTx, wallet); + await arweave.transactions.post(pstTx); }; diff --git a/bin/lib/git.js b/bin/lib/git.js index 1fe2f52..03a1b29 100644 --- a/bin/lib/git.js +++ b/bin/lib/git.js @@ -30,11 +30,7 @@ export default class GitHelper { if (await this.exists(oid)) return; - const objects = await fetchGitObjects( - this.helper._arweave, - this.helper.ArData, - this.helper.url - ); + const objects = await fetchGitObjects(this.helper.ArData, this.helper.url); for (const object of objects) { dumps.push(this.dump(object.oid, object.data)); diff --git a/bin/lib/graphql.js b/bin/lib/graphql.js index e133c72..dd72640 100644 --- a/bin/lib/graphql.js +++ b/bin/lib/graphql.js @@ -1,5 +1,5 @@ import axios from "axios"; -import { parseArgitRemoteURI } from "./arweave.js"; +import { arweave, parseArgitRemoteURI } from "./arweave.js"; import { newProgressBar } from "./util.js"; const graphQlEndpoint = "https://arweave.net/graphql"; @@ -12,7 +12,7 @@ const getTagValue = (tagName, tags) => { } }; -export const getOidByRef = async (arweave, remoteURI, ref) => { +export const getOidByRef = async (remoteURI, ref) => { const { repoOwnerAddress, repoName } = parseArgitRemoteURI(remoteURI); const { data } = await axios({ url: graphQlEndpoint, @@ -58,7 +58,7 @@ export const getOidByRef = async (arweave, remoteURI, ref) => { return JSON.parse(response); }; -export const getAllRefs = async (arweave, remoteURI) => { +export const getAllRefs = async (remoteURI) => { let refs = new Set(); let refOidObj = {}; const { repoOwnerAddress, repoName } = parseArgitRemoteURI(remoteURI); @@ -103,7 +103,7 @@ export const getAllRefs = async (arweave, remoteURI) => { } for (const ref of refs) { - const { oid } = await getOidByRef(arweave, remoteURI, ref); + const { oid } = await getOidByRef(remoteURI, ref); refOidObj[ref] = oid; } @@ -143,7 +143,7 @@ export const getTransactionIdByObjectId = async (remoteURI, oid) => { return edges[0].node.id; }; -export const fetchGitObjects = async (arweave, arData, remoteURI) => { +export const fetchGitObjects = async (arData, remoteURI) => { const objects = []; const { repoOwnerAddress, repoName } = parseArgitRemoteURI(remoteURI); const { data } = await axios({ @@ -188,9 +188,10 @@ export const fetchGitObjects = async (arweave, arData, remoteURI) => { decode: true, string: true, }); - const items = await arData.unbundleData(txData); - await Promise.all( - items.map(async (item) => { + + try { + const items = await arData.unbundleData(txData); + for (const item of items) { const data = await arData.decodeData(item, { string: false }); for (let i = 0; i < item.tags.length; i++) { const tag = await arData.decodeTag(item.tags[i]); @@ -200,8 +201,11 @@ export const fetchGitObjects = async (arweave, arData, remoteURI) => { break; } } - }) - ); + } + } catch (err) { + // corrupt bundled data + } + bar1.increment(); }) );