feat(src): merge asset trees based on asset cid

README.md

@@ -209,6 +209,58 @@ Asset Tree difference
 ```
 </details>
 
+### Get merge of Commits and Asset Trees of an Asset
+
+```shell
+nit merge <asset-cid> --from-index 3 --to-index 5
+```
+
+Example command of the mockup Asset
+
+```shell
+nit merge aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --from-index 71 --to-index 73
+```
+
+<details>
+<summary>Merge of Commits and Asset Trees in block 71 & 72 (exclude block 73)</summary>
+
+```shell
+from: block 10849040, tx 0x6d5902173255afe379cc4ae934a6c684ecfd865679286665622de3cf10eddcbe
+  to: block 10849133, tx 0xe383fdc0f4eaf44e8bde4737f33bcd45742dcb846f3beb890976793d7cc9358e
+
+Commit merge
+{
+  "assetTreeCid": "bafkreidptwydheqfybug4mmnzwzdg4rqxjvg4akl2pwjmrfxhqz33qv4tu",
+  "assetTreeSha256": "6f9db0339205c0686e318dcdb2337230ba6a6e014bd3ec9644b73c33bdc2bc9d",
+  "assetTreeSignature": "0xef547e124a9904dbdb5a418022ad03c621201b74111a3b4c5fac61b1d82350170766cef8a27737d21ca9b1bd4e04f7cdea460706b68b14e0ed17f2a3de83f9131b",
+  "author": "bafkreigzixvzu2tbxbvmvwcvlz2zwoagmb6c2q5egaq4lmd5sesyopmmx4",
+  "committer": "bafkreigzixvzu2tbxbvmvwcvlz2zwoagmb6c2q5egaq4lmd5sesyopmmx4",
+  "provider": "bafkreigtmno2wacf4ldb6ewbkboe7oojedsp3dky35djytor6guwzhbdpa",
+  "timestampCreated": 1655720763,
+  "action": "bafkreiavifzn7ntlb6p2lr55k3oezo6pofwvknecukc5auhng4miowcte4",
+  "actionResult": "https://bafkreidptwydheqfybug4mmnzwzdg4rqxjvg4akl2pwjmrfxhqz33qv4tu.ipfs.dweb.link",
+  "abstract": "Action action-initial-registration."
+}
+
+Asset Tree merge
+{
+  "assetCid": "bafybeif3ctgbmiso4oykvwj6jebyrkjxqr26bfrkesla5yr2ypgx47wgle",
+  "assetSha256": null,
+  "encodingFormat": "application/zip",
+  "assetTimestampCreated": null,
+  "assetCreator": null,
+  "license": {
+    "name": "Starling License",
+    "document": "https://starlinglab.org"
+  },
+  "abstract": "",
+  "assetTreeCustomKey1": "foo",
+  "assetTreeCustomKey2": "bar",
+  "nftRecord": "bafkreielfjf7sfxigb4r7tejt7jhl6kthxoujwziywixlwxjjho32may7y"
+}
+```
+</details>
+
 ## Configuration
 
 The Nit configuration mimics the [Hardhat configuration](https://hardhat.org/config) with additional fields.
@@ -292,7 +344,9 @@ nit config -e
   "infura": {
     "projectId": "aaaaaaaaaaaaaaaaaaaaaaaaaaa",
     "projectSecret": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-  }
+  },
+  // For IPFS cat source. We support w3s, infura and numbers
+  "ipfsCat": "w3s",
 }
 ```
 

src/ipfs.ts

@@ -11,6 +11,8 @@ let ProjectSecret = "";
 
 let EstuaryInstance;
 
+let IpfsCatFunc = w3sIpfsCat;
+
 export async function initInfura(projectId, projectSecret) {
   ProjectId = projectId;
   ProjectSecret = projectSecret;
@@ -66,12 +68,25 @@ export async function w3sIpfsCat(cid) {
   return r.rawBody;
 }
 
+export async function numbersIpfsCat(cid) {
+  const url = `https://ipfs-pin.numbersprotocol.io/ipfs/${cid}`;
+  const requestConfig = {
+    timeout: { request: 30000 },
+  }
+  /* FIXME: Axios's response.data.lenght is smaller than content length.
+   * Use Got as a temporary workardound.
+   * https://github.com/axios/axios/issues/3711
+   */
+  const r = await got.get(url, requestConfig);
+  return r.rawBody;
+}
+
 export async function ipfsAddBytes(bytes) {
   return await infuraIpfsAddBytes(bytes);
 }
 
 export async function ipfsCat(cid) {
-  return await w3sIpfsCat(cid);
+  return await IpfsCatFunc(cid);
 }
 
 export async function cidToJsonString(cid) {
@@ -107,4 +122,14 @@ export async function estuaryAdd(bytes) {
   } catch(error) {
     console.error(error);
   }
-}
+}
+
+export async function initIpfsCat(source: string) {
+  if (source == "numbers") {
+    IpfsCatFunc = numbersIpfsCat;
+  } else if(source == "infura") {
+    IpfsCatFunc = infuraIpfsCat;
+  } else {
+    IpfsCatFunc = w3sIpfsCat;
+  }
+}

src/nit.ts

@@ -76,6 +76,8 @@ export const nitconfigTemplate = {
     "projectId": "a".repeat(infuraSecretLength),
     "projectSecret": "a".repeat(infuraSecretLength)
   },
+  /* IPFS cat source. E.g. w3s, infura or numbers */
+  "ipfsCat": "w3s",
   "commitDatabase": {
     "updateUrl": "",
     "commitUrl": "",
@@ -521,6 +523,46 @@ export async function getCommitsSummary(events) {
   return commitsSummary;
 }
 
+export async function merge(assetCid: string, blockchainInfo, fromIndex: number = 0, toIndex: number = -1) {
+  let confirmedToIndex = toIndex;
+  if (toIndex <= fromIndex) {
+    const commitBlockNumbers = await getCommitBlockNumbers(assetCid, blockchainInfo);
+    const commitAmount: number = commitBlockNumbers.length;
+    confirmedToIndex = commitAmount;
+  }
+
+  const commitEvents = await iterateCommitEvents(assetCid, blockchainInfo, fromIndex, confirmedToIndex);
+  const commits:  { commit: object; blockNumber: number; transactionHash: string; }[] = await getCommits(commitEvents);
+  let assetTrees = []
+  for (let i = 0; i < commits.length; i += 10) {
+    assetTrees = [...assetTrees, ...await Promise.all(commits.slice(i, i + 10).map(
+      (async (commit) => {
+        if ("assetTreeCid" in commit.commit) {
+          try {
+            return await getAssetTree(commit.commit["assetTreeCid"]);
+          } catch (error) {
+            console.error(`Cannot get valid assetTree from block ${commit.blockNumber}`);
+          }
+        }
+        return {};
+      })
+    ))];
+  }
+
+  const fromCommit = commits[0];
+  const toCommit = commits[commits.length - 1];
+  return {
+    "fromIndex": fromIndex,
+    "fromBlockNumber": fromCommit?.blockNumber,
+    "fromTransactionHash": fromCommit?.transactionHash,
+    "toIndex": toIndex,
+    "toBlockNumber": toCommit?.blockNumber,
+    "toTransactionHash": toCommit?.transactionHash,
+    "commitMerge": util.mergeJsons(commits.map(commit => commit.commit)),
+    "assetTreeMerge": util.mergeJsons(assetTrees),
+  };
+}
+
 /* TODO: Remove this function in the next feature release.
  *
  * Query events on Ethereum. The performance is faster than eventLogIteratingQuery.

src/run.ts

@@ -121,6 +121,7 @@ async function help() {
         "verify    Verify integrity signature",
         "log       Show asset's commits",
         "diff      Show diff between two commits",
+        "merge     Show merged assetTree",
         "help      Show this usage tips",
       ]
     },
@@ -252,6 +253,14 @@ async function help() {
         "$ nit diff {underline assetCid} --from|-f {underline blockNumberIndex} --to|-t {underline blockNumberIndex}",
       ]
     },
+    {
+      header: "merge",
+      content: [
+        "$ nit merge {underline assetCid}",
+        "$ nit merge {underline assetCid} --from-index|-f {underline blockNumberIndex}",
+        "$ nit merge {underline assetCid} --from-index|-f {underline blockNumberIndex} --to-index|-t {underline blockNumberIndex}",
+      ]
+    },
   ]
   const usage = commandLineUsage(sections)
   console.log(usage)
@@ -377,6 +386,18 @@ async function parseArgs() {
       "command": "diff",
       "params": paramOptions
     }
+  } else if (commandOptions.command === "merge") {
+    const paramDefinitions = [
+      { name: "asset-cid", defaultOption: true },
+      { name: "from-index", alias: "f", defaultValue: 0 },
+      { name: "to-index", alias: "t", defaultValue: -1 },
+    ];
+    const paramOptions = commandLineArgs(paramDefinitions,
+                                         { argv, stopAtFirstUnknown: true });
+    return {
+      "command": "merge",
+      "params": paramOptions
+    }
   } else if (commandOptions.command === "config") {
     const paramDefinitions = [
       { name: "edit", alias: "e" },
@@ -454,6 +475,7 @@ async function main() {
   const blockchain = await nit.loadBlockchain(config);
 
   await ipfs.initInfura(config.infura.projectId, config.infura.projectSecret);
+  await ipfs.initIpfsCat(config.ipfsCat);
 
   if (args.command === "ipfsadd") {
     const contentBytes = fs.readFileSync(args.params.fileapth);
@@ -701,6 +723,20 @@ async function main() {
     } else {
       await help();
     }
+  } else if (args.command === "merge") {
+    if ("asset-cid" in args.params) {
+      const result = await nit.merge(args.params["asset-cid"], blockchain, args.params["from-index"], args.params["to-index"]);
+      console.log(`from: block ${result.fromBlockNumber}, tx ${result.fromTransactionHash}`);
+      console.log(`  to: block ${result.toBlockNumber}, tx ${result.toTransactionHash}`);
+
+      console.log("\nCommit merge");
+      console.log(JSON.stringify(result.commitMerge, null, 2));
+
+      console.log("\nAsset Tree merge");
+      console.log(JSON.stringify(result.assetTreeMerge, null, 2));
+    } else {
+      await help();
+    }
   } else if (args.command === "config") {
     if ("edit" in args.params) {
       await launch(`${configFilepath}`);

src/util.ts

@@ -13,6 +13,10 @@ export function deepCopy(data) {
   return JSON.parse(JSON.stringify(data));
 }
 
+export function mergeJsons(jsonArray: object[]): object {
+  return Object.assign({}, ...jsonArray);
+}
+
 export function timestampToIsoString(timestamp): string {
   try {
     return new Date((parseInt(timestamp) * 1000)).toISOString()

tests/testUtil.ts

@@ -0,0 +1,65 @@
+/* manual test: yarn run test tests/testUtil.ts
+ */
+
+import { expect } from "chai";
+import * as util from "../src/util";
+
+describe("Util Functions", function() {
+  it("Should merge JSON objects correctly", async function () {
+    const json1 = {
+      "assetCid": "bafkreid4ug5djtm6iq6hptfs337n3k3driqncivegexzpffzlapiaple44",
+      "assetSha256": "7ca1ba34cd9e443c77ccb2defeddab638a20d122a4312f9794b9581e803d64e7",
+      "encodingFormat": "image/jpeg",
+      "assetTimestampCreated": 1683287179,
+      "assetCreator": "",
+      "license": {
+        "name": null,
+        "document": null
+      },
+      "abstract": "",
+      "creatorWallet": "0x6059DFC1daFb109474aB6fAD87E93A11Bfa5e1D2"
+    }
+    const json2 = {
+      "assetCid": "bafkreid4ug5djtm6iq6hptfs337n3k3driqncivegexzpffzlapiaple44",
+      "assetSha256": "7ca1ba34cd9e443c77ccb2defeddab638a20d122a4312f9794b9581e803d64e7",
+      "encodingFormat": "image/jpeg",
+      "assetTimestampCreated": 1683287179,
+      "assetCreator": "",
+      "license": {
+        "name": "CC-BY-NC-ND-4.0",
+        "document": "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode"
+      },
+    }
+    const json3 = {
+      "assetCid": "bafkreid4ug5djtm6iq6hptfs337n3k3driqncivegexzpffzlapiaple44",
+      "assetSha256": "7ca1ba34cd9e443c77ccb2defeddab638a20d122a4312f9794b9581e803d64e7",
+      "encodingFormat": "image/jpeg",
+      "assetTimestampCreated": 1683287179,
+      "assetCreator": "",
+      "assetSourceType": "captureUpload",
+    };
+
+    const mergedJSON = util.mergeJsons([json1, json2, json3]);
+
+    expect(JSON.stringify(mergedJSON)).to.be.equal(JSON.stringify({
+      "assetCid": "bafkreid4ug5djtm6iq6hptfs337n3k3driqncivegexzpffzlapiaple44",
+      "assetSha256": "7ca1ba34cd9e443c77ccb2defeddab638a20d122a4312f9794b9581e803d64e7",
+      "encodingFormat": "image/jpeg",
+      "assetTimestampCreated": 1683287179,
+      "assetCreator": "",
+      "license": {
+        "name": "CC-BY-NC-ND-4.0",
+        "document": "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode"
+      },
+      "abstract": "",
+      "creatorWallet": "0x6059DFC1daFb109474aB6fAD87E93A11Bfa5e1D2",
+      "assetSourceType": "captureUpload",
+    }));
+  });
+
+  it("Should return an empty object when merged an empty array", async function () {
+    const mergedJson = util.mergeJsons([]);
+
+    expect(JSON.stringify(mergedJson)).to.be.equal(JSON.stringify({}))
+  });
+});