Reduce strict-order dependency resolution overhead during large DataPack deploys

packages/vlocity-deploy/src/__tests__/datapackDeployment.test.ts

@@ -212,6 +212,35 @@ describe('DatapackDeployment', () => {
             expect(deploySets[0].map(r => r.sourceKey)).toStrictEqual([ 'A/1', 'B/1', 'C/1', 'D/1']);
             expect(deploySets[1].map(r => r.sourceKey)).toStrictEqual([ 'A/2', 'B/2', 'C/2', 'D/2']);
         });
+        it('with [strictOrder: true] should reuse datapack status lookups while selecting deployable records', () => {
+            // Arrange
+            const deployment = container.new(DatapackDeployment, { strictOrder: true });
+            const parentB1 = mockDatapackRecord({ sourceKey: 'B/1', datapackKey: 'B' });
+            const parentB2 = mockDatapackRecord({ sourceKey: 'B/2', datapackKey: 'B' });
+            const childA1 = mockDatapackRecord({ sourceKey: 'A/1', datapackKey: 'A' });
+            const childA2 = mockDatapackRecord({ sourceKey: 'A/2', datapackKey: 'A' });
+            const childA3 = mockDatapackRecord({ sourceKey: 'A/3', datapackKey: 'A' });
+
+            childA1.addDependency(parentB1);
+            childA2.addDependency(parentB1);
+            childA3.addDependency(parentB2);
+            parentB1.updateStatus(DeploymentStatus.Deployed, 'ID');
+
+            deployment.add(parentB1, parentB2, childA1, childA2, childA3);
+            const getDatapackStatusSpy = jest.spyOn(deployment as any, 'getDatapackStatus');
+            const isCircularDatapackDependencySpy = jest.spyOn(deployment as any, 'isCircularDatapackDependency');
+
+            // Act
+            const records = deployment['getDeployableRecords']();
+
+            // Assert
+            if (!records) {
+                throw new Error('Expected deployment records to be returned');
+            }
+            expect([...records.values()].map(record => record.sourceKey)).toStrictEqual([ 'B/2' ]);
+            expect(getDatapackStatusSpy.mock.calls.filter(([datapackKey]) => datapackKey === 'B')).toHaveLength(1);
+            expect(isCircularDatapackDependencySpy.mock.calls.filter(([fromDatapackKey, toDatapackKey]) => fromDatapackKey === 'A' && toDatapackKey === 'B')).toHaveLength(1);
+        });
     });
 
     describe('hasCircularDependencies', () => {
@@ -254,4 +283,4 @@ describe('DatapackDeployment', () => {
             expect(result).toStrictEqual([ 'A', 'C', 'B', 'A' ]);	
         });
     });
-});
+});

packages/vlocity-deploy/src/datapackDeployment.ts

@@ -416,8 +416,10 @@ export class DatapackDeployment extends AsyncEventEmitter<DatapackDeploymentEven
      */
     private getDeployableRecords() {
         const records = new Map<string, DatapackDeploymentRecord>();
+        const datapackStatusCache = new Map<string, DeploymentStatus | undefined>();
+        const circularDependencyCache = new Map<string, boolean>();
         for (const record of this.records.values()) {
-            if (record.isPending && record.retryCount == 0 && !this.hasPendingDependencies(record)) {
+            if (record.isPending && record.retryCount == 0 && !this.hasPendingDependencies(record, datapackStatusCache, circularDependencyCache)) {
                 records.set(record.sourceKey, record);
             }
         }
@@ -456,7 +458,7 @@ export class DatapackDeployment extends AsyncEventEmitter<DatapackDeploymentEven
      * Check if a record has pending dependencies that are not yet deployed as part of the current deployment
      * @param record
      */
-    private hasPendingDependencies(record: DatapackDeploymentRecord) : boolean {
+    private hasPendingDependencies(record: DatapackDeploymentRecord, datapackStatusCache?: Map<string, DeploymentStatus | undefined>, circularDependencyCache?: Map<string, boolean>) : boolean {
         for(const key of record.getDependencySourceKeys()) {
             const dependentRecord = this.records.get(key);
             if (!dependentRecord) {
@@ -469,9 +471,16 @@ export class DatapackDeployment extends AsyncEventEmitter<DatapackDeploymentEven
 
             const isExternalDependency = dependentRecord.datapackKey !== record.datapackKey;
             if (this.options.strictOrder && isExternalDependency) {
-                const dependencyStatus = this.getDatapackStatus(dependentRecord.datapackKey);
+                const cacheKey = dependentRecord.datapackKey;
+                const dependencyStatus = datapackStatusCache
+                    ? mapGetOrCreate(datapackStatusCache, cacheKey, () => this.getDatapackStatus(cacheKey))
+                    : this.getDatapackStatus(cacheKey);
                 if (dependencyStatus !== undefined && dependencyStatus < DeploymentStatus.Deployed) {
-                    if (this.isCircularDatapackDependency(record.datapackKey, dependentRecord.datapackKey)) {
+                    const circularDependencyKey = `${record.datapackKey}\u0000${dependentRecord.datapackKey}`;
+                    const isCircularDependency = circularDependencyCache
+                        ? mapGetOrCreate(circularDependencyCache, circularDependencyKey, () => this.isCircularDatapackDependency(record.datapackKey, dependentRecord.datapackKey))
+                        : this.isCircularDatapackDependency(record.datapackKey, dependentRecord.datapackKey);
+                    if (isCircularDependency) {
                         this.reportWarning(record, `Circular datapack dependency: ${record.datapackKey}->${dependentRecord.datapackKey}->${record.datapackKey}`);
                         continue;
                     }