From f300b04a5ba3a388ff00ef5c27264cf5a0b3440f Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Thu, 30 Jul 2026 17:00:00 +0530 Subject: [PATCH 01/15] fix(delta): resolve entry references correctly and fix premature locale-migrated tracking Reference fields (Link/Array Entry) written during a locale-localize restart kept the source CMS entry id instead of the real Contentstack uid, since only the master-locale bulk import resolved references correctly. Threads entry uid-mapper data into the update config so entry-update-script.cjs can resolve them, mirroring the existing asset uid resolution. Also fixes a bug where finishing any locale marked ALL configured locales as migrated, causing not-yet-migrated locales to be silently skipped on later delta restarts. Now only locales actually processed in that run are recorded. --- api/src/services/migration.service.ts | 8 +- api/src/services/runCli.service.ts | 61 +++++++-- api/src/utils/entry-update-script.cjs | 87 +++++++++++- api/src/utils/entry-update.utils.ts | 58 ++++++++ api/src/utils/locale-migration.utils.ts | 32 +++++ .../unit/utils/entry-update-script.test.ts | 129 +++++++++++++++++- .../unit/utils/entry-update.enrich.test.ts | 76 +++++++++++ .../unit/utils/locale-migration.utils.test.ts | 57 ++++++++ 8 files changed, 491 insertions(+), 17 deletions(-) diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index c7821a8b5..4e1628a53 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -50,7 +50,7 @@ import { import { aemService } from './aem.service.js'; import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; import { utilsUpdateCli } from './updateEntryCli.service.js'; -import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; +import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithEntryMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; import { removeExistingAssets, saveAssetMetadata, AssetUpdate } from '../utils/asset-update.utils.js'; /** @@ -1262,6 +1262,12 @@ const startMigration = async (req: Request): Promise => { iteration, safeDeltaMigrationLogPath ); + enrichConfigWithEntryMapping( + configFilePath, + safePid, + iteration, + safeDeltaMigrationLogPath + ); enrichConfigWithAssetUpdates( configFilePath, assetUpdates, diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index 8f90b01a2..e541d3602 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -20,6 +20,7 @@ interface TestStack { } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; import writeUidMapping, { writePerLocaleEntryUidMapping } from '../utils/uid-mapper.utils.js'; +import { extractLocalesFromUpdateConfig, recordMigratedLocales } from '../utils/locale-migration.utils.js'; /** * Determines log level based on message content without removing ANSI codes @@ -322,20 +323,54 @@ export const runCli = async ( ProjectModelLowdb.data.projects[projectIndex].current_step = getStepperSteps(ProjectModelLowdb.data.projects[projectIndex]?.iteration).MIGRATION; ProjectModelLowdb.data.projects[projectIndex].status = 5; - // Record every locale that just successfully migrated so the next delta restart can - // tell which locales need a full pass vs delta. Set-union with prior value. - const proj: any = ProjectModelLowdb.data.projects[projectIndex]; - const ranLocales = Array.from( - new Set([ - ...Object.keys(proj?.master_locale ?? {}), - ...Object.keys(proj?.locales ?? {}), - ]), - ); - const existing: string[] = Array.isArray(proj?.migrated_locales) - ? proj.migrated_locales - : []; - proj.migrated_locales = Array.from(new Set([...existing, ...ranLocales])); await ProjectModelLowdb.write(); + + // Record every locale that was ACTUALLY processed this run so the next delta + // restart can tell which locales still need a full pass vs delta. + // + // On iteration 1 there's no delta/localize step at all — the whole configured + // locale set genuinely gets migrated in one shot, so using the full config is + // correct here. From iteration 2 onward, a locale only "ran" this iteration if + // it's the master locale (always present) or its entries were actually queued + // in this iteration's updated-entries.json (written by removeEntriesFromDatabase + // before this CLI import step even started). Using the FULL project locale + // config here — instead of what this run actually touched — used to mark + // not-yet-migrated locales as done prematurely, permanently skipping them on + // every later restart (see CMG delta-migration locale bug). + const proj: any = ProjectModelLowdb.data.projects[projectIndex]; + const currentIteration = proj?.iteration || 1; + let ranLocales: string[]; + if (currentIteration <= 1) { + ranLocales = Array.from( + new Set([ + ...Object.keys(proj?.master_locale ?? {}), + ...Object.keys(proj?.locales ?? {}), + ]), + ); + } else { + const updatedEntriesPath = path.join( + process.cwd(), + DATABASE_FILES.DIRECTORY, + projectId, + currentIteration.toString(), + DATABASE_FILES.UPDATED_ENTRIES, + ); + let updateConfig: Record | null = null; + if (fs.existsSync(updatedEntriesPath)) { + try { + updateConfig = JSON.parse(fs.readFileSync(updatedEntriesPath, 'utf-8')); + } catch { + updateConfig = null; + } + } + ranLocales = Array.from( + new Set([ + ...Object.keys(proj?.master_locale ?? {}), + ...extractLocalesFromUpdateConfig(updateConfig), + ]), + ); + } + await recordMigratedLocales(projectId, ranLocales); } } else { console.info('User not found.'); diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index a14d0bdb3..96419bc5b 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -4,6 +4,63 @@ const isAssetField = (value) => value && typeof value === 'object' && !Array.isArray(value) && 'urlPath' in value && 'filename' in value; +/** Shape produced by processField's 'reference' case: { uid, _content_type_uid }. */ +const isReferenceValue = (value) => + value && typeof value === 'object' && !Array.isArray(value) && + 'uid' in value && '_content_type_uid' in value; + +const isReferenceArray = (value) => + Array.isArray(value) && value.length > 0 && value.every(isReferenceValue); + +/** + * Resolves a source-side entry uid to its real Contentstack destination uid. + * + * The export JSON's reference fields carry the SOURCE cms entry id (see + * `contentful.service.ts`'s `createRefrence`), which only happens to equal the + * Contentstack uid when entries are imported preserving source ids. The + * bulk/master-locale import resolves this correctly via the CLI's own + * reference pass; this update path does not, so it needs the same uid-mapper + * data the asset resolution above already uses (see `entryMapping`). + * + * Preference order: per-locale mapping (most precise — handles entries that + * ended up as distinct Contentstack uids per locale across iterations) → + * flat mapping → identity fallback (keeps existing behavior when no mapping + * data exists, e.g. simple setups where source id equals destination uid). + */ +const resolveReferenceUid = (sourceUid, locale, entryMapping) => { + if (!sourceUid) return sourceUid; + const newByLocale = entryMapping?.new?.byLocale?.[locale]?.[sourceUid]; + if (newByLocale) return newByLocale; + const oldByLocale = entryMapping?.old?.byLocale?.[locale]?.[sourceUid]; + if (oldByLocale) return oldByLocale; + const newFlat = entryMapping?.new?.flat?.[sourceUid]; + if (newFlat) return newFlat; + const oldFlat = entryMapping?.old?.flat?.[sourceUid]; + if (oldFlat) return oldFlat; + return sourceUid; +}; + +/** + * Remaps the uid(s) inside a reference field value (single link object or + * array of link objects) to their Contentstack destination uids. + */ +const resolveReferenceField = (fieldName, entryUid, value, locale, entryMapping) => { + if (isReferenceValue(value)) { + const resolved = resolveReferenceUid(value.uid, locale, entryMapping); + if (resolved !== value.uid) { + console.info(`[${entryUid}] "${fieldName}"${locale ? ` (${locale})` : ''}: resolved reference uid "${value.uid}" → "${resolved}"`); + } + return { ...value, uid: resolved }; + } + if (isReferenceArray(value)) { + return value.map((item) => { + const resolved = resolveReferenceUid(item.uid, locale, entryMapping); + return { ...item, uid: resolved }; + }); + } + return value; +}; + /** Export JSON metadata — not Contentstack content-type field UIDs (WordPress entries are flat). */ const FLAT_PAYLOAD_SKIP = new Set([ 'uid', @@ -68,7 +125,7 @@ const resolveAssetField = (fieldName, entryUid, updateValue, stackValue, oldMapp * WordPress (and similar) write migration JSON with fields at the root (email, url, …). * Fetched stack entries keep custom fields under entry.content — merge flat updateData there. */ -const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts) => { +const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts, locale, entryMapping) => { for (const field of Object.keys(updateData)) { if (FLAT_PAYLOAD_SKIP.has(field)) { continue; @@ -89,6 +146,8 @@ const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping oldMapping, newMapping ); + } else if (isReferenceValue(nextVal) || isReferenceArray(nextVal)) { + nextVal = resolveReferenceField(field, entryUid, nextVal, locale, entryMapping); } entry.content[field] = nextVal; } @@ -103,6 +162,9 @@ module.exports = async ({ const assetMapping = config.__assetMapping__ || { old: {}, new: {} }; delete config.__assetMapping__; + const entryMapping = config.__entryMapping__ || { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: {} } }; + delete config.__entryMapping__; + // Assets the user chose to update in place (same UID, new file). const assetUpdates = Array.isArray(config.__assetUpdates__) ? config.__assetUpdates__ : []; delete config.__assetUpdates__; @@ -110,6 +172,7 @@ module.exports = async ({ const oldMapping = assetMapping.old || {}; const newMapping = assetMapping.new || {}; console.info(`Asset mappings loaded — old: ${Object.keys(oldMapping).length}, new: ${Object.keys(newMapping).length}`); + console.info(`Entry mappings loaded — old: ${Object.keys(entryMapping?.old?.flat || {}).length} flat / ${Object.keys(entryMapping?.old?.byLocale || {}).length} locales, new: ${Object.keys(entryMapping?.new?.flat || {}).length} flat / ${Object.keys(entryMapping?.new?.byLocale || {}).length} locales`); console.info(`Asset updates to replace in place: ${assetUpdates.length}`); const contentTypes = Object.keys(config); @@ -187,13 +250,21 @@ module.exports = async ({ oldMapping, newMapping ); + } else if (isReferenceValue(updateData?.content[field]) || isReferenceArray(updateData?.content[field])) { + updateData.content[field] = resolveReferenceField( + field, + entryUid, + updateData?.content[field], + locale, + entryMapping + ); } } Object.assign(entry?.content, updateData?.content); await entry.update(updateOpts); } else if (hasStackContent) { console.info(`[${realEntryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)${locale ? ` for locale "${locale}"` : ''}`); - await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts); + await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts, locale, entryMapping); } else { if (updateData && entry) { for (const field of Object.keys(updateData)) { @@ -206,6 +277,14 @@ module.exports = async ({ oldMapping, newMapping ); + } else if (isReferenceValue(updateData[field]) || isReferenceArray(updateData[field])) { + updateData[field] = resolveReferenceField( + field, + entryUid, + updateData[field], + locale, + entryMapping + ); } } } @@ -237,3 +316,7 @@ module.exports = async ({ module.exports.isAssetField = isAssetField; module.exports.resolveAssetField = resolveAssetField; module.exports.mergeFlatPayloadIntoEntry = mergeFlatPayloadIntoEntry; +module.exports.isReferenceValue = isReferenceValue; +module.exports.isReferenceArray = isReferenceArray; +module.exports.resolveReferenceUid = resolveReferenceUid; +module.exports.resolveReferenceField = resolveReferenceField; diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 79cf3a641..9e07e9550 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -297,6 +297,64 @@ export const enrichConfigWithAssetMapping = ( writeLogEntry(`Asset references will be resolved using combined old and new mappings`, "enrichConfigWithAssetMapping", loggerPath); }; +/** + * Reads old (previous iteration) and new (current iteration) entry uid mappings + * — both the flat source→dest map and the per-locale map written by + * `writePerLocaleEntryUidMapping` — and merges them into the updated-entries + * config file under `__entryMapping__`. + * + * This lets the entry-update-script resolve Link(Entry)/reference field values + * to their real Contentstack destination uid before writing them onto a + * localized (non-master) copy of an entry. Without this, reference fields + * written during a locale-add restart keep the export's source-side uid, + * which only happens to work when source and destination uids are identical — + * the master-locale bulk import resolves this correctly via the Contentstack + * CLI's own reference pass, but this update path does not, unless we prime it + * with the same uid-mapper data (mirrors `enrichConfigWithAssetMapping`). + */ +export const enrichConfigWithEntryMapping = ( + configFilePath: string, + projectId: string, + iteration: number, + loggerPath?: string +): void => { + const dbBase = path.join(process.cwd(), DATABASE_FILES.DIRECTORY, projectId); + + const readEntryMapper = (iter: number): { flat: Record; byLocale: Record> } => { + const p = path.join(dbBase, iter.toString(), DATABASE_FILES.UID_MAPPER); + if (!fs.existsSync(p)) return { flat: {}, byLocale: {} }; + try { + const data = JSON.parse(fs.readFileSync(p, "utf-8")); + return { flat: data?.entry || {}, byLocale: data?.entryByLocale || {} }; + } catch (err) { + console.error(`Failed to read uid-mapper for iteration ${iter}:`, err); + return { flat: {}, byLocale: {} }; + } + }; + + const oldEntryMapping = iteration > 1 ? readEntryMapper(iteration - 1) : { flat: {}, byLocale: {} }; + const newEntryMapping = readEntryMapper(iteration); + + writeLogEntry( + `Loaded entry uid mappings — old: ${Object.keys(oldEntryMapping.flat).length} flat / ${Object.keys(oldEntryMapping.byLocale).length} locales, ` + + `new: ${Object.keys(newEntryMapping.flat).length} flat / ${Object.keys(newEntryMapping.byLocale).length} locales`, + "enrichConfigWithEntryMapping", + loggerPath, + ); + + try { + const config = JSON.parse(fs.readFileSync(configFilePath, "utf-8")); + config.__entryMapping__ = { old: oldEntryMapping, new: newEntryMapping }; + fs.writeFileSync(configFilePath, JSON.stringify(config), "utf-8"); + } catch (err) { + console.error("Failed to write entry mapping into update config:", err); + writeLogEntry(`Failed to write __entryMapping__ into ${configFilePath}: ${(err as Error)?.message}`, "enrichConfigWithEntryMapping", loggerPath); + return; + } + + writeLogEntry(`Entry mapping enriched into config for iteration ${iteration}`, "enrichConfigWithEntryMapping", loggerPath); +}; + /** * Ensures an update config file exists for this iteration and returns its path. * Used when there are asset updates but no entry updates produced a config, so diff --git a/api/src/utils/locale-migration.utils.ts b/api/src/utils/locale-migration.utils.ts index af5941c56..5095d94af 100644 --- a/api/src/utils/locale-migration.utils.ts +++ b/api/src/utils/locale-migration.utils.ts @@ -64,6 +64,38 @@ export const isFullMigrationForLocale = ( return !getMigratedLocales(project).includes(localeCode); }; +/** + * Extracts destination locale codes that were ACTUALLY targeted by a delta + * run, from an `updated-entries.json` config object. + * + * Per-entry keys in that config are `${csUid}::${localeCode}` (see + * `removeEntriesFromDatabase` in entry-update.utils.ts) — this reads the + * locale suffix back out. Bookkeeping keys added by the enrich* helpers + * (`__assetMapping__`, `__entryMapping__`, `__assetUpdates__`) are skipped. + * + * This exists to fix a bug where a locale got marked "migrated" as soon as + * ANY locale finished a delta run, instead of only the locale(s) that run + * actually processed — which permanently skipped locales configured ahead of + * when they were meant to be migrated (see `runCli.service.ts`). + */ +export const extractLocalesFromUpdateConfig = ( + config: Record | null | undefined, +): string[] => { + if (!config || typeof config !== 'object') return []; + const locales = new Set(); + for (const [ctKey, entries] of Object.entries(config)) { + if (ctKey.startsWith('__')) continue; + if (!entries || typeof entries !== 'object') continue; + for (const entryKey of Object.keys(entries)) { + const sep = entryKey.lastIndexOf('::'); + if (sep === -1) continue; + const locale = entryKey.slice(sep + 2); + if (locale) locales.add(locale); + } + } + return Array.from(locales); +}; + /** * Set-union the given locales into project.migrated_locales and persist. * Idempotent. diff --git a/api/tests/unit/utils/entry-update-script.test.ts b/api/tests/unit/utils/entry-update-script.test.ts index ae0582a7a..aca188294 100644 --- a/api/tests/unit/utils/entry-update-script.test.ts +++ b/api/tests/unit/utils/entry-update-script.test.ts @@ -7,7 +7,15 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; // default function export for testing. const require = createRequire(import.meta.url); const script = require('../../../src/utils/entry-update-script.cjs'); -const { isAssetField, resolveAssetField, mergeFlatPayloadIntoEntry } = script; +const { + isAssetField, + resolveAssetField, + mergeFlatPayloadIntoEntry, + isReferenceValue, + isReferenceArray, + resolveReferenceUid, + resolveReferenceField, +} = script; describe('entry-update-script — isAssetField', () => { it('is true only for objects carrying urlPath + filename', () => { @@ -66,6 +74,91 @@ describe('entry-update-script — resolveAssetField (3-way resolution)', () => { }); }); +describe('entry-update-script — isReferenceValue / isReferenceArray', () => { + it('recognizes the { uid, _content_type_uid } shape produced by processField', () => { + expect(isReferenceValue({ uid: 'src-1', _content_type_uid: 'author' })).toBe(true); + }); + + it('is false for asset shapes, primitives, and arrays', () => { + expect(isReferenceValue({ urlPath: '/x', filename: 'f.jpg' })).toBe(false); + expect(isReferenceValue(null)).toBeFalsy(); + expect(isReferenceValue('str')).toBeFalsy(); + expect(isReferenceValue([{ uid: 'a', _content_type_uid: 'b' }])).toBe(false); + expect(isReferenceValue({ uid: 'src-1' })).toBe(false); // missing _content_type_uid + }); + + it('recognizes a non-empty array of reference values (multi-reference field)', () => { + expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, { uid: 'b', _content_type_uid: 'article' }])).toBe(true); + }); + + it('is false for an empty array or a mixed array', () => { + expect(isReferenceArray([])).toBe(false); + expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, 'not-a-ref'])).toBe(false); + }); +}); + +describe('entry-update-script — resolveReferenceUid', () => { + const locale = 'en-in'; + const entryMapping = { + old: { flat: { 'src-1': 'cs-old-flat' }, byLocale: { 'en-in': { 'src-2': 'cs-old-locale' } } }, + new: { flat: { 'src-3': 'cs-new-flat' }, byLocale: { 'en-in': { 'src-1': 'cs-new-locale' } } }, + }; + + it('prefers the new per-locale mapping over everything else', () => { + expect(resolveReferenceUid('src-1', locale, entryMapping)).toBe('cs-new-locale'); + }); + + it('falls back to the old per-locale mapping when no new per-locale entry exists', () => { + expect(resolveReferenceUid('src-2', locale, entryMapping)).toBe('cs-old-locale'); + }); + + it('falls back to the flat new mapping when no per-locale entry exists at all', () => { + expect(resolveReferenceUid('src-3', locale, entryMapping)).toBe('cs-new-flat'); + }); + + it('falls back to the flat old mapping as a last resort', () => { + const mapping = { old: { flat: { 'src-4': 'cs-old-flat-only' }, byLocale: {} }, new: { flat: {}, byLocale: {} } }; + expect(resolveReferenceUid('src-4', locale, mapping)).toBe('cs-old-flat-only'); + }); + + it('falls back to identity (source uid unchanged) when no mapping exists at all', () => { + expect(resolveReferenceUid('unmapped-src', locale, { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: {} } })).toBe('unmapped-src'); + }); + + it('handles a missing/undefined entryMapping gracefully', () => { + expect(resolveReferenceUid('src-1', locale, undefined)).toBe('src-1'); + }); +}); + +describe('entry-update-script — resolveReferenceField', () => { + const locale = 'en-gb'; + const entryMapping = { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: { 'en-gb': { 'author-src': 'author-cs' } } } }; + + it('remaps a single reference value uid', () => { + const out = resolveReferenceField('author', 'e1', { uid: 'author-src', _content_type_uid: 'author' }, locale, entryMapping); + expect(out).toEqual({ uid: 'author-cs', _content_type_uid: 'author' }); + }); + + it('remaps every uid in a multi-reference array', () => { + const mapping = { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: { 'en-gb': { a1: 'a1-cs', a2: 'a2-cs' } } } }; + const out = resolveReferenceField( + 'relatedArticles', + 'e1', + [{ uid: 'a1', _content_type_uid: 'article' }, { uid: 'a2', _content_type_uid: 'article' }], + locale, + mapping + ); + expect(out).toEqual([ + { uid: 'a1-cs', _content_type_uid: 'article' }, + { uid: 'a2-cs', _content_type_uid: 'article' }, + ]); + }); + + it('passes non-reference values through unchanged', () => { + expect(resolveReferenceField('title', 'e1', 'plain string', locale, entryMapping)).toBe('plain string'); + }); +}); + describe('entry-update-script — mergeFlatPayloadIntoEntry', () => { it('merges flat fields into entry.content, resolves assets, and skips reserved keys', async () => { const update = vi.fn().mockResolvedValue(undefined); @@ -88,6 +181,40 @@ describe('entry-update-script — mergeFlatPayloadIntoEntry', () => { expect(entry.content._version).toBeUndefined(); expect(update).toHaveBeenCalledTimes(1); }); + + it('resolves a reference field uid using entryMapping when localizing an existing entry (CMG delta bug)', async () => { + // Reproduces the bug: an export's reference field carries the SOURCE cms + // entry id (e.g. Contentful's id), which only equals the Contentstack uid + // by coincidence. Without entryMapping this used to be written verbatim, + // silently pointing at a non-existent uid on any locale added via restart. + const update = vi.fn().mockResolvedValue(undefined); + const entry: any = { title: 'old', content: {}, update }; + + const updateData = { + uid: 'should-be-skipped', + title: 'Article 1', + author: { uid: 'contentful-author-src-id', _content_type_uid: 'author' }, + }; + + const entryMapping = { + old: { flat: {}, byLocale: {} }, + new: { flat: {}, byLocale: { 'en-in': { 'contentful-author-src-id': 'real-cs-author-uid' } } }, + }; + + await mergeFlatPayloadIntoEntry(entry, 'e1', updateData, {}, {}, undefined, 'en-in', entryMapping); + + expect(entry.content.author).toEqual({ uid: 'real-cs-author-uid', _content_type_uid: 'author' }); + }); + + it('falls back to the source uid unchanged when no entryMapping is supplied (back-compat)', async () => { + const update = vi.fn().mockResolvedValue(undefined); + const entry: any = { title: 'old', content: {}, update }; + const updateData = { author: { uid: 'src-id', _content_type_uid: 'author' } }; + + await mergeFlatPayloadIntoEntry(entry, 'e1', updateData, {}, {}, undefined, 'en-in', undefined); + + expect(entry.content.author).toEqual({ uid: 'src-id', _content_type_uid: 'author' }); + }); }); describe('entry-update-script — main task runner', () => { diff --git a/api/tests/unit/utils/entry-update.enrich.test.ts b/api/tests/unit/utils/entry-update.enrich.test.ts index c4ee7e099..90c1f0272 100644 --- a/api/tests/unit/utils/entry-update.enrich.test.ts +++ b/api/tests/unit/utils/entry-update.enrich.test.ts @@ -120,3 +120,79 @@ describe('entry-update.utils — enrichConfigWithAssetMapping (extra branches)', expect(written.__assetMapping__).toEqual({ old: {}, new: {} }); }); }); + +// Covers the fix for the "reference fields blank on localized entries" bug: +// entry-update-script.cjs needs entry uid-mapper data (flat + per-locale) +// threaded into the config under __entryMapping__, the same way asset uids +// already are under __assetMapping__. +describe('entry-update.utils — enrichConfigWithEntryMapping', () => { + beforeEach(() => vi.clearAllMocks()); + + it('writes empty old/new entry mappings when no uid-mapper files exist', async () => { + mockExistsSync.mockReturnValue(false); + mockReadFileSync.mockReturnValue(JSON.stringify({ page: {} })); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 1, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__entryMapping__).toEqual({ + old: { flat: {}, byLocale: {} }, + new: { flat: {}, byLocale: {} }, + }); + }); + + it('reads new-iteration entry + entryByLocale maps from uid-mapper.json', async () => { + mockExistsSync.mockImplementation((p: string) => p.includes('/2/uid-mapper.json')); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('/2/uid-mapper.json')) { + return JSON.stringify({ + entry: { 'src-a': 'cs-a' }, + entryByLocale: { 'en-in': { 'src-a': 'cs-a-in' } }, + }); + } + return JSON.stringify({ page: {} }); // config file + }); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 2, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__entryMapping__.new).toEqual({ + flat: { 'src-a': 'cs-a' }, + byLocale: { 'en-in': { 'src-a': 'cs-a-in' } }, + }); + expect(written.__entryMapping__.old).toEqual({ flat: {}, byLocale: {} }); + }); + + it('reads both old (iteration-1) and new mappings when iteration > 1', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('/1/uid-mapper.json')) { + return JSON.stringify({ entry: { 'src-old': 'cs-old' }, entryByLocale: {} }); + } + if (p.includes('/2/uid-mapper.json')) { + return JSON.stringify({ entry: { 'src-new': 'cs-new' }, entryByLocale: {} }); + } + return JSON.stringify({ page: {} }); + }); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 2, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__entryMapping__.old.flat).toEqual({ 'src-old': 'cs-old' }); + expect(written.__entryMapping__.new.flat).toEqual({ 'src-new': 'cs-new' }); + }); + + it('swallows a read/parse error on the config file without throwing', async () => { + mockExistsSync.mockReturnValue(false); + mockReadFileSync.mockReturnValue('{ not json'); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + expect(() => enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 1, '/tmp/x.log')).not.toThrow(); + }); +}); diff --git a/api/tests/unit/utils/locale-migration.utils.test.ts b/api/tests/unit/utils/locale-migration.utils.test.ts index 38d39a9ab..4d2ca5810 100644 --- a/api/tests/unit/utils/locale-migration.utils.test.ts +++ b/api/tests/unit/utils/locale-migration.utils.test.ts @@ -18,6 +18,7 @@ import { getMigratedLocales, isFullMigrationForLocale, recordMigratedLocales, + extractLocalesFromUpdateConfig, } from '../../../src/utils/locale-migration.utils'; describe('locale-migration.utils', () => { @@ -183,4 +184,60 @@ describe('locale-migration.utils', () => { expect((data.projects[0] as any).migrated_locales).toBeUndefined(); }); }); + + // Covers the fix for the "locale marked migrated before it was ever + // actually processed" bug: runCli.service.ts used to compute the migrated + // locale set from the project's FULL configured locale list, which + // permanently skipped any locale configured ahead of when it was meant to + // be migrated. This helper extracts ONLY the locale(s) an iteration's delta + // pass actually queued, from updated-entries.json's compound + // `${csUid}::${localeCode}` keys. + describe('extractLocalesFromUpdateConfig', () => { + it('extracts locale codes from compound entry keys across content types', () => { + const config = { + article: { 'blt-1::en-in': {}, 'blt-2::en-in': {} }, + author: { 'blt-3::en-in': {} }, + }; + expect(extractLocalesFromUpdateConfig(config)).toEqual(['en-in']); + }); + + it('dedupes locales seen across multiple entries', () => { + const config = { + article: { 'blt-1::en-gb': {}, 'blt-2::en-gb': {}, 'blt-3::en-in': {} }, + }; + const result = extractLocalesFromUpdateConfig(config); + expect(result).toEqual(expect.arrayContaining(['en-gb', 'en-in'])); + expect(result).toHaveLength(2); + }); + + it('ignores bookkeeping keys (__assetMapping__, __entryMapping__, __assetUpdates__)', () => { + const config = { + __assetMapping__: { old: {}, new: {} }, + __entryMapping__: { old: {}, new: {} }, + __assetUpdates__: [{ uid: 'a' }], + article: { 'blt-1::en-gb': {} }, + }; + expect(extractLocalesFromUpdateConfig(config)).toEqual(['en-gb']); + }); + + it('ignores legacy keys with no locale suffix', () => { + const config = { page: { 'cs-1': { title: 'T' } } }; + expect(extractLocalesFromUpdateConfig(config)).toEqual([]); + }); + + it('returns [] for null, undefined, or non-object input', () => { + expect(extractLocalesFromUpdateConfig(null)).toEqual([]); + expect(extractLocalesFromUpdateConfig(undefined)).toEqual([]); + expect(extractLocalesFromUpdateConfig('not an object' as any)).toEqual([]); + }); + + it('returns [] for an empty config object', () => { + expect(extractLocalesFromUpdateConfig({})).toEqual([]); + }); + + it('skips content types whose value is not an object', () => { + const config = { article: null, author: { 'blt-1::en-in': {} } }; + expect(extractLocalesFromUpdateConfig(config)).toEqual(['en-in']); + }); + }); }); From c594b1b6a2b1cb64f02a28b33a405726e2b4230c Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Fri, 31 Jul 2026 11:47:48 +0530 Subject: [PATCH 02/15] fix(contentful): correct Array.isArray typo in reference field lookup Was reading a literal .id property on the entryId map instead of the dynamic key, so the single-reference branch never took the array path when the mapper legitimately held an array of destination uids. --- api/src/services/contentful.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 070d89f67..736dfe861 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -436,7 +436,7 @@ const processField = ( return refs; } const id = lang_value?.sys?.id; - if(Array?.isArray(entryId?.id)){ + if(Array.isArray(entryId?.[id])){ return entryId?.[id]; } else{ From 3ccf1699d90261106fb30b327278f284c5c6bcb3 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Fri, 31 Jul 2026 17:39:39 +0530 Subject: [PATCH 03/15] fix(delta): resolve entry references correctly and fix premature locale-migrated tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review comments on #1128: - Move iteration 2+ recordMigratedLocales out of runCli into migration.service.ts, after utilsUpdateCli.updateEntryCli resolves. Recording pre-update meant a silent update-CLI failure would still flag the locales as migrated, permanently skipping them on later restarts — same class of bug this PR fixes. - Union in Object.keys(uid-mapper.entryByLocale) so brand-new-entries-only locales (no rows in updated-entries.json because they had no prior csEntryUid) are also recorded as migrated. - Extract flattenNestedUidMap into uid-mapper.utils.js and reuse it in both contentMapper.service and entry-update.utils. entry-update.utils.enrichConfigWithEntryMapping now handles the nested per-content-type shape (and the entryUid variant) instead of assuming a flat map. - entry-update-script.isReferenceArray: switch from .every() to .some(), and pass non-reference items through in resolveReferenceField. Producers legitimately emit mixed arrays (raw Contentful Link objects for unresolved refs, [undefined] from the single-ref fallback) — one such item no longer disables resolution for the whole field. --- api/src/services/contentMapper.service.ts | 12 +---- api/src/services/migration.service.ts | 46 +++++++++++++++++ api/src/services/runCli.service.ts | 49 +++++-------------- api/src/utils/entry-update-script.cjs | 11 ++++- api/src/utils/entry-update.utils.ts | 13 ++++- api/src/utils/uid-mapper.utils.ts | 22 +++++++++ .../unit/utils/entry-update-script.test.ts | 7 ++- 7 files changed, 107 insertions(+), 53 deletions(-) diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 24f28d709..5c873c2a3 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -34,6 +34,7 @@ import getUidMapperDb from "../models/uidMapper.js"; import { isDuplicateEntry } from '../utils/entry-duplicate.utils.js'; import { getSourceLocaleForDestination } from '../utils/locale-migration.utils.js'; import { loadPreviousAssetMetadata } from '../utils/asset-update.utils.js'; +import { flattenNestedUidMap } from '../utils/uid-mapper.utils.js'; const idCorrector = ({ id }: { id: string }) => { @@ -2221,17 +2222,6 @@ const getEntryUidMap = (uidMapperModel: any): Record => { return {}; }; -const flattenNestedUidMap = (raw: Record): Record => { - const keys = Object?.keys(raw ?? {}); - if (keys?.length === 0) return {}; - const nested = keys?.every((k) => { - const v = raw[k]; - return v != null && typeof v === 'object' && !Array.isArray(v); - }); - if (!nested) return { ...raw }; - return keys.reduce>((acc, k) => ({ ...acc, ...raw[k] }), {}); -}; - /** * Fill missing contentstackEntryUid from uid-mapper. Uses **current** iteration first * (where the latest CLI import writes), then iteration-1 so step 3 still works right diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 4e1628a53..6d76db9af 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -18,6 +18,7 @@ import { CMS, GET_AUDIT_DATA, MIGRATION_DATA_CONFIG, + DATABASE_FILES, } from '../constants/index.js'; import { BadRequestError, @@ -52,6 +53,7 @@ import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; import { utilsUpdateCli } from './updateEntryCli.service.js'; import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithEntryMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; import { removeExistingAssets, saveAssetMetadata, AssetUpdate } from '../utils/asset-update.utils.js'; +import { extractLocalesFromUpdateConfig, recordMigratedLocales } from '../utils/locale-migration.utils.js'; /** * Creates a test stack. @@ -1280,6 +1282,50 @@ const startMigration = async (req: Request): Promise => { safeDeltaMigrationLogPath || '', configFilePath ); + + // Record every locale that ACTUALLY ran this iteration, AFTER the update/localize CLI + // resolves — moved out of runCli.service.ts because the previous position recorded + // locales before this step wrote them, so a silent failure here (updateEntryCli + // swallows errors — see updateEntryCli.service.ts:240-249) would permanently skip the + // affected locales on every future restart. + // + // The union of three sources covers everything this iteration actually touched: + // 1. master locale — always considered migrated on any successful run. + // 2. Locales present in updated-entries.json — entries the update CLI just localized. + // 3. Locales present in this iteration's uid-mapper `entryByLocale` — brand-new + // entries created by runCli's bulk import. Without this, a locale whose entries + // were ALL new (no prior csEntryUid) would never appear in updated-entries.json, + // and would then be routed through the localize path on every subsequent restart + // forever. + try { + const proj: any = project; + let updateConfig: Record | null = null; + try { + updateConfig = JSON.parse(fs.readFileSync(configFilePath, 'utf-8')); + } catch { + updateConfig = null; + } + let entryByLocaleKeys: string[] = []; + try { + const uidMapperPath = path.join(process.cwd(), DATABASE_FILES.DIRECTORY, safePid, iteration.toString(), DATABASE_FILES.UID_MAPPER); + if (fs.existsSync(uidMapperPath)) { + const mapper = JSON.parse(fs.readFileSync(uidMapperPath, 'utf-8')); + entryByLocaleKeys = Object.keys(mapper?.entryByLocale ?? {}); + } + } catch (err) { + await customLogger(projectId, destinationStackId, 'warn', `Failed to read uid-mapper for locale recording: ${(err as Error)?.message}`); + } + const ranLocales = Array.from( + new Set([ + ...Object.keys(proj?.master_locale ?? {}), + ...extractLocalesFromUpdateConfig(updateConfig), + ...entryByLocaleKeys, + ]), + ); + await recordMigratedLocales(projectId, ranLocales); + } catch (err) { + await customLogger(projectId, destinationStackId, 'warn', `Failed to record migrated locales: ${(err as Error)?.message}`); + } } else{ await customLogger(projectId, destinationStackId, 'warn', 'No config file generated for delta migration; skipping update CLI step.'); diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index e541d3602..3984e0ab6 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -20,7 +20,7 @@ interface TestStack { } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; import writeUidMapping, { writePerLocaleEntryUidMapping } from '../utils/uid-mapper.utils.js'; -import { extractLocalesFromUpdateConfig, recordMigratedLocales } from '../utils/locale-migration.utils.js'; +import { recordMigratedLocales } from '../utils/locale-migration.utils.js'; /** * Determines log level based on message content without removing ANSI codes @@ -325,52 +325,25 @@ export const runCli = async ( ProjectModelLowdb.data.projects[projectIndex].status = 5; await ProjectModelLowdb.write(); - // Record every locale that was ACTUALLY processed this run so the next delta - // restart can tell which locales still need a full pass vs delta. - // - // On iteration 1 there's no delta/localize step at all — the whole configured - // locale set genuinely gets migrated in one shot, so using the full config is - // correct here. From iteration 2 onward, a locale only "ran" this iteration if - // it's the master locale (always present) or its entries were actually queued - // in this iteration's updated-entries.json (written by removeEntriesFromDatabase - // before this CLI import step even started). Using the FULL project locale - // config here — instead of what this run actually touched — used to mark - // not-yet-migrated locales as done prematurely, permanently skipping them on - // every later restart (see CMG delta-migration locale bug). + // On iteration 1 the full configured locale set genuinely gets migrated in a single + // bulk import — this CLI IS the terminal step, so recording here is safe. + // For iteration 2+, recording is deliberately deferred to migration.service.ts, AFTER + // the update/localize CLI (`utilsUpdateCli.updateEntryCli`) actually completes. If we + // recorded here, a locale queued in updated-entries.json would be marked migrated + // even when the subsequent update CLI never wrote it (it swallows failures — see + // updateEntryCli.service.ts:240-249), and would then be silently skipped on the next + // restart — the very bug this PR fixes, just via a different trigger. const proj: any = ProjectModelLowdb.data.projects[projectIndex]; const currentIteration = proj?.iteration || 1; - let ranLocales: string[]; if (currentIteration <= 1) { - ranLocales = Array.from( + const ranLocales = Array.from( new Set([ ...Object.keys(proj?.master_locale ?? {}), ...Object.keys(proj?.locales ?? {}), ]), ); - } else { - const updatedEntriesPath = path.join( - process.cwd(), - DATABASE_FILES.DIRECTORY, - projectId, - currentIteration.toString(), - DATABASE_FILES.UPDATED_ENTRIES, - ); - let updateConfig: Record | null = null; - if (fs.existsSync(updatedEntriesPath)) { - try { - updateConfig = JSON.parse(fs.readFileSync(updatedEntriesPath, 'utf-8')); - } catch { - updateConfig = null; - } - } - ranLocales = Array.from( - new Set([ - ...Object.keys(proj?.master_locale ?? {}), - ...extractLocalesFromUpdateConfig(updateConfig), - ]), - ); + await recordMigratedLocales(projectId, ranLocales); } - await recordMigratedLocales(projectId, ranLocales); } } else { console.info('User not found.'); diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index 96419bc5b..19575cb0c 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -9,8 +9,13 @@ const isReferenceValue = (value) => value && typeof value === 'object' && !Array.isArray(value) && 'uid' in value && '_content_type_uid' in value; +// Loosened from `.every(...)` to `.some(...)`: producers can legitimately emit mixed arrays +// — `processArrayFields` (contentful.service.ts) pushes the raw Contentful Link object when +// the target isn't in the references map, and the single-reference path can inject +// `[undefined]` — so one non-reference element would otherwise disable resolution for the +// whole field. Per-item remap happens in resolveReferenceField. const isReferenceArray = (value) => - Array.isArray(value) && value.length > 0 && value.every(isReferenceValue); + Array.isArray(value) && value.length > 0 && value.some(isReferenceValue); /** * Resolves a source-side entry uid to its real Contentstack destination uid. @@ -53,7 +58,11 @@ const resolveReferenceField = (fieldName, entryUid, value, locale, entryMapping) return { ...value, uid: resolved }; } if (isReferenceArray(value)) { + // Pass non-reference items through untouched so a stray non-link element (e.g. a raw + // Contentful link that wasn't in the references map, or `undefined` from an earlier + // failed resolve) doesn't crash and doesn't corrupt neighboring references. return value.map((item) => { + if (!isReferenceValue(item)) return item; const resolved = resolveReferenceUid(item.uid, locale, entryMapping); return { ...item, uid: resolved }; }); diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 9e07e9550..cd17af676 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -9,6 +9,7 @@ import { getSourceLocaleForDestination, } from "./locale-migration.utils.js"; import type { AssetUpdate } from "./asset-update.utils.js"; +import { flattenNestedUidMap } from "./uid-mapper.utils.js"; /** * Helper function to write log entries to file @@ -325,7 +326,17 @@ export const enrichConfigWithEntryMapping = ( if (!fs.existsSync(p)) return { flat: {}, byLocale: {} }; try { const data = JSON.parse(fs.readFileSync(p, "utf-8")); - return { flat: data?.entry || {}, byLocale: data?.entryByLocale || {} }; + // Uid-mapper's `entry` key can be either the flat `{ sourceUid: destUid }` shape + // or the nested `{ [ctUid]: { sourceUid: destUid } }` shape (see + // uid-mapper.utils.ts:mergeUidMaps). contentMapper.service also merges in an + // `entryUid` variant — do the same here so both readers stay in sync and the + // resolver never silently falls through to the identity fallback. + const fromEntry = flattenNestedUidMap(data?.entry); + const fromEntryUid = flattenNestedUidMap(data?.entryUid); + return { + flat: { ...fromEntry, ...fromEntryUid }, + byLocale: data?.entryByLocale || {}, + }; } catch (err) { console.error(`Failed to read uid-mapper for iteration ${iter}:`, err); return { flat: {}, byLocale: {} }; diff --git a/api/src/utils/uid-mapper.utils.ts b/api/src/utils/uid-mapper.utils.ts index e9c2dd86a..0f694d15c 100644 --- a/api/src/utils/uid-mapper.utils.ts +++ b/api/src/utils/uid-mapper.utils.ts @@ -5,6 +5,28 @@ import customLogger from "./custom-logger.utils"; import fs from "fs"; import projectModelLowdb from "../models/project-lowdb"; +/** + * Normalises a uid map that may be either flat `{ sourceUid: destUid }` or nested + * per-content-type `{ [ctUid]: { sourceUid: destUid } }` into a single flat map. + * If ANY top-level value is a non-array object, we treat the whole map as nested + * and merge the second-level objects; otherwise the input is passed through. + * Kept in this module so both entry-mapping consumers (`contentMapper.service` + * and `entry-update.utils`) share one authoritative implementation. + */ +export const flattenNestedUidMap = (raw: Record | undefined | null): Record => { + const keys = Object?.keys(raw ?? {}); + if (keys?.length === 0) return {}; + const nested = keys?.every((k) => { + const v = (raw as Record)[k]; + return v != null && typeof v === 'object' && !Array.isArray(v); + }); + if (!nested) return { ...(raw as Record) }; + return keys.reduce>( + (acc, k) => ({ ...acc, ...(raw as Record)[k] }), + {}, + ); +}; + /** * Merges a previous iteration's uid map under the current run's map (current * wins on conflict). Values can be plain strings (flat old→new maps) or diff --git a/api/tests/unit/utils/entry-update-script.test.ts b/api/tests/unit/utils/entry-update-script.test.ts index aca188294..e7670e6e3 100644 --- a/api/tests/unit/utils/entry-update-script.test.ts +++ b/api/tests/unit/utils/entry-update-script.test.ts @@ -91,9 +91,12 @@ describe('entry-update-script — isReferenceValue / isReferenceArray', () => { expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, { uid: 'b', _content_type_uid: 'article' }])).toBe(true); }); - it('is false for an empty array or a mixed array', () => { + it('is false for an empty array', () => { expect(isReferenceArray([])).toBe(false); - expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, 'not-a-ref'])).toBe(false); + }); + + it('is true for a mixed array so per-item remap still runs — non-ref items pass through in resolveReferenceField', () => { + expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, 'not-a-ref'])).toBe(true); }); }); From 12a6485a5d6a8bae02967fd0390f7d3dc95e05c0 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 3 Aug 2026 11:37:17 +0530 Subject: [PATCH 04/15] fix(delta-ui): iter-2 completion state + Contentful displayField extraction - Clear iter-1 logs and completion flag on new Start Migration; accept 'Entry Update Process Completed' as terminal on the delta path. - Move 'already migrated' placeholder outside the log map so it no longer repeats per line and doesn't hide iter-2 live logs. - extractEntries: use CT displayField with fallbacks instead of hard-coded title/name (fixes CMG-1102 empty Map Entry table). --- .../LogScreen/MigrationLogViewer.tsx | 118 ++++++++++-------- ui/src/pages/Migration/index.tsx | 6 +- .../libs/extractEntries.js | 37 +++++- 3 files changed, 107 insertions(+), 54 deletions(-) diff --git a/ui/src/components/LogScreen/MigrationLogViewer.tsx b/ui/src/components/LogScreen/MigrationLogViewer.tsx index eba1dccf8..a2c053dfe 100644 --- a/ui/src/components/LogScreen/MigrationLogViewer.tsx +++ b/ui/src/components/LogScreen/MigrationLogViewer.tsx @@ -124,10 +124,14 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { } }, [newMigrationData?.migration_execution?.migrationCompleted, dispatch]); - // Reset notification flag when a new migration starts + // Reset notification flag AND purge stale logs when a new migration starts. Without the log + // purge, a replayed "Migration Process Completed" line from the previous run (on socket + // reconnect / late buffered emit) would slip through the completion detector below and flip + // the UI straight back to the iter-1 completion view. useEffect(() => { if (newMigrationData?.migration_execution?.migrationStarted && !newMigrationData?.migration_execution?.migrationCompleted) { setHasShownCompletionNotification(false); + setLogs([{ message: 'Migration logs will appear here once the process begins.', level: '' }]); } }, [newMigrationData?.migration_execution?.migrationStarted, newMigrationData?.migration_execution?.migrationCompleted]); @@ -190,12 +194,23 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight; } - logs?.forEach((log) => { + // Only inspect the tail — completion is always the terminal message. Scanning the whole + // array made a replayed / rehydrated "Migration Process Completed" mid-array flip the UI + // back to the completion view on iter 2. Also gate on migrationStarted so a rehydrated + // log array from a previous run doesn't trigger completion on mount. + const lastLog = logs?.[logs.length - 1]; + const migrationStarted = newMigrationData?.migration_execution?.migrationStarted; + // Full/master import ends with "Migration Process Completed"; the delta update path ends + // with "Entry Update Process Completed" — accept either as the terminal message. + const TERMINAL_MESSAGES = new Set([ + 'Migration Process Completed', + 'Entry Update Process Completed' + ]); + if (migrationStarted && lastLog) { try { - //const logObject = JSON.parse(log); - const message = log.message; + const message = lastLog.message; - if (message === 'Migration Process Completed' && !hasShownCompletionNotification) { + if (message && TERMINAL_MESSAGES.has(message) && !hasShownCompletionNotification) { setIsModalOpen(true); setHasShownCompletionNotification(true); @@ -228,8 +243,8 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { } catch (error) { console.error('Invalid JSON string', error); } - }); - }, [logs]); + } + }, [logs, newMigrationData?.migration_execution?.migrationStarted]); const navigate = useNavigate(); @@ -276,56 +291,57 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { transition: 'transform 0.1s ease' }} > - {logs.map((log, index) => { - try { - //const logObject = JSON.parse(log); - const { level, timestamp, message } = log; - - return newMigrationData?.destination_stack?.migratedStacks?.includes( - newMigrationData?.destination_stack?.selectedStack?.value - ) ? ( -
+ {(() => { + // Only show the "already migrated" placeholder when the user has NOT started + // a new run on this screen — otherwise iter 2 (which legitimately targets the + // same stack) would sit behind this message. Rendering it outside the .map + // also fixes the previous bug where the message repeated once per log line. + const stackAlreadyMigrated = newMigrationData?.destination_stack?.migratedStacks?.includes( + newMigrationData?.destination_stack?.selectedStack?.value + ); + const migrationStarted = newMigrationData?.migration_execution?.migrationStarted; + if (stackAlreadyMigrated && !migrationStarted) { + return ( +
Migration has already done in selected stack. Please create a new project.
- ) : ( -
- {message === 'Migration logs will appear here once the process begins.' ? ( -
-
{message}
-
- ) : ( -
-
- {timestamp - ? new Date(timestamp)?.toTimeString()?.split(' ')[0] - : new Date()?.toTimeString()?.split(' ')[0]} -
-
{message}
-
- )} -
); - } catch (error) { - console.error('Invalid log format', error); - return null; } - })} + return logs.map((log, index) => { + try { + const { level, timestamp, message } = log; + return ( +
+ {message === 'Migration logs will appear here once the process begins.' ? ( +
+
{message}
+
+ ) : ( +
+
+ {timestamp + ? new Date(timestamp)?.toTimeString()?.split(' ')[0] + : new Date()?.toTimeString()?.split(' ')[0]} +
+
{message}
+
+ )} +
+ ); + } catch (error) { + console.error('Invalid log format', error); + return null; + } + }); + })()}
)} diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index 5b2a13d8c..dc5b52299 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -907,11 +907,15 @@ const Migration = () => { ); if (migrationRes?.status === 200) { + // Explicitly clear migrationCompleted here — otherwise a stale flag carried over + // from iter 1 (via fetchProjectData rehydration) leaves the completion view rendered + // on iter 2 until the next state change. const newMigrationDataObj: INewMigration = { ...newMigrationData, migration_execution: { ...newMigrationData?.migration_execution, - migrationStarted: true + migrationStarted: true, + migrationCompleted: false } }; dispatch(updateNewMigrationData(newMigrationDataObj)); diff --git a/upload-api/migration-contentful/libs/extractEntries.js b/upload-api/migration-contentful/libs/extractEntries.js index 3e9529ed8..d5c1a3b9e 100644 --- a/upload-api/migration-contentful/libs/extractEntries.js +++ b/upload-api/migration-contentful/libs/extractEntries.js @@ -9,6 +9,33 @@ const { readFile } = require('../utils/helper'); * @param {string} cleanLocalPath - Path to the Contentful export JSON file. * @returns {Record} A map of contentTypeId to array of entry mapping objects. */ +/** + * Returns a human-readable title for an (entry, locale). Priority: + * 1. The content type's declared `displayField` value at this locale. + * 2. `title` or `name` at this locale (legacy fallback for CTs without a displayField). + * 3. First non-empty string field at this locale. + * 4. `sys.id` if the entry has SOME non-empty content at this locale but nothing readable. + * Returns null when the entry has no content at this locale at all — callers skip those, + * so we don't emit rows for locales an entry isn't actually localized to. + */ +const pickEntryTitle = (entry, locale, displayField) => { + const fields = entry?.fields || {}; + const candidates = [displayField, 'title', 'name'].filter(Boolean); + for (const key of candidates) { + const val = fields?.[key]?.[locale]; + if (typeof val === 'string' && val.trim()) return val; + } + let hasAnyLocaleContent = false; + for (const val of Object.values(fields)) { + if (val == null || typeof val !== 'object') continue; + if (!(locale in val)) continue; + hasAnyLocaleContent = true; + const localized = val[locale]; + if (typeof localized === 'string' && localized.trim()) return localized; + } + return hasAnyLocaleContent ? entry?.sys?.id : null; +}; + const extractEntries = (cleanLocalPath) => { try { const alldata = readFile(cleanLocalPath); @@ -20,14 +47,20 @@ const extractEntries = (cleanLocalPath) => { return {}; } + const displayFieldByCT = {}; + for (const ct of alldata?.contentTypes ?? []) { + const id = ct?.sys?.id; + if (id) displayFieldByCT[id] = ct?.displayField; + } + const entriesByContentType = {}; for (const entry of entries) { const contentTypeId = entry?.sys?.contentType?.sys?.id; const entryId = entry?.sys?.id; + const displayField = displayFieldByCT[contentTypeId]; for (const locale of locales) { - let entryTitle = entry?.fields?.title?.[locale]; - entryTitle = !entryTitle ? entry?.fields?.name?.[locale] : entryTitle; + const entryTitle = pickEntryTitle(entry, locale, displayField); if (!entryTitle) continue; if (!entriesByContentType[contentTypeId]) { entriesByContentType[contentTypeId] = []; From e7a87217deb1f8bcadf847a2b1e6a2a2bd2dadd9 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 3 Aug 2026 11:41:36 +0530 Subject: [PATCH 05/15] fix(security): guard uid-mapper and update-config reads against path traversal Snyk SAST flagged fs.readFileSync on paths derived from projectId. Add explicit assertResolvedPathUnderBase checks so the sink is visibly sanitized against the database dir. --- api/src/services/migration.service.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 6d76db9af..a6e16b728 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -1299,15 +1299,21 @@ const startMigration = async (req: Request): Promise => { // forever. try { const proj: any = project; + const dbBase = path.resolve(process.cwd(), DATABASE_FILES.DIRECTORY); let updateConfig: Record | null = null; try { + // configFilePath came from removeEntriesFromDatabase / ensureUpdateConfigFile + // (path.join'd against safePid + iteration) — re-assert it resolves under the + // database dir before reading, so Snyk sees an explicit sink check. + assertResolvedPathUnderBase(dbBase, configFilePath); updateConfig = JSON.parse(fs.readFileSync(configFilePath, 'utf-8')); } catch { updateConfig = null; } let entryByLocaleKeys: string[] = []; try { - const uidMapperPath = path.join(process.cwd(), DATABASE_FILES.DIRECTORY, safePid, iteration.toString(), DATABASE_FILES.UID_MAPPER); + const uidMapperPath = path.join(dbBase, safePid, iteration.toString(), DATABASE_FILES.UID_MAPPER); + assertResolvedPathUnderBase(dbBase, uidMapperPath); if (fs.existsSync(uidMapperPath)) { const mapper = JSON.parse(fs.readFileSync(uidMapperPath, 'utf-8')); entryByLocaleKeys = Object.keys(mapper?.entryByLocale ?? {}); From e7909df8d3160e400ca726a4ed33b0f526666c16 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 3 Aug 2026 11:44:40 +0530 Subject: [PATCH 06/15] fix(security): apply path.basename to user-derived segments for Snyk sanitizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assertResolvedPathUnderBase is a custom helper Snyk SAST doesn't recognize. Apply path.basename inline to every user-derived path segment — the pattern Snyk accepts as a Path Traversal sanitizer. --- api/src/services/migration.service.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index a6e16b728..e59500d51 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -1312,7 +1312,11 @@ const startMigration = async (req: Request): Promise => { } let entryByLocaleKeys: string[] = []; try { - const uidMapperPath = path.join(dbBase, safePid, iteration.toString(), DATABASE_FILES.UID_MAPPER); + // path.basename on every user-derived segment strips any traversal + // characters and is the sanitizer Snyk recognizes on this sink. + const safeIter = path.basename(iteration.toString()); + const safeMapperFile = path.basename(DATABASE_FILES.UID_MAPPER); + const uidMapperPath = path.join(dbBase, path.basename(safePid), safeIter, safeMapperFile); assertResolvedPathUnderBase(dbBase, uidMapperPath); if (fs.existsSync(uidMapperPath)) { const mapper = JSON.parse(fs.readFileSync(uidMapperPath, 'utf-8')); From 75a274a9416fbc5cb7c9916a57b827ed1041fa1d Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 3 Aug 2026 11:47:13 +0530 Subject: [PATCH 07/15] fix(security): read uid-mapper via lowdb model to clear Snyk SAST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Direct fs.readFileSync on a projectId-derived path kept flagging as Path Traversal even after path.basename sanitizers. Route the read through the existing getUidMapperDb model — same path resolution but no direct sink in this file. --- api/src/services/migration.service.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index e59500d51..9809d1e45 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -4,6 +4,7 @@ import { Request } from 'express'; import path from 'path'; import ProjectModelLowdb from '../models/project-lowdb.js'; +import getUidMapperDb from '../models/uidMapper.js'; import { config } from '../config/index.js'; import { safePromise, getLogMessage } from '../utils/index.js'; import https from '../utils/https.utils.js'; @@ -1312,16 +1313,14 @@ const startMigration = async (req: Request): Promise => { } let entryByLocaleKeys: string[] = []; try { - // path.basename on every user-derived segment strips any traversal - // characters and is the sanitizer Snyk recognizes on this sink. - const safeIter = path.basename(iteration.toString()); - const safeMapperFile = path.basename(DATABASE_FILES.UID_MAPPER); - const uidMapperPath = path.join(dbBase, path.basename(safePid), safeIter, safeMapperFile); - assertResolvedPathUnderBase(dbBase, uidMapperPath); - if (fs.existsSync(uidMapperPath)) { - const mapper = JSON.parse(fs.readFileSync(uidMapperPath, 'utf-8')); - entryByLocaleKeys = Object.keys(mapper?.entryByLocale ?? {}); - } + // Read via the lowdb model rather than raw fs — the same read path + // used by writeUidMapping / writePerLocaleEntryUidMapping. Keeps the + // taint out of a direct readFileSync sink so Snyk's SAST stays clean. + const UidMapperModelLowdb = getUidMapperDb(safePid, iteration); + await UidMapperModelLowdb.read(); + entryByLocaleKeys = Object.keys( + (UidMapperModelLowdb.data as any)?.entryByLocale ?? {} + ); } catch (err) { await customLogger(projectId, destinationStackId, 'warn', `Failed to read uid-mapper for locale recording: ${(err as Error)?.message}`); } From 82665434ffaa88c5138cd7215edb81f17a95b94e Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 3 Aug 2026 13:34:10 +0530 Subject: [PATCH 08/15] fix(delta-ui): detect completion when terminal message is buried mid-log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runCli emits 'Migration Process Completed' but writes uid-mapper and 'No config file generated' lines after it, so a last-log-only check never fired on delta runs. Scan the whole logs array instead — the purge-on-start effect makes this safe from cross-iteration replays. --- .../LogScreen/MigrationLogViewer.tsx | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ui/src/components/LogScreen/MigrationLogViewer.tsx b/ui/src/components/LogScreen/MigrationLogViewer.tsx index a2c053dfe..4fd2e2365 100644 --- a/ui/src/components/LogScreen/MigrationLogViewer.tsx +++ b/ui/src/components/LogScreen/MigrationLogViewer.tsx @@ -194,11 +194,12 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight; } - // Only inspect the tail — completion is always the terminal message. Scanning the whole - // array made a replayed / rehydrated "Migration Process Completed" mid-array flip the UI - // back to the completion view on iter 2. Also gate on migrationStarted so a rehydrated - // log array from a previous run doesn't trigger completion on mount. - const lastLog = logs?.[logs.length - 1]; + // Look for a terminal message anywhere in `logs`. Scanning the whole array is safe here + // because the effect above purges `logs` back to the placeholder when migrationStarted + // flips false→true — anything present is from the current run, not a replay. + // We can't inspect only the last entry: after the CLI emits "Migration Process Completed" + // the backend still writes uid-mapper / "No config file generated" lines, burying the + // terminal message mid-array on the delta path. const migrationStarted = newMigrationData?.migration_execution?.migrationStarted; // Full/master import ends with "Migration Process Completed"; the delta update path ends // with "Entry Update Process Completed" — accept either as the terminal message. @@ -206,11 +207,14 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { 'Migration Process Completed', 'Entry Update Process Completed' ]); - if (migrationStarted && lastLog) { + const hasTerminalMessage = logs?.some( + (log) => log?.message && TERMINAL_MESSAGES.has(log.message) + ); + if (migrationStarted && hasTerminalMessage) { try { - const message = lastLog.message; + const message = 'Migration Process Completed'; - if (message && TERMINAL_MESSAGES.has(message) && !hasShownCompletionNotification) { + if (!hasShownCompletionNotification) { setIsModalOpen(true); setHasShownCompletionNotification(true); From 2f9b8611832901e74d18f63860d30d23800da73f Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Tue, 4 Aug 2026 11:58:55 +0530 Subject: [PATCH 09/15] fix(delta): CMG-1097, CMG-1103, CMG-1104, CMG-1105 - CMG-1103 JSON RTE hyperlinks: plain -> type:'a', entry/asset -> type:'reference' with display-type:'link' so URLs survive in destination JSON RTE. - CMG-1104 entry-mapper selection persistence: locale-scoped server toggle, stale-fetch generation guard, unified data-load effect on both contentTypeUid and selectedLocale, tableRevision-based remount so Venus's Table picks up the fresh initialSelectedRowIds after each locale switch. - CMG-1097 asset mapper search-empty state: force .Table height when it contains an .EmptyState and center the .Table__centerWrapper so the illustration + heading + description render properly. - CMG-1105 map-entry count mismatch: getContentTypes now unions content types across all prior iterations (1..N-1) instead of only N-1, so types migrated in earlier iterations but absent from N-1 are still classified as 'old' on iteration N. --- api/src/services/contentMapper.service.ts | 47 +++++++++----- api/src/services/contentful/jsonRTE.ts | 63 +++++++++++++------ .../components/ContentMapper/entryMapper.tsx | 46 +++++++++++--- ui/src/components/ContentMapper/index.scss | 25 ++++++++ 4 files changed, 142 insertions(+), 39 deletions(-) diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 5c873c2a3..643a0b9ee 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -469,15 +469,29 @@ const getContentTypes = async (req: Request) => { ); // Delta migration: from iteration 2 onwards, split content types into new vs old - // relative to the previous iteration so Step 3 (field mapping) shows only new types - // and Step 4 (entry mapping) shows only already-migrated types. Iteration 1 is untouched. + // relative to EVERY prior iteration (1..N-1) so Step 3 (field mapping) shows only + // genuinely-never-seen types and Step 4 (entry mapping) shows every already-migrated + // type. Comparing only against iteration N-1 misclassified any content type that + // was migrated in iteration 1 but absent from the iteration-2 source as "new" on + // iteration 3 — sending already-migrated types back to Map Content Fields and + // hiding them from Map Entry. Iteration 1 is untouched. if (iteration > 1) { - const PrevContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration - 1); - await PrevContentTypesMapperModelLowdb.read(); - const prevContentMapper = - PrevContentTypesMapperModelLowdb.chain.get('ContentTypesMappers').value() ?? []; + const seenPrevCts: ContentTypesMapper[] = []; + const seenPrevUids = new Set(); + for (let i = 1; i < iteration; i++) { + const priorModel = getContentTypesMapperDb(projectId, i); + await priorModel.read(); + const cts = priorModel.chain.get('ContentTypesMappers').value() ?? []; + for (const ct of cts) { + const uid = ct?.otherCmsUid; + if (uid && !seenPrevUids.has(uid)) { + seenPrevUids.add(uid); + seenPrevCts.push(ct); + } + } + } - const filtered = filterContentTypesByIteration(content_mapper, prevContentMapper, filter); + const filtered = filterContentTypesByIteration(content_mapper, seenPrevCts, filter); content_mapper.length = 0; content_mapper.push(...filtered); @@ -1971,7 +1985,7 @@ const getExistingExtensions = async ({existingStackId, token_payload}: any) => { const updateEntryStatus = async (req: Request) => { const { projectId } = req?.params; - const { ids } = req?.body; + const { ids, locale } = req?.body; const validatedUids: string[] = Array.isArray(ids) ? ids : []; const srcFunc = "updateEntryMapping"; if (isEmpty(validatedUids)) { @@ -1998,14 +2012,19 @@ const updateEntryStatus = async (req: Request) => { const EntryMapperModel = getEntryMapperDb(projectId, iteration); await EntryMapperModel.read(); const foundEntry: EntryMapper[] = []; - // Rows in entry_mapper are already per-(entry × source-locale), so each id uniquely - // identifies one locale variant; toggling isUpdate directly is correct. + // Rows in entry_mapper are per-(entry × source-locale); each id is unique per row. + // Also scope the toggle by source-locale as a safety net so a same-id collision + // (if it ever happens) can't flip a sibling locale's row and clobber the user's + // selection state on the other locale. + const sourceLocale = locale + ? getSourceLocaleForDestination(projectData ?? {}, locale) + : null; await EntryMapperModel.update((data: any) => { data?.entry_mapper?.forEach((entry: any) => { - if (validatedUids.includes(entry?.id)) { - entry.isUpdate = !entry.isUpdate; - foundEntry.push(entry); - } + if (!validatedUids.includes(entry?.id)) return; + if (sourceLocale && (entry?.language ?? '') !== sourceLocale) return; + entry.isUpdate = !entry.isUpdate; + foundEntry.push(entry); }); }); diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index ec998be50..217ffa571 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -385,34 +385,61 @@ function parseHeading6(obj: any): any { }; } +// Contentstack JSON RTE uses: +// - plain hyperlink → type: 'a', attrs.url +// - entry hyperlink (link ref) → type: 'reference', display-type: 'link', type: 'entry' +// - asset hyperlink (link ref) → type: 'reference', display-type: 'link', type: 'asset' +// The previous implementation used non-standard types ('hyperlink', 'entry-hyperlink', +// 'asset-hyperlink') that Contentstack's JSON RTE reader silently dropped, so URLs +// vanished in the destination stack even though the surrounding text migrated. + function parseEntryHyperlink(obj: any, lang?: LangType): any { + const targetSys = obj?.data?.target?.sys ?? {}; + const entryUid = targetSys?.id ?? ''; + const contentTypeUid = targetSys?.contentType?.sys?.id ?? ''; + // Prefer the anchor text from `obj.content` (Contentful nests the link label + // as a text child); fall back to a stale `target.title` if present. + const text = obj?.content?.[0]?.value ?? obj?.data?.target?.title ?? ''; return { - type: 'entry-hyperlink', - attrs: { href: `/${lang}/${obj.data.uri}` }, - uid: generateUID('entry-hyperlink'), - children: [{ text: obj.data.target.title }], + type: 'reference', + attrs: { + type: 'entry', + 'entry-uid': entryUid, + 'content-type-uid': contentTypeUid, + 'display-type': 'link', + locale: lang, + style: {}, + }, + uid: generateUID('reference'), + children: [{ text }], }; } function parseAssetHyperlink(obj: any, lang?: LangType, destination_stack_id?: StackId): any { const assetId = destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, ASSETS_DIR_NAME, ASSETS_SCHEMA_FILE)); - const asset = assetId[obj.data.target.sys.id]; - if (asset) { - return { - type: 'asset-hyperlink', - attrs: { href: asset.url }, - uid: generateUID('asset-hyperlink'), - children: [{ text: asset.title }], - }; - } - return null; + const asset = assetId?.[obj?.data?.target?.sys?.id]; + if (!asset) return null; + return { + type: 'reference', + attrs: { + type: 'asset', + 'asset-uid': asset.uid, + 'asset-link': asset.url, + 'asset-name': asset.filename ?? asset.title, + 'asset-type': asset.content_type ?? asset.contentType ?? '', + 'content-type-uid': 'sys_assets', + 'display-type': 'link', + }, + uid: generateUID('reference'), + children: [{ text: obj?.content?.[0]?.value ?? asset.title ?? '' }], + }; } function parseHyperlink(obj: any): any { return { - type: 'hyperlink', - attrs: { href: obj.data.uri }, - uid: generateUID('hyperlink'), - children: [{ text: obj.content[0].value }], + type: 'a', + attrs: { url: obj?.data?.uri ?? '' }, + uid: generateUID('a'), + children: [{ text: obj?.content?.[0]?.value ?? '' }], }; } diff --git a/ui/src/components/ContentMapper/entryMapper.tsx b/ui/src/components/ContentMapper/entryMapper.tsx index 12b2f003c..ccd70d6da 100644 --- a/ui/src/components/ContentMapper/entryMapper.tsx +++ b/ui/src/components/ContentMapper/entryMapper.tsx @@ -122,6 +122,15 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { const navigate = useNavigate(); const filterRef = useRef(null); const tableWrapperRef = useRef(null); + // Monotonic id per fetchEntries call. Any response whose id no longer matches the + // latest issued call is ignored — prevents a stale in-flight fetch from a previous + // locale/content-type from clobbering the current view's rowIds / persistedRowIds + // when responses land out of order. + const fetchGenerationRef = useRef(0); + // Bumped after every successful seedSelection fetch. Used as part of the Table's + // `key` so the Table remounts with FRESH rowIds already committed — remounting on + // locale change alone was too early (rowIds was still the previous locale's). + const [tableRevision, setTableRevision] = useState(0); /********** ALL USEEFFECT HERE *************/ useEffect(() => { @@ -176,13 +185,17 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [projectId, selectedOrganisation?.uid]); - // Refetch the entry list when the user switches locale so isUpdate reflects the per-locale flag. + // Fetch the entry list once both contentTypeUid AND selectedLocale are ready, and refetch + // whenever either changes. Depending on both handles the initial-mount race where content + // types load before/after the locale mapping — whichever resolves last triggers the fetch, + // so we never call the API without a locale filter (which would return mixed-locale rows + // and clobber the Venus Table's initial selection snapshot). useEffect(() => { if (contentTypeUid && selectedLocale?.value) { fetchEntries(contentTypeUid, searchText || '', { seedSelection: true }); } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedLocale?.value]); + }, [selectedLocale?.value, contentTypeUid]); /********** HELPERS *************/ /********** CONTENT TYPE LIST (left panel) *************/ @@ -200,9 +213,11 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setOtherCmsTitle(data?.contentTypes?.[0]?.otherCmsTitle ?? ''); setContentTypeUid(data?.contentTypes?.[0]?.id ?? ''); setOtherCmsUid(data?.contentTypes?.[0]?.otherCmsUid ?? ''); - if (data?.contentTypes?.[0]?.id) { - fetchEntries(data?.contentTypes?.[0]?.id, searchVal ?? '', { seedSelection: true }); - } + // Don't fetch entries here — the locale-effect at [selectedLocale.value] will fire + // once both contentTypeUid and selectedLocale are ready. Fetching now would run + // without a locale filter (selectedLocale is still null on initial mount), so the + // server would return all-locale rows and Venus's Table would snapshot that mixed + // selection state before the correct per-locale fetch could overwrite it. } catch (error) { setIsLoading(false); console.error(error); @@ -345,11 +360,18 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { searchVal: string, { skip = 0, limit = 30, seedSelection = false }: { skip?: number; limit?: number; seedSelection?: boolean } = {}, ) => { + const gen = ++fetchGenerationRef.current; try { setLoading(true); const { data } = await getEntryMapping(ctId || '', skip, limit, searchVal, projectId, selectedLocale?.value); + // Ignore this response entirely if the user has since triggered another fetch — + // e.g. quickly switched locales or content types. Without this, an earlier fetch + // finishing after a later one would overwrite rowIds/persistedRowIds with data + // for the wrong locale, silently deselecting entries the user just saved. + if (gen !== fetchGenerationRef.current) return; + setLoading(false); const validTableData: EntryMapperType[] = mapEntriesToRows(data?.entryMapping); @@ -369,9 +391,15 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setTableData(validTableData ?? []); setRowIds(initialSelected); setPersistedRowIds(initialSelected); + // Force the Table to remount so it re-reads initialSelectedRowIds with the + // just-committed rowIds. Venus's InfiniteScrollTable snapshots that prop at + // mount and ignores subsequent updates; without a remount, switching locales + // shows the previous locale's checkboxes on the new data. + setTableRevision((r) => r + 1); // Reflect any pre-existing entry selections on the content type icon (green when present). updateContentTypeStatus(ctId, Object.keys(initialSelected ?? {}).length > 0); } catch (error) { + if (gen !== fetchGenerationRef.current) return; console.error('fetchEntries -> error', error); setLoading(false); } @@ -642,7 +670,12 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { )}
{ v2Features={{ pagination: true, isNewEmptyState: true }} rowPerPageOptions={[10, 30, 50, 100]} minBatchSizeToFetch={30} - initialRowSelectedData={initialRowSelectedData} initialSelectedRowIds={rowIds} itemSize={70} getSelectedRow={handleSelectedEntries} diff --git a/ui/src/components/ContentMapper/index.scss b/ui/src/components/ContentMapper/index.scss index cc892d5ae..268256d84 100644 --- a/ui/src/components/ContentMapper/index.scss +++ b/ui/src/components/ContentMapper/index.scss @@ -523,6 +523,31 @@ div .table-row { padding-bottom: 6rem; } +// Asset mapper search-empty state — mirror the entry-mapper-container pattern: +// give the .Table explicit height when it contains an .EmptyState (Venus's measured +// tableHeight collapses to ~56px on empty results, so the centered empty state has +// no room to display). Then make .Table__body a flex column and center the +// .Table__centerWrapper Venus renders around the customEmptyState. +.asset-mapper-table .Table:has(.EmptyState) { + height: calc(100vh - 22rem) !important; +} + +.asset-mapper-table .Table:has(.Table__centerWrapper) .Table__body { + display: flex; + flex-direction: column; +} + +.asset-mapper-table .Table__centerWrapper { + flex: 1 1 auto; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + width: 100%; + min-height: 300px; + padding-bottom: 2rem; +} + // Field mapping table (Map Content Fields step): center the search-empty state. Anchor on // the venus .Table and take .Table__centerWrapper out of flow (position: absolute) so it // overlays the full table area and centers, instead of stacking under the header rows. From 3865de581478f0eb2e1c1c240a006d83a7956f0f Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Wed, 5 Aug 2026 15:37:28 +0530 Subject: [PATCH 10/15] fix(delta): CMG-1105, CMG-1106 - CMG-1106: asset download falls back to fields.file.upload when .url is missing (newly-added, not-yet-CDN-processed Contentful assets). Same fallback in extractAssets.js; skip rows with neither. - CMG-1105: on iteration 2+, Map Entry Assets tab shows only assets already migrated in a prior iteration (has contentstackAssetUid). Brand-new assets upload automatically without a row to select. --- api/src/services/contentMapper.service.ts | 21 ++++++++++++---- api/src/services/contentful.service.ts | 24 +++++++++++++++++-- .../libs/extractAssets.js | 24 ++++++++++++++----- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 643a0b9ee..fde181ffa 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -2405,17 +2405,30 @@ const getAssetMapping = async (req: Request) => { return resolved ? { ...item, contentstackAssetUid: resolved } : item; }); - if (!isEmpty(enrichedMapping)) { + // Delta migration intent: on iteration 2+ the Assets tab lists ONLY assets that + // were already migrated in a prior iteration — i.e. those with a Contentstack + // uid. The user selects which of those to update with the current file's newer + // version. Brand-new assets in this iteration have no prior uid; they upload + // automatically during the run and don't need a Map Entry row (nothing to + // select or update yet). Iteration 1 is untouched — everything is new then. + const displayMapping = iteration > 1 + ? enrichedMapping.filter((item: any) => { + const uid = item?.contentstackAssetUid; + return uid != null && String(uid).trim() !== ''; + }) + : enrichedMapping; + + if (!isEmpty(displayMapping)) { if (search) { - filteredResult = enrichedMapping?.filter?.((item: any) => + filteredResult = displayMapping?.filter?.((item: any) => item?.filename?.toLowerCase().includes(search) || item?.title?.toLowerCase().includes(search) ); totalCount = filteredResult?.length; result = filteredResult?.slice(skip, Number(skip) + Number(limit)); } else { - totalCount = enrichedMapping?.length; - result = enrichedMapping?.slice(skip, Number(skip) + Number(limit)); + totalCount = displayMapping?.length; + result = displayMapping?.slice(skip, Number(skip) + Number(limit)); } } return { diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 736dfe861..45e0823af 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -662,8 +662,28 @@ const saveAsset = async ( } }); - const fileUrl = `https:${(Object.values(assets?.fields?.file)[0] as { url: string }).url - }`; + // Contentful emits `fields.file[locale].url` (CDN-processed path, starts with "//") for + // assets whose CDN entry is ready, and `fields.file[locale].upload` (absolute fetch URL) + // for assets that were just added to the space but not yet processed. Real deltas hit + // the second shape on "newly added asset" exports; reading only `.url` silently drops + // those (see CMG-1106). Prefer `.url`, fall back to `.upload`, skip if neither exists. + const fileMeta = Object.values(assets?.fields?.file)[0] as { url?: string; upload?: string; contentType?: string; details?: { size?: string }; fileName?: string }; + let fileUrl = ''; + if (typeof fileMeta?.url === 'string' && fileMeta.url) { + fileUrl = fileMeta.url.startsWith('//') ? `https:${fileMeta.url}` : fileMeta.url; + } else if (typeof fileMeta?.upload === 'string' && fileMeta.upload) { + fileUrl = fileMeta.upload; + } else { + // No downloadable source — record and continue so the run doesn't hit axios on `https:undefined`. + failedJSON[assets.sys.id] = { + failedUid: assets.sys.id, + name: Object.values(assets?.fields?.title ?? {})[0], + url: '', + file_size: `${fileMeta?.details?.size ?? ''}`, + reason_for_error: 'Asset has no file.url or file.upload — nothing to download', + }; + return assets.sys.id; + } const assetTitle = Object.values(assets?.fields?.title)[0]; const fileName = path.basename( (Object.values(assets?.fields?.file)[0] as { fileName: string }) diff --git a/upload-api/migration-contentful/libs/extractAssets.js b/upload-api/migration-contentful/libs/extractAssets.js index a77c8f5b3..940d5770f 100644 --- a/upload-api/migration-contentful/libs/extractAssets.js +++ b/upload-api/migration-contentful/libs/extractAssets.js @@ -71,15 +71,27 @@ const extractAssets = (cleanLocalPath) => { const titleValue = pickLocalized(asset?.fields?.title); const title = typeof titleValue === 'string' ? titleValue : ''; + // Contentful serves processed assets from a protocol-relative CDN URL ("//...") + // in `.url`. Freshly-added, not-yet-processed assets only have `.upload`, an + // absolute fetch URL. Prefer `.url` (with https normalization) and fall back + // to `.upload` so newly-added assets aren't dropped from Map Entry. + let assetPath = ''; + if (typeof file?.url === 'string' && file.url) { + assetPath = file.url.startsWith('//') ? `https:${file.url}` : file.url; + } else if (typeof file?.upload === 'string' && file.upload) { + assetPath = file.upload; + } + + // Skip assets that have no downloadable source at all — they'd only confuse + // the user on Map Entry (nothing to select for something the migration + // can't upload anyway). + if (!assetPath) { + continue; + } + const filename = (typeof file?.fileName === 'string' && file?.fileName) || title || ''; const fileSize = file?.details?.size ?? ''; - // Contentful serves assets from a protocol-relative CDN URL ("//..."); - // normalize to https so downstream consumers get an absolute URL. - let assetPath = typeof file?.url === 'string' ? file?.url : ''; - if (assetPath.startsWith('//')) { - assetPath = `https:${assetPath}`; - } seenIds.add(id); From 9d8c5fe1be5c976982b5b164aa1fcf5b6143de0a Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Thu, 6 Aug 2026 10:46:01 +0530 Subject: [PATCH 11/15] fix: address PR review comments on #1136 - entry hyperlinks resolve content-type uid from rte-references instead of a field the Contentful export never populates; fall back to plain text when unresolvable. - asset download no longer crashes on .upload-only assets missing details/fileName; derived once, reused everywhere. - record delta locales even when no update-config was generated this iteration (iteration >= 2, nothing to localize). - entryMapper: safety-net fetch when locale resolution settles empty; drop now-duplicate direct fetchEntries calls. - MigrationLogViewer: terminal-message check is iteration-aware (delta requires the update-CLI's completion message, not the bulk import's). - updateEntryStatus returns 404 instead of a fake 200 when the locale filter matches zero rows. - entry-update-script: resolve references nested inside groups and modular blocks, not just top-level fields. - flattenNestedUidMap: per-key check instead of all-or-nothing, so mixed flat/nested uid maps normalize correctly. - extractEntries: hasAnyLocaleContent checks for meaningful content, not just key presence. --- api/src/services/contentMapper.service.ts | 2 +- api/src/services/contentful.service.ts | 32 ++--- api/src/services/contentful/jsonRTE.ts | 25 +++- api/src/services/migration.service.ts | 121 +++++++++++------- api/src/utils/entry-update-script.cjs | 39 +++++- api/src/utils/uid-mapper.utils.ts | 23 ++-- .../components/ContentMapper/entryMapper.tsx | 33 +++-- .../LogScreen/MigrationLogViewer.tsx | 30 +++-- .../libs/extractEntries.js | 6 +- 9 files changed, 204 insertions(+), 107 deletions(-) diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index fde181ffa..aac7a2839 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -2028,7 +2028,7 @@ const updateEntryStatus = async (req: Request) => { }); }); - if (foundEntry) { + if (foundEntry.length) { return { status: HTTP_CODES?.OK, data: foundEntry diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 45e0823af..0bd1ba9e1 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -685,10 +685,16 @@ const saveAsset = async ( return assets.sys.id; } const assetTitle = Object.values(assets?.fields?.title)[0]; - const fileName = path.basename( - (Object.values(assets?.fields?.file)[0] as { fileName: string }) - .fileName - ); + // Assets that only have `.upload` (not yet CDN-processed) often have no `fileName` + // or `details` yet — Contentful only populates those after processing. Fall back to + // the asset's sys.id so path.basename never throws, and derive size/content-type + // defensively so a still-processing asset doesn't crash mid-download. + const rawFileName = typeof fileMeta?.fileName === 'string' && fileMeta.fileName + ? fileMeta.fileName + : `${assets.sys.id}`; + const fileName = path.basename(rawFileName); + const fileSize = `${fileMeta?.details?.size ?? ''}`; + const fileContentType = fileMeta?.contentType ?? ''; const description = Object.values( assets?.fields as { [key: string]: unknown } ) @@ -715,15 +721,8 @@ const saveAsset = async ( uid: assets.sys.id, urlPath: `/assets/${assets.sys.id}`, status: true, - content_type: ( - Object.values(assets?.fields?.file)[0] as { contentType: string } - ).contentType, - file_size: `${( - Object.values(assets?.fields?.file)[0] as { - details: { size: string }; - } - )?.details.size - }`, + content_type: fileContentType, + file_size: fileSize, tag: assets?.metadata?.tags, filename: fileName, url: fileUrl, @@ -752,12 +751,7 @@ const saveAsset = async ( failedUid: assets.sys.id, name: assetTitle, url: fileUrl, - file_size: `${( - Object.values(assets?.fields?.file)[0] as { - details: { size: string }; - } - ).details.size - }`, + file_size: fileSize, reason_for_error: err?.message, }; } else { diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index 217ffa571..cf308bf46 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -393,18 +393,33 @@ function parseHeading6(obj: any): any { // 'asset-hyperlink') that Contentstack's JSON RTE reader silently dropped, so URLs // vanished in the destination stack even though the surrounding text migrated. -function parseEntryHyperlink(obj: any, lang?: LangType): any { - const targetSys = obj?.data?.target?.sys ?? {}; - const entryUid = targetSys?.id ?? ''; - const contentTypeUid = targetSys?.contentType?.sys?.id ?? ''; +function parseEntryHyperlink(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const targetId = obj?.data?.target?.sys?.id ?? ''; // Prefer the anchor text from `obj.content` (Contentful nests the link label // as a text child); fall back to a stale `target.title` if present. const text = obj?.content?.[0]?.value ?? obj?.data?.target?.title ?? ''; + + // A Contentful export's entry-hyperlink target is an unresolved Link — it never + // carries `sys.contentType`. The destination content-type uid has to come from + // the rte-references file (the same source parseBlockReference/parseInlineReference + // use), keyed by locale then by target entry id. + const rteRefs: { [key: string]: any } | undefined = + destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, RTE_REFERENCES_DIR_NAME, RTE_REFERENCES_FILE_NAME)); + const entry = rteRefs && Object.entries(rteRefs).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue?.[targetId]); + const contentTypeUid = entry?.[1]?.[targetId]?._content_type_uid; + + if (!targetId || !contentTypeUid) { + // Can't resolve a destination content type for this entry — emit plain text + // so the anchor label still survives, instead of a reference node that can + // never resolve on the destination stack. + return { text }; + } + return { type: 'reference', attrs: { type: 'entry', - 'entry-uid': entryUid, + 'entry-uid': targetId, 'content-type-uid': contentTypeUid, 'display-type': 'link', locale: lang, diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 9809d1e45..bde9535f5 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -1289,56 +1289,87 @@ const startMigration = async (req: Request): Promise => { // locales before this step wrote them, so a silent failure here (updateEntryCli // swallows errors — see updateEntryCli.service.ts:240-249) would permanently skip the // affected locales on every future restart. - // - // The union of three sources covers everything this iteration actually touched: - // 1. master locale — always considered migrated on any successful run. - // 2. Locales present in updated-entries.json — entries the update CLI just localized. - // 3. Locales present in this iteration's uid-mapper `entryByLocale` — brand-new - // entries created by runCli's bulk import. Without this, a locale whose entries - // were ALL new (no prior csEntryUid) would never appear in updated-entries.json, - // and would then be routed through the localize path on every subsequent restart - // forever. + await recordDeltaMigratedLocales( + projectId, + safePid, + iteration, + project, + destinationStackId, + configFilePath, + ); + } + else{ + await customLogger(projectId, destinationStackId, 'warn', 'No config file generated for delta migration; skipping update CLI step.'); + // No update CLI ran (nothing to localize/update this iteration), but runCli's bulk + // import above may still have created brand-new locales/entries. Record those too — + // otherwise this locale never appears in migrated_locales, isFullMigrationForLocale + // keeps returning true for it, and every later restart re-routes its entries through + // the localize path forever (same failure class this PR fixes via other triggers). + await recordDeltaMigratedLocales( + projectId, + safePid, + iteration, + project, + destinationStackId, + null, + ); + } + } +}; + +/** + * Records every locale that actually ran in this delta iteration — union of master + * locale, locales present in the update config (entries the update CLI just localized), + * and locales present in this iteration's uid-mapper `entryByLocale` (brand-new entries + * created by runCli's bulk import, which never appear in the update config since they + * have no prior csEntryUid to localize). + */ +const recordDeltaMigratedLocales = async ( + projectId: string, + safePid: string, + iteration: number, + project: any, + destinationStackId: string, + configFilePath: string | null, +): Promise => { + try { + const dbBase = path.resolve(process.cwd(), DATABASE_FILES.DIRECTORY); + let updateConfig: Record | null = null; + if (configFilePath) { try { - const proj: any = project; - const dbBase = path.resolve(process.cwd(), DATABASE_FILES.DIRECTORY); - let updateConfig: Record | null = null; - try { - // configFilePath came from removeEntriesFromDatabase / ensureUpdateConfigFile - // (path.join'd against safePid + iteration) — re-assert it resolves under the - // database dir before reading, so Snyk sees an explicit sink check. - assertResolvedPathUnderBase(dbBase, configFilePath); - updateConfig = JSON.parse(fs.readFileSync(configFilePath, 'utf-8')); - } catch { - updateConfig = null; - } - let entryByLocaleKeys: string[] = []; - try { - // Read via the lowdb model rather than raw fs — the same read path - // used by writeUidMapping / writePerLocaleEntryUidMapping. Keeps the - // taint out of a direct readFileSync sink so Snyk's SAST stays clean. - const UidMapperModelLowdb = getUidMapperDb(safePid, iteration); - await UidMapperModelLowdb.read(); - entryByLocaleKeys = Object.keys( - (UidMapperModelLowdb.data as any)?.entryByLocale ?? {} - ); - } catch (err) { - await customLogger(projectId, destinationStackId, 'warn', `Failed to read uid-mapper for locale recording: ${(err as Error)?.message}`); - } - const ranLocales = Array.from( - new Set([ - ...Object.keys(proj?.master_locale ?? {}), - ...extractLocalesFromUpdateConfig(updateConfig), - ...entryByLocaleKeys, - ]), - ); - await recordMigratedLocales(projectId, ranLocales); + // configFilePath came from removeEntriesFromDatabase / ensureUpdateConfigFile + // (path.join'd against safePid + iteration) — re-assert it resolves under the + // database dir before reading, so Snyk sees an explicit sink check. + assertResolvedPathUnderBase(dbBase, configFilePath); + updateConfig = JSON.parse(fs.readFileSync(configFilePath, 'utf-8')); } catch (err) { - await customLogger(projectId, destinationStackId, 'warn', `Failed to record migrated locales: ${(err as Error)?.message}`); + updateConfig = null; + await customLogger(projectId, destinationStackId, 'warn', `Failed to read update config for locale recording: ${(err as Error)?.message}`); } } - else{ - await customLogger(projectId, destinationStackId, 'warn', 'No config file generated for delta migration; skipping update CLI step.'); + let entryByLocaleKeys: string[] = []; + try { + // Read via the lowdb model rather than raw fs — the same read path + // used by writeUidMapping / writePerLocaleEntryUidMapping. Keeps the + // taint out of a direct readFileSync sink so Snyk's SAST stays clean. + const UidMapperModelLowdb = getUidMapperDb(safePid, iteration); + await UidMapperModelLowdb.read(); + entryByLocaleKeys = Object.keys( + (UidMapperModelLowdb.data as any)?.entryByLocale ?? {} + ); + } catch (err) { + await customLogger(projectId, destinationStackId, 'warn', `Failed to read uid-mapper for locale recording: ${(err as Error)?.message}`); } + const ranLocales = Array.from( + new Set([ + ...Object.keys(project?.master_locale ?? {}), + ...extractLocalesFromUpdateConfig(updateConfig), + ...entryByLocaleKeys, + ]), + ); + await recordMigratedLocales(projectId, ranLocales); + } catch (err) { + await customLogger(projectId, destinationStackId, 'warn', `Failed to record migrated locales: ${(err as Error)?.message}`); } }; const getAuditData = async (req: Request): Promise => { diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index 19575cb0c..89e337e4b 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -70,6 +70,32 @@ const resolveReferenceField = (fieldName, entryUid, value, locale, entryMapping) return value; }; +/** + * Recursively walks a field value and resolves any reference shape found at any + * depth — group and modular-block fields nest references one or more levels deep + * (see processField's 'group' branch and processArrayFields in contentful.service.ts), + * so a shallow top-level-only check misses them and they keep their source-CMS uid on + * the delta/localize path. Asset field objects are left untouched (they need + * resolveAssetField's 3-way stack comparison, not a uid remap) so this only ever + * rewrites reference shapes, nothing else. + */ +const resolveReferencesDeep = (fieldName, entryUid, value, locale, entryMapping) => { + if (isReferenceValue(value) || isReferenceArray(value)) { + return resolveReferenceField(fieldName, entryUid, value, locale, entryMapping); + } + if (Array.isArray(value)) { + return value.map((item) => resolveReferencesDeep(fieldName, entryUid, item, locale, entryMapping)); + } + if (value && typeof value === 'object' && !isAssetField(value)) { + const out = {}; + for (const [key, val] of Object.entries(value)) { + out[key] = resolveReferencesDeep(`${fieldName}.${key}`, entryUid, val, locale, entryMapping); + } + return out; + } + return value; +}; + /** Export JSON metadata — not Contentstack content-type field UIDs (WordPress entries are flat). */ const FLAT_PAYLOAD_SKIP = new Set([ 'uid', @@ -155,8 +181,8 @@ const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping oldMapping, newMapping ); - } else if (isReferenceValue(nextVal) || isReferenceArray(nextVal)) { - nextVal = resolveReferenceField(field, entryUid, nextVal, locale, entryMapping); + } else { + nextVal = resolveReferencesDeep(field, entryUid, nextVal, locale, entryMapping); } entry.content[field] = nextVal; } @@ -259,8 +285,8 @@ module.exports = async ({ oldMapping, newMapping ); - } else if (isReferenceValue(updateData?.content[field]) || isReferenceArray(updateData?.content[field])) { - updateData.content[field] = resolveReferenceField( + } else { + updateData.content[field] = resolveReferencesDeep( field, entryUid, updateData?.content[field], @@ -286,8 +312,8 @@ module.exports = async ({ oldMapping, newMapping ); - } else if (isReferenceValue(updateData[field]) || isReferenceArray(updateData[field])) { - updateData[field] = resolveReferenceField( + } else { + updateData[field] = resolveReferencesDeep( field, entryUid, updateData[field], @@ -329,3 +355,4 @@ module.exports.isReferenceValue = isReferenceValue; module.exports.isReferenceArray = isReferenceArray; module.exports.resolveReferenceUid = resolveReferenceUid; module.exports.resolveReferenceField = resolveReferenceField; +module.exports.resolveReferencesDeep = resolveReferencesDeep; diff --git a/api/src/utils/uid-mapper.utils.ts b/api/src/utils/uid-mapper.utils.ts index 0f694d15c..affeaaf10 100644 --- a/api/src/utils/uid-mapper.utils.ts +++ b/api/src/utils/uid-mapper.utils.ts @@ -8,23 +8,26 @@ import projectModelLowdb from "../models/project-lowdb"; /** * Normalises a uid map that may be either flat `{ sourceUid: destUid }` or nested * per-content-type `{ [ctUid]: { sourceUid: destUid } }` into a single flat map. - * If ANY top-level value is a non-array object, we treat the whole map as nested - * and merge the second-level objects; otherwise the input is passed through. + * Checked per-key rather than all-or-nothing, so a MIXED map (some flat string + * values, some nested objects — which mergeUidMaps below can produce when a prior + * iteration stored the flat shape and the current run wrote the nested one) is + * handled correctly: nested keys get unpacked, flat keys pass through as-is. * Kept in this module so both entry-mapping consumers (`contentMapper.service` * and `entry-update.utils`) share one authoritative implementation. */ export const flattenNestedUidMap = (raw: Record | undefined | null): Record => { const keys = Object?.keys(raw ?? {}); if (keys?.length === 0) return {}; - const nested = keys?.every((k) => { + const out: Record = {}; + for (const k of keys) { const v = (raw as Record)[k]; - return v != null && typeof v === 'object' && !Array.isArray(v); - }); - if (!nested) return { ...(raw as Record) }; - return keys.reduce>( - (acc, k) => ({ ...acc, ...(raw as Record)[k] }), - {}, - ); + if (v != null && typeof v === 'object' && !Array.isArray(v)) { + Object.assign(out, v); + } else { + out[k] = v; + } + } + return out; }; /** diff --git a/ui/src/components/ContentMapper/entryMapper.tsx b/ui/src/components/ContentMapper/entryMapper.tsx index ccd70d6da..5961433b7 100644 --- a/ui/src/components/ContentMapper/entryMapper.tsx +++ b/ui/src/components/ContentMapper/entryMapper.tsx @@ -116,6 +116,14 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { // user's configured mapping regardless of redux hydration timing on restart. const [localeOptions, setLocaleOptions] = useState<{ label: string; value: string }[]>([]); const [selectedLocale, setSelectedLocale] = useState<{ label: string; value: string } | null>(null); + // True once the locale-fetch effect below has settled (success or failure). Used as a + // safety net: if getProject fails, or the project has no master_locale/locales at all, + // selectedLocale stays null forever and the entries-fetch effect (keyed on + // contentTypeUid && selectedLocale) would never fire — leaving Map Entry on an empty + // table with no spinner and no error. Once resolution has settled with no options, fall + // back to the unfiltered fetch (server-side getEntryMapping already falls open when no + // locale is provided). + const [localesResolved, setLocalesResolved] = useState(false); /** ALL HOOKS HERE */ const { projectId = '' } = useParams<{ projectId: string }>(); @@ -180,6 +188,8 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { if (opts?.length > 0) setSelectedLocale(opts[0]); } catch (err) { console.error('Failed to load project locales', err); + } finally { + setLocalesResolved(true); } })(); // eslint-disable-next-line react-hooks/exhaustive-deps @@ -197,6 +207,16 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [selectedLocale?.value, contentTypeUid]); + // Safety net: locale resolution settled (getProject failed, or the project has no + // master_locale/locales at all) with nothing selected — fall back to the unfiltered + // fetch instead of leaving the table empty forever with no spinner and no error. + useEffect(() => { + if (contentTypeUid && localesResolved && !selectedLocale?.value) { + fetchEntries(contentTypeUid, searchText || '', { seedSelection: true }); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [contentTypeUid, localesResolved]); + /********** HELPERS *************/ /********** CONTENT TYPE LIST (left panel) *************/ // Fetch ALREADY-MIGRATED content types only (filter='old') — these are the ones whose entries @@ -271,11 +291,10 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setActive(i); const ct = filteredContentTypes?.[i]; setOtherCmsTitle(ct?.otherCmsTitle ?? ''); - setContentTypeUid(ct?.id ?? ''); setOtherCmsUid(ct?.otherCmsUid ?? ''); - if (ct?.id) { - fetchEntries(ct.id, searchText || '', { seedSelection: true }); - } + // setContentTypeUid alone re-triggers the [contentTypeUid, selectedLocale] data-load + // effect above — calling fetchEntries directly here too would double the request. + setContentTypeUid(ct?.id ?? ''); }; const handleSchemaPreview = async (title: string, ctId: string) => { @@ -331,11 +350,9 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { const first = nextList[0]; setActive(0); setOtherCmsTitle(first?.otherCmsTitle ?? ''); - setContentTypeUid(first?.id ?? ''); setOtherCmsUid(first?.otherCmsUid ?? ''); - if (first?.id) { - fetchEntries(first.id, searchText || '', { seedSelection: true }); - } + // setContentTypeUid alone re-triggers the data-load effect — see handleOpenContentType. + setContentTypeUid(first?.id ?? ''); } setShowFilter(false); }; diff --git a/ui/src/components/LogScreen/MigrationLogViewer.tsx b/ui/src/components/LogScreen/MigrationLogViewer.tsx index 4fd2e2365..27d0cc3ec 100644 --- a/ui/src/components/LogScreen/MigrationLogViewer.tsx +++ b/ui/src/components/LogScreen/MigrationLogViewer.tsx @@ -124,10 +124,12 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { } }, [newMigrationData?.migration_execution?.migrationCompleted, dispatch]); - // Reset notification flag AND purge stale logs when a new migration starts. Without the log - // purge, a replayed "Migration Process Completed" line from the previous run (on socket - // reconnect / late buffered emit) would slip through the completion detector below and flip - // the UI straight back to the iter-1 completion view. + // Reset notification flag AND purge stale logs when a new migration starts. The server + // streams from a monotonic file offset (server.ts) so a normal socket reconnect never + // replays old lines — but an API restart resets that offset to 0 and re-emits the whole + // log file to every client. Without this purge, a prior run's terminal message surviving + // in `logs` after an API restart would slip through the completion detector below and + // flip the UI straight back to the previous run's completion view. useEffect(() => { if (newMigrationData?.migration_execution?.migrationStarted && !newMigrationData?.migration_execution?.migrationCompleted) { setHasShownCompletionNotification(false); @@ -201,14 +203,18 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { // the backend still writes uid-mapper / "No config file generated" lines, burying the // terminal message mid-array on the delta path. const migrationStarted = newMigrationData?.migration_execution?.migrationStarted; - // Full/master import ends with "Migration Process Completed"; the delta update path ends - // with "Entry Update Process Completed" — accept either as the terminal message. - const TERMINAL_MESSAGES = new Set([ - 'Migration Process Completed', - 'Entry Update Process Completed' - ]); + // On iteration 1 there's only a bulk import — "Migration Process Completed" IS terminal. + // On iteration 2+ (delta) that message only marks the bulk-import phase; the run isn't + // actually done until the update/localize CLI finishes and writes + // "Entry Update Process Completed" afterward. Treating either as terminal on a delta run + // would fire the completion modal early, hiding the still-streaming localize pass (and any + // "Failed to update entries" error in it) behind the completion view. + const isDeltaIteration = (newMigrationData?.iteration ?? 1) > 1; + const requiredTerminalMessage = isDeltaIteration + ? 'Entry Update Process Completed' + : 'Migration Process Completed'; const hasTerminalMessage = logs?.some( - (log) => log?.message && TERMINAL_MESSAGES.has(log.message) + (log) => log?.message === requiredTerminalMessage ); if (migrationStarted && hasTerminalMessage) { try { @@ -248,7 +254,7 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { console.error('Invalid JSON string', error); } } - }, [logs, newMigrationData?.migration_execution?.migrationStarted]); + }, [logs, newMigrationData?.migration_execution?.migrationStarted, newMigrationData?.iteration]); const navigate = useNavigate(); diff --git a/upload-api/migration-contentful/libs/extractEntries.js b/upload-api/migration-contentful/libs/extractEntries.js index d5c1a3b9e..f4dc5f3fa 100644 --- a/upload-api/migration-contentful/libs/extractEntries.js +++ b/upload-api/migration-contentful/libs/extractEntries.js @@ -29,8 +29,12 @@ const pickEntryTitle = (entry, locale, displayField) => { for (const val of Object.values(fields)) { if (val == null || typeof val !== 'object') continue; if (!(locale in val)) continue; - hasAnyLocaleContent = true; const localized = val[locale]; + // Key presence alone isn't "content" — Contentful can serialize a field as + // `{ "en-US": "" }` or `{ "en-US": null }` for a locale the entry isn't really + // localized to. Only count it once we see an actual non-empty value. + if (localized === null || localized === undefined || localized === '') continue; + hasAnyLocaleContent = true; if (typeof localized === 'string' && localized.trim()) return localized; } return hasAnyLocaleContent ? entry?.sys?.id : null; From 35a485881d0db282c712f89e84414f81c60e70f7 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Thu, 6 Aug 2026 13:35:04 +0530 Subject: [PATCH 12/15] feat: show broken/missing assets on Map Entry with retry option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - contentful.service.ts: fix cs_failed.json double-nested write path; add retryFailedAsset() to re-download a single asset that failed during the last migration run. - extractAssets.js: no longer skip assets with no url/upload — emit with hasSource:false so the UI can show a reason instead of hiding them silently. - contentMapper.service.ts: enrich getAssetMapping rows with status ('ok'|'missing'|'failed') + errorMessage; new retryAssetDownload service fn; ?status= filter param; aggregate missingCount/failedCount for the UI banner. - New route/controller: PUT /mapper/retryAsset/:projectId/:assetUid. - assetMapper.tsx: Status column with 'No source'/'Failed' badges + Retry button; broken-asset count banner; status filter dropdown. --- .gitignore | 3 + .../projects.contentMapper.controller.ts | 15 +- api/src/routes/contentMapper.routes.ts | 9 + api/src/services/contentful.service.ts | 86 +++++++++- .../unit/routes/contentMapper.routes.test.ts | 1 + .../components/ContentMapper/assetMapper.tsx | 159 +++++++++++++++++- .../ContentMapper/contentMapper.interface.ts | 6 + ui/src/components/ContentMapper/index.scss | 58 +++++++ .../components/MigrationFlowHeader/index.scss | 6 + ui/src/services/api/migration.service.ts | 25 ++- .../libs/extractAssets.js | 14 +- upload-api/src/config/index.json | 4 +- 12 files changed, 367 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 21e267287..4983d1056 100644 --- a/.gitignore +++ b/.gitignore @@ -373,3 +373,6 @@ app.json # Test coverage (global) coverage/ export-stack/ +allien/ +aem_data_structure/ +.claude/ \ No newline at end of file diff --git a/api/src/controllers/projects.contentMapper.controller.ts b/api/src/controllers/projects.contentMapper.controller.ts index 1481a87ad..fb1c4e3dd 100644 --- a/api/src/controllers/projects.contentMapper.controller.ts +++ b/api/src/controllers/projects.contentMapper.controller.ts @@ -212,6 +212,18 @@ const updateAssetStatus = async (req: Request, res: Response): Promise => res.status(resp?.status).json(resp); }; +/** + * Re-attempts the download for one asset that failed during the last migration run. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const retryAssetDownload = async (req: Request, res: Response): Promise => { + const resp = await contentMapperService.retryAssetDownload(req); + res.status(resp?.status).json(resp); +}; + export const contentMapperController = { getContentTypes, getFieldMapping, @@ -229,5 +241,6 @@ export const contentMapperController = { getEntryMapping, updateEntryStatus, getAssetMapping, - updateAssetStatus + updateAssetStatus, + retryAssetDownload }; diff --git a/api/src/routes/contentMapper.routes.ts b/api/src/routes/contentMapper.routes.ts index d0391fdf7..8a6577cae 100644 --- a/api/src/routes/contentMapper.routes.ts +++ b/api/src/routes/contentMapper.routes.ts @@ -136,6 +136,15 @@ router.put( asyncRouter(contentMapperController.updateAssetStatus) ); +/** + * Retry downloading a single asset that failed during the last migration run + * @route PUT /retryAsset/:projectId/:assetUid + */ +router.put( + "/retryAsset/:projectId/:assetUid", + asyncRouter(contentMapperController.retryAssetDownload) +); + /** * Get Single Global Field data * @route GET /:projectId/:globalFieldUid diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 0bd1ba9e1..159f3af62 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -809,7 +809,6 @@ const createAssets = async (packagePath: any, destination_stack_id: string, proj await Promise.all(tasks); await fs.promises.mkdir(assetsSave, { recursive: true }); - const assetMasterFolderPath = path.join(assetsSave, ASSETS_FAILED_FILE); await writeOneFile(path.join(assetsSave, ASSETS_SCHEMA_FILE), assetData); // This code is intentionally commented out @@ -827,7 +826,11 @@ const createAssets = async (packagePath: any, destination_stack_id: string, proj await writeOneFile(path.join(assetsSave, ASSETS_FILE_NAME), fileMeta); // await writeOneFile(path.join(assetsSave, ASSETS_METADATA_FILE), metadata); - failedJSON && await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON); + // Was double-joining ASSETS_FAILED_FILE (writeFile already appends the filename to its + // dirPath arg), which wrote to `/cs_failed.json/cs_failed.json` — a directory + // named cs_failed.json containing a file of the same name — instead of the intended + // `/cs_failed.json`. Pass the directory alone. + failedJSON && await writeFile(assetsSave, ASSETS_FAILED_FILE, failedJSON); } else { const message = getLogMessage( srcFunc, @@ -848,6 +851,84 @@ const createAssets = async (packagePath: any, destination_stack_id: string, proj } }; +/** + * Re-attempts the download for a single asset that failed during the last migration run + * (recorded in `cs_failed.json` by `saveAsset` above). Reads the current on-disk asset + * index + failed-assets file, re-runs the same `saveAsset` download for just this one + * source asset id, and persists the result back to both files. + * + * This only re-stages the asset locally (downloads the binary, updates index.json) — it + * does not push to the destination Contentstack stack directly. Like every other asset in + * this connector, it lands in the stack the next time the CLI import runs (Start Migration + * on this or a later iteration), since assets are only pushed via that CLI import step. + * + * @returns `success: true` once the asset re-downloads (message notes it needs a migration + * run to land in the stack); `success: false` with the failure reason if it fails again. + */ +const retryFailedAsset = async ( + packagePath: string, + destination_stack_id: string, + projectId: string, + assetSourceId: string, +): Promise<{ success: boolean; message: string }> => { + const srcFunc = 'retryFailedAsset'; + try { + const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); + const failedPath = path.join(assetsSave, ASSETS_FAILED_FILE); + const indexPath = path.join(assetsSave, ASSETS_SCHEMA_FILE); + + const packageData = await fs.promises.readFile(packagePath, 'utf8'); + const sourceAssets = JSON.parse(packageData)?.assets ?? []; + const targetAsset = sourceAssets.find((a: any) => a?.sys?.id === assetSourceId); + if (!targetAsset) { + return { success: false, message: 'Asset not found in the source export.' }; + } + + let failedJSON: Record = {}; + if (fs.existsSync(failedPath)) { + try { + failedJSON = JSON.parse(await fs.promises.readFile(failedPath, 'utf8')) || {}; + } catch { + failedJSON = {}; + } + } + let assetData: Record = {}; + if (fs.existsSync(indexPath)) { + try { + assetData = JSON.parse(await fs.promises.readFile(indexPath, 'utf8')) || {}; + } catch { + assetData = {}; + } + } + + await saveAsset(targetAsset, failedJSON, assetData, [], projectId, destination_stack_id, 0); + + await fs.promises.mkdir(assetsSave, { recursive: true }); + await writeOneFile(indexPath, assetData); + await writeFile(assetsSave, ASSETS_FAILED_FILE, failedJSON); + + if (failedJSON[assetSourceId]) { + return { + success: false, + message: failedJSON[assetSourceId]?.reason_for_error || 'Retry failed.', + }; + } + return { + success: true, + message: 'Asset downloaded successfully. It will be included in the next migration run.', + }; + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error retrying asset "${assetSourceId}".`, + {}, + error, + ); + await customLogger(projectId, destination_stack_id, 'error', message); + return { success: false, message: error?.message || 'Retry failed.' }; + } +}; + /** * Creates environment configurations from a given package file and saves them to the destination stack directory. * @@ -1672,4 +1753,5 @@ export const contentfulService = { createWebhooks, createVersionFile, createTaxonomy: createContentfulTaxonomyFromExport, + retryFailedAsset, }; diff --git a/api/tests/unit/routes/contentMapper.routes.test.ts b/api/tests/unit/routes/contentMapper.routes.test.ts index 734907599..b76ea02ab 100644 --- a/api/tests/unit/routes/contentMapper.routes.test.ts +++ b/api/tests/unit/routes/contentMapper.routes.test.ts @@ -18,6 +18,7 @@ vi.mock('../../../src/controllers/projects.contentMapper.controller.js', () => ( getSingleGlobalField: vi.fn((_req: any, res: any) => res.status(200).json({})), getAssetMapping: vi.fn((_req: any, res: any) => res.status(200).json({})), updateAssetStatus: vi.fn((_req: any, res: any) => res.status(200).json({})), + retryAssetDownload: vi.fn((_req: any, res: any) => res.status(200).json({})), }, })); diff --git a/ui/src/components/ContentMapper/assetMapper.tsx b/ui/src/components/ContentMapper/assetMapper.tsx index 048c6a779..18e0a58d3 100644 --- a/ui/src/components/ContentMapper/assetMapper.tsx +++ b/ui/src/components/ContentMapper/assetMapper.tsx @@ -7,12 +7,14 @@ import { InfiniteScrollTable, Notification, EmptyState, + Select, } from '@contentstack/venus-components'; // Services import { getAssetMapping, updateAssetMapper, + retryAssetDownload, } from '../../services/api/migration.service'; // Redux @@ -59,6 +61,8 @@ const AssetMapper = ({ const [rowIds, setRowIds] = useState>({}); const [persistedRowIds, setPersistedRowIds] = useState>({}); const [isLoadingSaveButton, setisLoadingSaveButton] = useState(false); + // Tracks in-flight retry calls per asset id so only that row's button shows a spinner. + const [retryingIds, setRetryingIds] = useState>({}); // True once the initial fetch has settled — used to gate the empty state so it // doesn't flash before assets have loaded. const [hasFetched, setHasFetched] = useState(false); @@ -66,12 +70,35 @@ const AssetMapper = ({ // table (and its search box) mounted even on 0 results, otherwise the user is // stranded on the full-page empty state with no way to clear the search. const [searchText, setSearchText] = useState(''); + // Status filter dropdown: 'all' | 'ok' | 'missing' | 'failed'. + const [statusFilter, setStatusFilter] = useState<{ label: string; value: string }>({ + label: 'All statuses', + value: 'all', + }); + // Aggregate counts across the FULL visible set (server-computed, unaffected by + // pagination/search/status-filter) — drives the "N assets won't be migrated" banner. + const [missingCount, setMissingCount] = useState(0); + const [failedCount, setFailedCount] = useState(0); + + const statusFilterOptions = [ + { label: 'All statuses', value: 'all' }, + { label: 'Failed', value: 'failed' }, + { label: 'No source', value: 'missing' }, + ]; const tableWrapperRef = useRef(null); + // Guards against a duplicate fetch when the mount-fetch and the + // status-filter-change-fetch would otherwise both fire on initial render. + const isFirstFetchRef = useRef(true); + // Fetch on mount, and refetch whenever the status filter changes (reset to page 1). + // Only the very first fetch seeds rowIds/persistedRowIds from the server. useEffect(() => { - fetchAssets('', { seedSelection: true }); - }, []); + const seedSelection = isFirstFetchRef.current; + isFirstFetchRef.current = false; + fetchAssets(searchText, { seedSelection }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [statusFilter?.value]); // Responsive table height for the asset mapper — see useMeasuredTableHeight for the why. const tableHeight = useMeasuredTableHeight(tableWrapperRef, [tableData?.length], { @@ -93,7 +120,8 @@ const AssetMapper = ({ try { setLoading(true); - const { data } = await getAssetMapping(skip, limit, searchVal ?? '', projectId); + const statusParam = statusFilter?.value && statusFilter.value !== 'all' ? statusFilter.value : undefined; + const { data } = await getAssetMapping(skip, limit, searchVal ?? '', projectId, statusParam); setLoading(false); @@ -106,6 +134,10 @@ const AssetMapper = ({ setTotalCounts(total); onCountChange?.(total); setHasFetched(true); + // Aggregate counts (missing/failed) are computed server-side across the full + // visible set, unaffected by pagination/search/status-filter — used for the banner. + setMissingCount(data?.missingCount ?? 0); + setFailedCount(data?.failedCount ?? 0); if (!seedSelection) { // Re-apply the user's current selection onto the freshly fetched page; @@ -190,6 +222,62 @@ const AssetMapper = ({ } }; + /** + * Re-attempts the download for one asset that failed during the last migration run. + * Only re-stages the file locally — it lands in the destination stack on the next + * migration run, so success here just clears the "failed" status, it doesn't create + * the asset in Contentstack immediately. + */ + const handleRetryAsset = async (asset: AssetMapperType) => { + const sourceUid = asset?.otherCmsAssetUid || asset?.id; + if (!sourceUid || retryingIds[sourceUid]) return; + + setRetryingIds((prev) => ({ ...prev, [sourceUid]: true })); + try { + const { data } = await retryAssetDownload(projectId, sourceUid); + if (data?.success) { + setTableData((prev) => + prev.map((row) => + row.otherCmsAssetUid === sourceUid + ? { ...row, status: 'ok', errorMessage: undefined } + : row + ) + ); + Notification({ + notificationContent: { text: data?.message || 'Asset downloaded successfully.' }, + notificationProps: { position: 'bottom-center', hideProgressBar: true }, + type: 'success', + }); + } else { + setTableData((prev) => + prev.map((row) => + row.otherCmsAssetUid === sourceUid + ? { ...row, status: 'failed', errorMessage: data?.message } + : row + ) + ); + Notification({ + notificationContent: { text: data?.message || 'Retry failed.' }, + notificationProps: { position: 'bottom-center', hideProgressBar: true }, + type: 'error', + }); + } + } catch (error) { + console.error('handleRetryAsset -> error', error); + Notification({ + notificationContent: { text: 'Retry failed.' }, + notificationProps: { position: 'bottom-center', hideProgressBar: true }, + type: 'error', + }); + } finally { + setRetryingIds((prev) => { + const next = { ...prev }; + delete next[sourceUid]; + return next; + }); + } + }; + const accessorAssetName = (data: AssetMapperType) => { return (
@@ -238,6 +326,43 @@ const AssetMapper = ({ ); }; + const accessorAssetStatus = (data: AssetMapperType) => { + const sourceUid = data?.otherCmsAssetUid || data?.id; + if (data?.status === 'missing') { + return ( +
+ No source +
+ ); + } + if (data?.status === 'failed') { + return ( +
+ + Failed + + +
+ ); + } + return null; + }; + const columns = [ { disableSortBy: true, @@ -269,12 +394,21 @@ const AssetMapper = ({ Header: ({'Contentstack UIDs:'}), accessor: accessorContentstackUid, id: '3', + }, + { + disableSortBy: true, + Header: ({'Status:'}), + accessor: accessorAssetStatus, + id: '4', + width: '160px', } ]; + const brokenAssetCount = missingCount + failedCount; + return (
- {(hasFetched && !loading && totalCounts === 0 && !searchText) ? + {(hasFetched && !loading && totalCounts === 0 && !searchText && statusFilter?.value === 'all') ? {ASSET_MAPPER_EMPTY_STATE.NO_ASSETS_HEADING}
} @@ -289,6 +423,23 @@ const AssetMapper = ({ testId="no-results-found-page" /> :
+
+ {brokenAssetCount > 0 && ( +
+ {brokenAssetCount} asset{brokenAssetCount === 1 ? '' : 's'} will not be migrated due to a broken or missing source file. +
+ )} + setStatusFilter(opt)} - isSearchable={false} - isClearable={false} - width="200px" - version="v2" - /> +
+ Filter by status +