diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts index 6313f32cc..072247532 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts @@ -6,6 +6,7 @@ import { ChangeTrackerConfig } from "."; interface TableRequiresUpdateOptions { table: Table; + metadata: TableMetadata; config: ChangeTrackerConfig; documentIdColExists: boolean; pathParamsColExists: boolean; @@ -14,13 +15,13 @@ interface TableRequiresUpdateOptions { export async function tableRequiresUpdate({ table, + metadata, config, documentIdColExists, pathParamsColExists, oldDataColExists, }: TableRequiresUpdateOptions): Promise { /* Setup checks */ - const { metadata } = table; /** Check clustering */ const configCluster = JSON.stringify(config.clustering); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts index 7b6f8a15e..058a90aca 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts @@ -341,14 +341,13 @@ export class FirestoreBigQueryEventHistoryTracker await clustering.updateClustering(metadata); - const documentIdColExists = fields.find( + const documentIdColExists = fields.some( (column) => column.name === "document_id" ); - const pathParamsColExists = fields.find( + const pathParamsColExists = fields.some( (column) => column.name === "path_params" ); - - const oldDataColExists = fields.find( + const oldDataColExists = fields.some( (column) => column.name === "old_data" ); @@ -372,6 +371,7 @@ export class FirestoreBigQueryEventHistoryTracker /** Updated table metadata if required */ const shouldUpdate = await tableRequiresUpdate({ table, + metadata, config: this.config, documentIdColExists, pathParamsColExists, diff --git a/firestore-bigquery-export/scripts/import/src/config.ts b/firestore-bigquery-export/scripts/import/src/config.ts index 2c956374f..61746d7b4 100644 --- a/firestore-bigquery-export/scripts/import/src/config.ts +++ b/firestore-bigquery-export/scripts/import/src/config.ts @@ -1,4 +1,4 @@ -import * as program from "commander"; +import { program } from "commander"; import filenamify from "filenamify"; import inquirer from "inquirer"; @@ -14,6 +14,12 @@ const PROJECT_ID_MAX_CHARS = 6144; export const FIRESTORE_COLLECTION_NAME_MAX_CHARS = 6144; const BIGQUERY_RESOURCE_NAME_MAX_CHARS = 1024; +const VALID_VIEW_TYPES = [ + "view", + "materialized_incremental", + "materialized_non_incremental", +]; + const validateBatchSize = (value: string) => { return parseInt(value, 10) > 0; }; @@ -53,6 +59,15 @@ const validateLocation = (value: string) => { return index !== -1; }; +function parseClustering(value?: string): string[] | undefined { + if (value === undefined) return undefined; + if (value.trim() === "") return []; + return value + .split(",") + .map((field) => field.trim()) + .filter(Boolean); +} + export const validateInput = ( value: string, name: string, @@ -204,6 +219,21 @@ const questions = [ name: "failedBatchOutput", type: "input", }, + { + message: + "What type of latest view is used by the deployed extension? (view, materialized_incremental, materialized_non_incremental)", + name: "viewType", + type: "list", + choices: VALID_VIEW_TYPES, + default: "view", + }, + { + message: + "What clustering fields should be preserved on the raw changelog table? (Comma-separated, leave blank for none)", + name: "clustering", + type: "input", + default: "", + }, ]; export async function parseConfig(): Promise { @@ -248,6 +278,15 @@ export async function parseConfig(): Promise { errors.push("Invalid batch size."); } + if ( + program.viewType !== undefined && + !VALID_VIEW_TYPES.includes(program.viewType) + ) { + errors.push( + "ViewType must be one of: view, materialized_incremental, materialized_non_incremental." + ); + } + if (errors.length !== 0) { program.outputHelp(); return { kind: "ERROR", errors }; @@ -279,6 +318,8 @@ export async function parseConfig(): Promise { failedBatchOutput: program.failedBatchOutput, transformFunctionUrl: program.transformFunctionUrl, firestoreInstanceId: program.firestoreInstanceId || "(default)", + clustering: parseClustering(program.clustering), + viewType: program.viewType || "view", }; } const { @@ -296,6 +337,8 @@ export async function parseConfig(): Promise { useEmulator, failedBatchOutput, transformFunctionUrl, + viewType, + clustering, } = await inquirer.prompt(questions); const rawChangeLogName = `${table}_raw_changelog`; @@ -324,6 +367,8 @@ export async function parseConfig(): Promise { failedBatchOutput, transformFunctionUrl, firestoreInstanceId: firestoreInstanceId || "(default)", + clustering: parseClustering(clustering), + viewType: viewType || "view", }; } diff --git a/firestore-bigquery-export/scripts/import/src/index.ts b/firestore-bigquery-export/scripts/import/src/index.ts index e28abaa74..a16665b9c 100644 --- a/firestore-bigquery-export/scripts/import/src/index.ts +++ b/firestore-bigquery-export/scripts/import/src/index.ts @@ -52,6 +52,8 @@ const run = async (): Promise => { cursorPositionFile, transformFunctionUrl, firestoreInstanceId, + clustering, + viewType, } = config; if (useEmulator) { console.log("Using emulator"); @@ -94,6 +96,11 @@ const run = async (): Promise => { bqProjectId: bigQueryProjectId, transformFunction: transformFunctionUrl, firestoreInstanceId: firestoreInstanceId, + clustering, + useMaterializedView: + viewType === "materialized_incremental" || + viewType === "materialized_non_incremental", + useIncrementalMaterializedView: viewType === "materialized_incremental", }); await initializeDataSink(dataSink, config); diff --git a/firestore-bigquery-export/scripts/import/src/program.ts b/firestore-bigquery-export/scripts/import/src/program.ts index 24f4df2c8..492b3acdc 100644 --- a/firestore-bigquery-export/scripts/import/src/program.ts +++ b/firestore-bigquery-export/scripts/import/src/program.ts @@ -1,4 +1,4 @@ -import * as program from "commander"; +import { program } from "commander"; const packageJson = require("../package.json"); @@ -70,5 +70,14 @@ export const getCLIOptions = () => { "--firestore-instance-id ", "The Firestore database instance ID. Use '(default)' for the default database.", "(default)" + ) + .option( + "--view-type ", + "View type used by the deployed extension: view, materialized_incremental, materialized_non_incremental", + "view" + ) + .option( + "--clustering ", + "Comma-separated clustering fields for the raw changelog table, e.g. timestamp or timestamp,document_id" ); }; diff --git a/firestore-bigquery-export/scripts/import/src/types.ts b/firestore-bigquery-export/scripts/import/src/types.ts index 2c384000b..2f947cb4e 100644 --- a/firestore-bigquery-export/scripts/import/src/types.ts +++ b/firestore-bigquery-export/scripts/import/src/types.ts @@ -18,6 +18,12 @@ export interface CliConfig { failedBatchOutput?: string; transformFunctionUrl?: string; firestoreInstanceId: string; + + clustering?: string[] | null; + viewType?: + | "view" + | "materialized_incremental" + | "materialized_non_incremental"; } export interface CliConfigError {