Skip to content

refactor(build): one BuildPlan pipeline for git and local builds #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
metalwarrior665 wants to merge 1 commit into from
Closed

refactor(build): one BuildPlan pipeline for git and local builds #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
228 changes: 114 additions & 114 deletions .github/cli/apify-test-tools.mjs
View file
Open in desktop

Large diffs are not rendered by default.

70 changes: 29 additions & 41 deletions bin/build-from-local.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@ import os from 'node:os';
import path from 'node:path';

import AdmZip from 'adm-zip';
import type * as ApifyClientTypes from 'apify-client';
import { ActorSourceType } from 'apify-client';

import { ApifyBuilder, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
import { type BuildPlan, executeBuildPlans, loadActor, LOCAL_SOURCE_VERSION_NUMBER } from './build.js';
import { loadDockerIgnore } from './dockerignore.js';
import { logger } from './logger.js';
import { isPathWithinScope } from './path-utils.js';
Expand Down Expand Up @@ -179,7 +178,7 @@ export const rewriteActorJsonPaths = async (
await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4));
};

type LocalBuildPlan = {
type LocalBuildSummary = {
actorFullName: string;
versionNumber: string;
folder: string;
Expand All @@ -193,52 +192,41 @@ export const runBuildsFromLocal = async ({
}: {
actorConfigs: ActorConfig[];
dryRun: boolean;
}): Promise<BuildData[] | DryRunPlan<LocalBuildPlan>> => {
// Read-only preparation shared with the real build, so a dry run fails on a missing token, an unknown
}): Promise<BuildData[] | DryRunPlan<LocalBuildSummary>> => {
// Everything but the upload is prepared for a dry run too, so it fails on a missing token, an unknown
// Actor or unreadable sources just like the build would
const preparedBuilds = await Promise.all(
actorConfigs.map(async (actorConfig) => {
const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
const plans = await Promise.all(
actorConfigs.map(async (actorConfig): Promise<BuildPlan<LocalBuildSummary>> => {
const { builder, actorInfo } = await loadActor(actorConfig);
// Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
const zip = new AdmZip();
const sourceFiles = await collectSourceFiles(actorConfig);
for (const { name, content } of sourceFiles) {
zip.addFile(name, content);
}
return { actorConfig, actorInfo, fileCount: sourceFiles.length, zipBuffer: zip.toBuffer() };
}),
);

if (dryRun) {
logger.info('[DRY RUN] Would build from local source:');
for (const { actorConfig, fileCount, zipBuffer } of preparedBuilds) {
logger.info(
` ${actorConfig.actorFullName} (${actorConfig.folder}): ${fileCount} files, ${zipBuffer.length} bytes zipped`,
);
}
return {
dryRun: true,
builds: preparedBuilds.map(({ actorConfig, fileCount, zipBuffer }) => ({
actorFullName: actorConfig.actorFullName,
const zipBuffer = zip.toBuffer();
const { actorFullName, folder } = actorConfig;
return {
builder,
actorInfo,
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
folder: actorConfig.folder.path,
fileCount,
zipSizeBytes: zipBuffer.length,
})),
};
}

const preparedByActorFullName = new Map(
preparedBuilds.map((prepared) => [prepared.actorConfig.actorFullName, prepared]),
useCache: false,
summary: {
actorFullName,
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
folder: folder.path,
fileCount: sourceFiles.length,
zipSizeBytes: zipBuffer.length,
},
description: `${actorFullName} (${folder}): ${sourceFiles.length} files, ${zipBuffer.length} bytes zipped`,
toActorVersion: async () => ({
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
tarballUrl: await builder.uploadSourceZip(actorInfo.id, LOCAL_SOURCE_VERSION_NUMBER, zipBuffer),
sourceType: ActorSourceType.Tarball,
}),
};
}),
);

return runAndSummarizeBuilds(actorConfigs, 'LOCAL BUILDS', async (actorConfig, builder) => {
const { actorInfo, zipBuffer } = preparedByActorFullName.get(actorConfig.actorFullName)!;
const actorVersion: ApifyClientTypes.ActorVersion = {
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zipBuffer),
sourceType: ActorSourceType.Tarball,
};
return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false, actorInfo);
});
return executeBuildPlans(plans, { label: 'LOCAL BUILDS', dryRun });
};
172 changes: 81 additions & 91 deletions bin/build.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,6 @@ import { getCurrentBranch, normalizeRepoUrl, resolveRepoUrl } from './git.js';
import { logger } from './logger.js';
import type { ActorConfig, BuildData, DryRunPlan } from './types.js';

type BuildPrActorOptions = {
versionNumber: string;
gitRepoUrl: string;
actorConfig: ActorConfig;
actorInfo: ApifyClientTypes.Actor;
};

// Fixed version number used to build from local source files, since there is no real version to track.
export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
// Stands for each Actor's default version: the one its default build tag points to
Expand Down Expand Up @@ -98,15 +91,10 @@ export class ApifyBuilder {
return actorInfo;
};

getDefaultVersionAndTag = async () => resolveDefaultVersion(this.actorFullName, await this.getActorInfo());

// Mirrors `apify push`: stores the zip in the Actor's `actor-<id>-source` KV store and returns a signed
// record URL the builder can download without a token.
uploadSourceZip = async (versionNumber: string, zip: Buffer): Promise<string> => {
const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
if (!actorInfo) throw new Error(`No actor named '${this.actorFullName}' was found on the platform.`);

const store = await this.apifyClient.keyValueStores().getOrCreate(`actor-${actorInfo.id}-source`);
uploadSourceZip = async (actorId: string, versionNumber: string, zip: Buffer): Promise<string> => {
const store = await this.apifyClient.keyValueStores().getOrCreate(`actor-${actorId}-source`);
const storeClient = this.apifyClient.keyValueStore(store.id);
const key = `version-${versionNumber}.zip`;
await storeClient.setRecord({ key, value: zip as never, contentType: 'application/zip' });
Expand All @@ -116,15 +104,13 @@ export class ApifyBuilder {
return url.toString();
};

// Pass actorInfo when the caller already fetched it, to save an API call
createVersionAndBuild = async (
versionNumber: string,
actorVersion: ApifyClientTypes.ActorVersion,
useCache: boolean,
actorInfo?: ApifyClientTypes.Actor,
{ versions }: ApifyClientTypes.Actor,
): Promise<BuildData> => {
const actorClient = this.apifyClient.actor(this.actorFullName);
const { versions } = actorInfo ?? (await this.getActorInfo());

// Prepare version
const versionMissing = !versions.find((version) => version.versionNumber === versionNumber);
Expand Down Expand Up @@ -272,50 +258,71 @@ export class ApifyBuilder {
}
}

export const waitAndSummarizeBuilds = async (
startedBuilds: BuildData[],
buildersMap: Map<string, ApifyBuilder>,
label: string,
): Promise<BuildData[]> => {
/**
* One Actor build with everything read and checked and nothing written yet. Each way of building (from Git, from
* local files) only prepares plans; `executeBuildPlans` is the single place that starts, awaits and summarizes them,
* so a dry run reports exactly what the real run would start.
*/
export interface BuildPlan<Summary extends { actorFullName: string }> {
builder: ApifyBuilder;
actorInfo: ApifyClientTypes.Actor;
versionNumber: string;
useCache: boolean;
/** What a dry run reports as JSON for this build */
summary: Summary;
/** What a dry run logs for this build */
description: string;
/** The version to create or update. Runs for real builds only, since it may write (e.g. upload the sources). */
toActorVersion: () => Promise<ApifyClientTypes.ActorVersion>;
}

/** Fetches the Actor once, for the checks and for the plan's build. Fails early on a missing token or Actor. */
export const loadActor = async (actorConfig: ActorConfig) => {
const builder = ApifyBuilder.fromActorConfig(actorConfig);
return { builder, actorInfo: await builder.getActorInfo() };
};

/**
* Starts the planned builds, waits for all of them and fails if any fails. Used by every command that builds.
* A dry run starts nothing and returns the plans' summaries instead, in a shape that can't pass for real builds.
*/
export const executeBuildPlans = async <Summary extends { actorFullName: string }>(
plans: BuildPlan<Summary>[],
{ label, dryRun }: { label: string; dryRun: boolean },
): Promise<BuildData[] | DryRunPlan<Summary>> => {
if (dryRun) {
logger.info(`[DRY RUN] Would build (${label}):`);
for (const { description } of plans) {
logger.info(` ${description}`);
}
return { dryRun: true, builds: plans.map(({ summary }) => summary) };
}

logger.info('=========================================');
logger.info(`FINISHED ${label}:`);
await Promise.all(
startedBuilds.map(async (buildData) => {
const builder = buildersMap.get(buildData.actorFullName)!;
await builder.waitForBuildToFinish(buildData.buildId);
}),
logger.info(`STARTED ${label}:`);
const started = await Promise.all(
plans.map(async ({ builder, actorInfo, versionNumber, useCache, toActorVersion }) => ({
builder,
build: await builder.createVersionAndBuild(versionNumber, await toActorVersion(), useCache, actorInfo),
})),
);

logger.info('=========================================');
logger.info(`FINISHED ${label}:`);
await Promise.all(started.map(async ({ builder, build }) => builder.waitForBuildToFinish(build.buildId)));

logger.info('=========================================');
logger.info('SUMMARY:');
for (const buildData of startedBuilds.sort((a, b) => a.actorFullName.localeCompare(b.actorFullName))) {
logger.info(`[${buildData.actorFullName}]: ${buildData.buildNumber}`);
const builds = started.map(({ build }) => build).sort((a, b) => a.actorFullName.localeCompare(b.actorFullName));
for (const { actorFullName, buildNumber } of builds) {
logger.info(`[${actorFullName}]: ${buildNumber}`);
}
logger.info('=========================================');

return startedBuilds;
return builds;
};

export const runAndSummarizeBuilds = async (
actorConfigs: ActorConfig[],
label: string,
buildOneActor: (actorConfig: ActorConfig, builder: ApifyBuilder) => Promise<BuildData>,
): Promise<BuildData[]> => {
const buildersByActorFullName = new Map<string, ApifyBuilder>(
actorConfigs.map((actorConfig) => [actorConfig.actorFullName, ApifyBuilder.fromActorConfig(actorConfig)]),
);
logger.info('=========================================');
logger.info(`STARTED ${label}:`);
const startedBuilds = await Promise.all(
actorConfigs.map(async (actorConfig) =>
buildOneActor(actorConfig, buildersByActorFullName.get(actorConfig.actorFullName)!),
),
);

return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, label);
};

type GitBuildPlan = {
type GitBuildSummary = {
actorFullName: string;
versionNumber: string;
gitRepoUrl: string;
Expand All @@ -339,71 +346,54 @@ export const runBuilds = async ({
actorConfigs,
versionNumber,
dryRun,
}: RunBuildsOptions): Promise<BuildData[] | DryRunPlan<GitBuildPlan>> => {
// Fetched once per Actor and reused for the checks, the version numbers and the build itself
const buildConfigs: BuildPrActorOptions[] = await Promise.all(
actorConfigs.map(async (actorConfig) => {
const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
}: RunBuildsOptions): Promise<BuildData[] | DryRunPlan<GitBuildSummary>> => {
const plans = await Promise.all(
actorConfigs.map(async (actorConfig): Promise<BuildPlan<GitBuildSummary>> => {
const { actorFullName } = actorConfig;
const { builder, actorInfo } = await loadActor(actorConfig);
const { defaultVersionNumber, defaultBuildTag, defaultVersion } = resolveDefaultVersion(
actorConfig.actorFullName,
actorFullName,
actorInfo,
);
// Builds keep the tags their version gives them, so a default version without the default build tag
// would build fine but leave users on the previous build
if (versionNumber === DEFAULT_VERSION_KEYWORD && defaultVersion?.buildTag !== defaultBuildTag) {
logger.warn(
`[${actorConfig.actorFullName}] Default version ${defaultVersionNumber} has build tag ` +
`[${actorFullName}] Default version ${defaultVersionNumber} has build tag ` +
`"${defaultVersion?.buildTag ?? 'none'}", not "${defaultBuildTag}", so users keep running the previous build. ` +
'Set the build tag of the version on the platform to change that.',
);
}
if (shouldVerifyRepoUrl) {
assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
assertRepoUrlMatchesDefaultVersion(actorFullName, defaultVersion, repoUrl);
}

// Depending on if these are miniactors or standaloneActors
let gitRepoUrl = `${repoUrl}#${branch}`;
if (!actorConfig.folder.isRoot()) {
gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
}
const targetVersionNumber =
versionNumber === DEFAULT_VERSION_KEYWORD ? defaultVersionNumber : versionNumber;
return {
actorConfig,
builder,
actorInfo,
gitRepoUrl,
versionNumber: versionNumber === DEFAULT_VERSION_KEYWORD ? defaultVersionNumber : versionNumber,
versionNumber: targetVersionNumber,
// Always with the Docker cache: CI always asked for it, and a clean build can still be started on the platform
useCache: true,
summary: { actorFullName, versionNumber: targetVersionNumber, gitRepoUrl },
description: `${actorFullName} (${targetVersionNumber}) from ${gitRepoUrl}`,
// No buildTag: the version keeps the one it has, so its build gets the tag the version already gives it
toActorVersion: async () => ({
versionNumber: targetVersionNumber,
gitRepoUrl,
sourceType: ActorSourceType.GitRepo,
}),
};
}),
);

// Everything above only reads, so a dry run has checked what the real run would, short of writing
if (dryRun) {
logger.info('[DRY RUN] Would build:');
const builds = buildConfigs.map((buildConfig) => ({
actorFullName: buildConfig.actorConfig.actorFullName,
versionNumber: buildConfig.versionNumber,
gitRepoUrl: buildConfig.gitRepoUrl,
}));
for (const build of builds) {
logger.info(` ${build.actorFullName} (${build.versionNumber}) from ${build.gitRepoUrl}`);
}
return { dryRun: true, builds };
}

const buildConfigsByActorFullName = new Map(
buildConfigs.map((buildConfig) => [buildConfig.actorConfig.actorFullName, buildConfig]),
);

return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
const buildConfig = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
// No buildTag: the version keeps the one it has, so its build gets the tag the version already gives it
const actorVersion: ApifyClientTypes.ActorVersion = {
versionNumber: buildConfig.versionNumber,
gitRepoUrl: buildConfig.gitRepoUrl,
sourceType: ActorSourceType.GitRepo,
};
// Always with the Docker cache: CI always asked for it, and a clean build can still be started on the platform
return builder.createVersionAndBuild(buildConfig.versionNumber, actorVersion, true, buildConfig.actorInfo);
});
return executeBuildPlans(plans, { label: 'BUILDS', dryRun });
};

type BuildActorsOptions = Omit<RunBuildsOptions, 'repoUrl' | 'shouldVerifyRepoUrl' | 'branch'> & {
Expand Down
Loading
Loading