diff --git a/ci/Dockerfile b/ci/Dockerfile index d926e93df..861523c50 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -15,16 +15,18 @@ ARG NODE_VERSION=16 ARG FLAKYBOT_VERSION=1.1.0 -FROM node:${NODE_VERSION}-alpine as build +FROM node:${NODE_VERSION}-slim as build -RUN apk add --no-cache curl tar python3 +RUN apt-get update && apt-get install -y curl tar python3 # Install gcloud RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz RUN mkdir -p /usr/local/gcloud \ && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ - && /usr/local/gcloud/google-cloud-sdk/install.sh + && /usr/local/gcloud/google-cloud-sdk/install.sh --quiet --usage-reporting=false + +RUN ln -s /usr/bin/python3 /usr/bin/python # Download flakybot release RUN curl https://github.com/googleapis/repo-automation-bots/releases/download/flakybot-${FLAKYBOT_VERSION}/flakybot \ @@ -34,16 +36,14 @@ RUN curl https://github.com/googleapis/repo-automation-bots/releases/download/fl ENV PNPM_VERSION=7.32.2 RUN curl https://get.pnpm.io/install.sh | ENV="$HOME/.shrc" SHELL="$(which sh)" sh - -FROM node:${NODE_VERSION}-alpine - -# Hack for not found error with flakybot -RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 +FROM node:${NODE_VERSION}-slim COPY --from=build /usr/local/gcloud /usr/local/gcloud COPY --from=build /bin/flakybot /bin/flakybot COPY --from=build /root/.local/share/pnpm /root/.local/share/pnpm -RUN apk add --no-cache git bash python3 +RUN apt-get update && apt-get install -y git bash python3 \ + && ln -s /usr/bin/python3 /usr/bin/python ENV PNPM_HOME=/root/.local/share/pnpm -ENV PATH=$PNPM_HOME:/bin/flakybot:usr/local/gcloud/google-cloud-sdk/bin:$PATH +ENV PATH=$PNPM_HOME:/bin/flakybot:/usr/local/gcloud/google-cloud-sdk/bin:$PATH diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index 7f5131854..ee67e1d83 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -14,6 +14,9 @@ options: dynamic_substitutions: true + # Increased from default to 8 vCPUs / 8GB RAM to prevent OOM/silent crashes + machineType: 'E2_HIGHCPU_32' + diskSizeGb: '100' substitutions: _BUILD_TYPE: "presubmit" @@ -26,22 +29,12 @@ logsBucket: 'gs://${_LOGS_BUCKET}/logs/google-cloud-node-core/${_BUILD_TYPE}/${C timeout: 32400s steps: - - name: 'gcr.io/kaniko-project/executor:v1.24.0' - args: [ - '--log-format=text', - '--context=dir:///workspace/testing', - '--build-arg=NODE_VERSION=${_NODE_VERSION}', - '--dockerfile=ci/Dockerfile', - '--cache=true', - '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', - '--push-retry=3', - '--image-fs-extract-retry=3' - ] - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' entrypoint: "bash" args: ["ci/deletecloudbuild.sh"] env: - 'REF_NAME=${REF_NAME}' + - name: gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION} id: "run-tests" timeout: 32400s @@ -53,4 +46,5 @@ steps: - 'PROJECT_ID=$PROJECT_ID' - 'REPO_OWNER=${_REPO_OWNER}' - 'REPO_NAME=${_REPO_NAME}' - - 'COMMIT_SHA=$COMMIT_SHA' \ No newline at end of file + - 'COMMIT_SHA=$COMMIT_SHA' + - 'PATH=/usr/local/gcloud/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' diff --git a/ci/cloudbuild_with_credentials.yaml b/ci/cloudbuild_with_credentials.yaml index fdb4f3400..de8b99350 100644 --- a/ci/cloudbuild_with_credentials.yaml +++ b/ci/cloudbuild_with_credentials.yaml @@ -24,17 +24,6 @@ logsBucket: 'gs://${_LOGS_BUCKET}/logs/google-cloud-node-core/${_BUILD_TYPE}/${C timeout: 7200s steps: - - name: 'gcr.io/kaniko-project/executor:v1.9.1' - args: [ - '--log-format=text', - '--context=dir:///workspace/testing', - '--build-arg=NODE_VERSION=${_NODE_VERSION}', - '--dockerfile=ci/Dockerfile', - '--cache=true', - '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', - '--push-retry=3', - '--image-fs-extract-retry=3' - ] - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: [ '-c', "gcloud secrets versions access latest --project=cloud-devrel-kokoro-resources --secret=long-door-651-kokoro-system-test-service-account --format='get(payload.data)' | tr '_-' '/+' | base64 -d > /workspace/google_application_credentials.json" ] @@ -50,4 +39,5 @@ steps: - 'PROJECT_ID=$PROJECT_ID' - 'REPO_OWNER=${_REPO_OWNER}' - 'REPO_NAME=${_REPO_NAME}' - - 'COMMIT_SHA=$COMMIT_SHA' \ No newline at end of file + - 'COMMIT_SHA=$COMMIT_SHA' + - 'PATH=/usr/local/gcloud/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ No newline at end of file diff --git a/ci/cloudbuild_with_gapic_showcase.yaml b/ci/cloudbuild_with_gapic_showcase.yaml index c43b2b31e..3413fcb7b 100644 --- a/ci/cloudbuild_with_gapic_showcase.yaml +++ b/ci/cloudbuild_with_gapic_showcase.yaml @@ -14,6 +14,8 @@ options: dynamic_substitutions: true + machineType: 'E2_HIGHCPU_32' + diskSizeGb: '100' substitutions: _BUILD_TYPE: "presubmit" @@ -31,7 +33,7 @@ steps: args: ['clone', 'https://github.com/googleapis/gapic-showcase.git'] - id: 'Build Showcase' - name: 'golang:1.24' + name: 'golang:1.25' entrypoint: 'go' args: ['install', './cmd/gapic-showcase'] dir: 'gapic-showcase' @@ -39,22 +41,36 @@ steps: - 'GOPATH=/workspace' - 'GOCACHE=/workspace/.cache/go-build' waitFor: ['Clone Showcase'] - - name: 'gcr.io/kaniko-project/executor:v1.24.0' + - name: 'gcr.io/cloud-builders/docker' + id: 'Pull Image for Cache' + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION} || exit 0'] + waitFor: ['Build Showcase'] + + - name: 'gcr.io/cloud-builders/docker' + id: 'Build Test Image' args: [ - '--log-format=text', - '--context=dir:///workspace/testing', - '--build-arg=NODE_VERSION=${_NODE_VERSION}', - '--dockerfile=ci/Dockerfile', - '--cache=true', - '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', - '--push-retry=3', - '--image-fs-extract-retry=3' + 'build', + '-t', 'gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', + '--cache-from', 'gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', + '-f', 'ci/Dockerfile', + '--build-arg', 'NODE_VERSION=${_NODE_VERSION}', + '.' ] + waitFor: ['Pull Image for Cache'] + + - name: 'gcr.io/cloud-builders/docker' + id: 'Push Test Image' + args: ['push', 'gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}'] + waitFor: ['Build Test Image'] + - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' entrypoint: "bash" args: ["ci/deletecloudbuild.sh"] env: - 'REF_NAME=${REF_NAME}' + waitFor: ['Push Test Image'] + - name: gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION} id: "run-tests" timeout: 32400s @@ -80,4 +96,7 @@ steps: - 'REPO_OWNER=${_REPO_OWNER}' - 'REPO_NAME=${_REPO_NAME}' - 'COMMIT_SHA=$COMMIT_SHA' - waitFor: ['Build Showcase'] + waitFor: ['Push Test Image'] + +images: + - 'gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}' diff --git a/ci/export/gcnc-system-continuous-node.yaml b/ci/export/gcnc-system-continuous-node.yaml index d1a803830..ae639079b 100644 --- a/ci/export/gcnc-system-continuous-node.yaml +++ b/ci/export/gcnc-system-continuous-node.yaml @@ -1,6 +1,6 @@ createTime: '2025-09-18T18:59:44.053456230Z' description: Continuous build with node 18 -filename: ci/cloudbuild.yaml +filename: ci/cloudbuild_with_gapic_showcase.yaml github: name: google-cloud-node-core owner: googleapis diff --git a/ci/export/gcnc-system-nightly-node.yaml b/ci/export/gcnc-system-nightly-node.yaml index 78a9cdc18..c14f79875 100644 --- a/ci/export/gcnc-system-nightly-node.yaml +++ b/ci/export/gcnc-system-nightly-node.yaml @@ -1,7 +1,7 @@ createTime: '2025-09-18T18:59:45.514768458Z' description: Nightly build with node 18 gitFileSource: - path: ci/cloudbuild.yaml + path: ci/cloudbuild_with_gapic_showcase.yaml repoType: GITHUB revision: refs/heads/main uri: https://github.com/googleapis/google-cloud-node-core diff --git a/ci/export/gcnc-system-presubmit-node.yaml b/ci/export/gcnc-system-presubmit-node.yaml index fe945affa..0f5c1da35 100644 --- a/ci/export/gcnc-system-presubmit-node.yaml +++ b/ci/export/gcnc-system-presubmit-node.yaml @@ -1,6 +1,6 @@ createTime: '2025-09-18T18:59:42.228485468Z' description: Presubmit build with node 18 -filename: ci/cloudbuild.yaml +filename: ci/cloudbuild_with_gapic_showcase.yaml github: name: google-cloud-node-core owner: googleapis diff --git a/packages/gaxios/package.json b/packages/gaxios/package.json index 6af4838ef..1cbe2051d 100644 --- a/packages/gaxios/package.json +++ b/packages/gaxios/package.json @@ -86,7 +86,7 @@ "multiparty": "^4.2.1", "mv": "^2.1.1", "ncp": "^2.0.0", - "nock": "^14.0.5", + "nock": "14.0.5", "null-loader": "^4.0.1", "pack-n-play": "^4.0.0", "puppeteer": "^24.0.0", diff --git a/packages/gcp-metadata/package.json b/packages/gcp-metadata/package.json index 09d23f750..556f71700 100644 --- a/packages/gcp-metadata/package.json +++ b/packages/gcp-metadata/package.json @@ -58,7 +58,6 @@ "chai": "^4.3.10", "cross-env": "^7.0.3", "gcbuild": "^1.3.39", - "gcx": "^2.0.27", "gts": "^6.0.2", "jsdoc": "^4.0.4", "jsdoc-fresh": "^5.0.0", diff --git a/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore b/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore index 27c5fe2d9..6b5cddb8f 100644 --- a/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore +++ b/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore @@ -1,19 +1,4 @@ -# This file specifies files that are *not* uploaded to Google Cloud Platform -# using gcloud. It follows the same syntax as .gitignore, with the addition of -# "#!include" directives (which insert the entries of the given .gitignore-style -# file at that point). -# -# For more information, run: -# $ gcloud topic gcloudignore -# -.gcloudignore -# If you would like to upload your .git directory, .gitignore file or files -# from your .gitignore file, remove the corresponding line -# below: +node_modules .git .gitignore - -node_modules -#!include:.gitignore - test/ diff --git a/packages/gcp-metadata/system-test/fixtures/hook/index.js b/packages/gcp-metadata/system-test/fixtures/hook/index.js index 684add2db..3ff969093 100644 --- a/packages/gcp-metadata/system-test/fixtures/hook/index.js +++ b/packages/gcp-metadata/system-test/fixtures/hook/index.js @@ -16,7 +16,18 @@ const gcpMetadata = require('gcp-metadata'); +// Log availability on startup to allow verification via logs +const init = (async () => { + try { + const isAvailable = await gcpMetadata.isAvailable(); + console.log(`GCF_METADATA_CHECK: isAvailable=${isAvailable}`); + } catch (e) { + console.error(`GCF_METADATA_CHECK: error=${e.message}`); + } +})(); + exports.getMetadata = async (req, res) => { + await init; const isAvailable = await gcpMetadata.isAvailable(); const instance = await gcpMetadata.instance(); const svc = await gcpMetadata.instance({ diff --git a/packages/gcp-metadata/system-test/system.ts b/packages/gcp-metadata/system-test/system.ts index 8ad5ef77c..3a219e59d 100644 --- a/packages/gcp-metadata/system-test/system.ts +++ b/packages/gcp-metadata/system-test/system.ts @@ -18,18 +18,17 @@ import assert from 'assert'; import {before, after, describe, it} from 'mocha'; import fs from 'fs'; import * as gcbuild from 'gcbuild'; -import {CloudFunctionsServiceClient} from '@google-cloud/functions'; +import {v2, CloudFunctionsServiceClient} from '@google-cloud/functions'; import * as path from 'path'; import {promisify} from 'util'; import {execSync} from 'child_process'; import {request} from 'gaxios'; -const loadGcx = () => import('gcx'); - const copy = promisify(fs.copyFile); const pkg = require('../../package.json'); // eslint-disable-line let gcf: CloudFunctionsServiceClient; +let gcfV2: v2.FunctionServiceClient; let projectId: string; const shortPrefix = 'gcloud-tests'; const randomUUID = () => @@ -41,7 +40,10 @@ describe('gcp metadata', () => { // pack up the gcp-metadata module and copy to the target dir await packModule(); gcf = new CloudFunctionsServiceClient(); + gcfV2 = new v2.FunctionServiceClient(); projectId = await gcf.auth.getProjectId(); + console.log(`Using Project ID: ${projectId}`); + console.log(`Function Name: ${fullPrefix}`); }); describe('cloud functions', () => { @@ -51,23 +53,60 @@ describe('gcp metadata', () => { // deploy the function to GCF await deployApp(); - // cloud functions now require authentication by default, see: - // https://cloud.google.com/functions/docs/release-notes - await gcf.setIamPolicy({ - resource: `projects/${projectId}/locations/us-central1/functions/${fullPrefix}`, - policy: { - bindings: [ - {members: ['allUsers'], role: 'roles/cloudfunctions.invoker'}, - ], - }, - }); }); it('should access the metadata service on GCF', async () => { - const url = `https://us-central1-${projectId}.cloudfunctions.net/${fullPrefix}`; - const res = await request<{isAvailable: boolean}>({url}); - console.dir(res.data); - assert.strictEqual(res.data.isAvailable, true); + // Fetch the function metadata + const name = `projects/${projectId}/locations/us-central1/functions/${fullPrefix}`; + const [func] = await gcfV2.getFunction({name}); + + // 2nd Gen URLs are stored in serviceConfig.uri + const url = func.serviceConfig?.uri; + + if (!url) { + throw new Error( + `Could not find URI for function: ${fullPrefix}. Is it a Gen 2 function?`, + ); + } + + console.log(`Verifying Gen 2 function via logs: ${fullPrefix}`); + + // Poll for the log entry + let found = false; + const maxRetries = 20; + const filter = `resource.type="cloud_run_revision" AND resource.labels.service_name="${fullPrefix}" AND textPayload:"GCF_METADATA_CHECK"`; + const cmd = `gcloud logging read '${filter}' --project=${projectId} --format="json" --limit=5`; + + console.log(`Polling for logs with command: ${cmd}`); + + for (let i = 0; i < maxRetries; i++) { + process.stdout.write('.'); + try { + const output = execSync(cmd).toString(); + const logs = JSON.parse(output); + if (logs && logs.length > 0) { + console.log('\nFound log entries:'); + console.dir(logs, {depth: null}); + const latestLog = logs[0].textPayload; + assert.ok( + latestLog.includes('isAvailable=true'), + `Metadata check failed: ${latestLog}`, + ); + found = true; + break; + } + } catch (e) { + console.error(`\nError reading logs: ${(e as any).message}`); + } + await new Promise(resolve => setTimeout(resolve, 5000)); + } + + if (!found) { + throw new Error( + `Could not find GCF_METADATA_CHECK log entry for ${fullPrefix} after ${maxRetries} retries.`, + ); + } + console.log('\nSuccessfully verified metadata access via logs.'); }); after(() => pruneFunctions(true)); @@ -98,12 +137,12 @@ describe('gcp metadata', () => { */ async function pruneFunctions(sessionOnly: boolean) { console.log('Pruning leaked functions...'); - const [fns] = await gcf.listFunctions({ + const [fns] = await gcfV2.listFunctions({ parent: `projects/${projectId}/locations/-`, }); await Promise.all( fns - .filter(fn => { + .filter((fn: any) => { if (sessionOnly) { return fn.name!.includes(fullPrefix); } @@ -112,8 +151,8 @@ async function pruneFunctions(sessionOnly: boolean) { const minutesSinceUpdate = (currentDate - updateDate) / 1000 / 60; return minutesSinceUpdate > 60 && fn.name!.includes(shortPrefix); }) - .map(async fn => { - await gcf.deleteFunction({name: fn.name}).catch(e => { + .map(async (fn: any) => { + await gcfV2.deleteFunction({name: fn.name}).catch((e: any) => { console.error(`There was a problem deleting function ${fn.name}.`); console.error(e); }); @@ -126,15 +165,40 @@ async function pruneFunctions(sessionOnly: boolean) { */ async function deployApp() { const targetDir = path.join(__dirname, '../../system-test/fixtures/hook'); - const gcx = await loadGcx(); - await gcx.deploy({ - name: fullPrefix, - entryPoint: 'getMetadata', - triggerHTTP: true, - runtime: 'nodejs20', - region: 'us-central1', - targetDir, - }); + const files = fs.readdirSync(targetDir); + console.log(`Files to package: ${files.join(', ')}`); + + console.log(`PATH: ${process.env.PATH}`); + try { + const whichGcloud = execSync('which gcloud').toString().trim(); + console.log(`Using gcloud at: ${whichGcloud}`); + } catch (e) { + console.error('gcloud CLI not found in PATH'); + } + + console.log( + `Deploying function ${fullPrefix} from ${targetDir} using gcloud...`, + ); + const cmd = + `gcloud functions deploy ${fullPrefix} ` + + '--gen2 ' + + '--region=us-central1 ' + + '--runtime=nodejs20 ' + + `--source=${targetDir} ` + + '--entry-point=getMetadata ' + + '--ingress-settings=internal-only ' + + '--allow-unauthenticated ' + + '--trigger-http ' + + `--project=${projectId} ` + + '--quiet'; + + try { + execSync(cmd, {stdio: 'inherit'}); + console.log(`Successfully deployed ${fullPrefix}`); + } catch (error) { + console.error(`Deployment failed: ${(error as any).message}`); + throw error; + } } /**