diff --git a/ci/Dockerfile b/ci/Dockerfile index d926e93df..861523c50 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -15,16 +15,18 @@ ARG NODE_VERSION=16 ARG FLAKYBOT_VERSION=1.1.0 -FROM node:${NODE_VERSION}-alpine as build +FROM node:${NODE_VERSION}-slim as build -RUN apk add --no-cache curl tar python3 +RUN apt-get update && apt-get install -y curl tar python3 # Install gcloud RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz RUN mkdir -p /usr/local/gcloud \ && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ - && /usr/local/gcloud/google-cloud-sdk/install.sh + && /usr/local/gcloud/google-cloud-sdk/install.sh --quiet --usage-reporting=false + +RUN ln -s /usr/bin/python3 /usr/bin/python # Download flakybot release RUN curl https://github.com/googleapis/repo-automation-bots/releases/download/flakybot-${FLAKYBOT_VERSION}/flakybot \ @@ -34,16 +36,14 @@ RUN curl https://github.com/googleapis/repo-automation-bots/releases/download/fl ENV PNPM_VERSION=7.32.2 RUN curl https://get.pnpm.io/install.sh | ENV="$HOME/.shrc" SHELL="$(which sh)" sh - -FROM node:${NODE_VERSION}-alpine - -# Hack for not found error with flakybot -RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 +FROM node:${NODE_VERSION}-slim COPY --from=build /usr/local/gcloud /usr/local/gcloud COPY --from=build /bin/flakybot /bin/flakybot COPY --from=build /root/.local/share/pnpm /root/.local/share/pnpm -RUN apk add --no-cache git bash python3 +RUN apt-get update && apt-get install -y git bash python3 \ + && ln -s /usr/bin/python3 /usr/bin/python ENV PNPM_HOME=/root/.local/share/pnpm -ENV PATH=$PNPM_HOME:/bin/flakybot:usr/local/gcloud/google-cloud-sdk/bin:$PATH +ENV PATH=$PNPM_HOME:/bin/flakybot:/usr/local/gcloud/google-cloud-sdk/bin:$PATH diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index 7f5131854..450a958ea 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -14,6 +14,9 @@ options: dynamic_substitutions: true + # Increased from default to 8 vCPUs / 8GB RAM to prevent OOM/silent crashes + machineType: 'E2_HIGHCPU_32' + diskSizeGb: '100' substitutions: _BUILD_TYPE: "presubmit" @@ -26,17 +29,6 @@ logsBucket: 'gs://${_LOGS_BUCKET}/logs/google-cloud-node-core/${_BUILD_TYPE}/${C timeout: 32400s steps: - - name: 'gcr.io/kaniko-project/executor:v1.24.0' - args: [ - '--log-format=text', - '--context=dir:///workspace/testing', - '--build-arg=NODE_VERSION=${_NODE_VERSION}', - '--dockerfile=ci/Dockerfile', - '--cache=true', - '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', - '--push-retry=3', - '--image-fs-extract-retry=3' - ] - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' entrypoint: "bash" args: ["ci/deletecloudbuild.sh"] @@ -53,4 +45,5 @@ steps: - 'PROJECT_ID=$PROJECT_ID' - 'REPO_OWNER=${_REPO_OWNER}' - 'REPO_NAME=${_REPO_NAME}' - - 'COMMIT_SHA=$COMMIT_SHA' \ No newline at end of file + - 'COMMIT_SHA=$COMMIT_SHA' + - 'PATH=/usr/local/gcloud/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ No newline at end of file diff --git a/ci/cloudbuild_with_credentials.yaml b/ci/cloudbuild_with_credentials.yaml index fdb4f3400..de8b99350 100644 --- a/ci/cloudbuild_with_credentials.yaml +++ b/ci/cloudbuild_with_credentials.yaml @@ -24,17 +24,6 @@ logsBucket: 'gs://${_LOGS_BUCKET}/logs/google-cloud-node-core/${_BUILD_TYPE}/${C timeout: 7200s steps: - - name: 'gcr.io/kaniko-project/executor:v1.9.1' - args: [ - '--log-format=text', - '--context=dir:///workspace/testing', - '--build-arg=NODE_VERSION=${_NODE_VERSION}', - '--dockerfile=ci/Dockerfile', - '--cache=true', - '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-core-${_NODE_VERSION}', - '--push-retry=3', - '--image-fs-extract-retry=3' - ] - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: [ '-c', "gcloud secrets versions access latest --project=cloud-devrel-kokoro-resources --secret=long-door-651-kokoro-system-test-service-account --format='get(payload.data)' | tr '_-' '/+' | base64 -d > /workspace/google_application_credentials.json" ] @@ -50,4 +39,5 @@ steps: - 'PROJECT_ID=$PROJECT_ID' - 'REPO_OWNER=${_REPO_OWNER}' - 'REPO_NAME=${_REPO_NAME}' - - 'COMMIT_SHA=$COMMIT_SHA' \ No newline at end of file + - 'COMMIT_SHA=$COMMIT_SHA' + - 'PATH=/usr/local/gcloud/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ No newline at end of file diff --git a/ci/cloudbuild_with_gapic_showcase.yaml b/ci/cloudbuild_with_gapic_showcase.yaml index c43b2b31e..ddfddddce 100644 --- a/ci/cloudbuild_with_gapic_showcase.yaml +++ b/ci/cloudbuild_with_gapic_showcase.yaml @@ -31,9 +31,9 @@ steps: args: ['clone', 'https://github.com/googleapis/gapic-showcase.git'] - id: 'Build Showcase' - name: 'golang:1.24' - entrypoint: 'go' - args: ['install', './cmd/gapic-showcase'] + name: 'golang:1.25' + entrypoint: 'sh' + args: ['-c', 'mkdir -p /workspace/bin && CGO_ENABLED=0 go build -v -o /workspace/bin/gapic-showcase ./cmd/gapic-showcase && ls -l /workspace/bin/gapic-showcase'] dir: 'gapic-showcase' env: - 'GOPATH=/workspace' @@ -62,6 +62,8 @@ steps: args: - '-c' - | + echo "Verifying gapic-showcase binary..." + ls -l /workspace/bin/gapic-showcase || echo "BINARY NOT FOUND IN /workspace/bin" echo "Starting gapic-showcase server in background..." /workspace/bin/gapic-showcase run & echo "Waiting for server at localhost:7469..." @@ -75,6 +77,7 @@ steps: env: - 'BUILD_TYPE=${_BUILD_TYPE}' - 'TEST_TYPE=${_TEST_TYPE}' + - 'GAPIC_SHOWCASE_PATH=/workspace/bin/gapic-showcase' - 'BUILD_ID=$BUILD_ID' - 'PROJECT_ID=$PROJECT_ID' - 'REPO_OWNER=${_REPO_OWNER}' diff --git a/packages/gax/test/showcase-server/src/index.ts b/packages/gax/test/showcase-server/src/index.ts index a7f904cc6..0d6c28c73 100644 --- a/packages/gax/test/showcase-server/src/index.ts +++ b/packages/gax/test/showcase-server/src/index.ts @@ -46,8 +46,20 @@ export class ShowcaseServer { process.chdir(testDir); console.log(`Server will be run from ${testDir}.`); - await download(fallbackServerUrl, testDir); - await execa('tar', ['xzf', tarballFilename]); + const prebuiltPath = + process.env['GAPIC_SHOWCASE_PATH'] || '/workspace/bin/gapic-showcase'; + if (fs.existsSync(prebuiltPath)) { + console.log(`Using pre-built gapic-showcase binary from ${prebuiltPath}`); + await fsp.copyFile(prebuiltPath, path.join(testDir, 'gapic-showcase')); + await fsp.chmod(path.join(testDir, 'gapic-showcase'), 0o755); + } else { + console.log( + `Pre-built binary not found at ${prebuiltPath}. Downloading...` + ); + await download(fallbackServerUrl, testDir); + await execa('tar', ['xzf', tarballFilename]); + } + const childProcess = execa(binaryName, ['run'], { cwd: testDir, stdio: 'inherit', diff --git a/packages/gaxios/package.json b/packages/gaxios/package.json index 6af4838ef..1cbe2051d 100644 --- a/packages/gaxios/package.json +++ b/packages/gaxios/package.json @@ -86,7 +86,7 @@ "multiparty": "^4.2.1", "mv": "^2.1.1", "ncp": "^2.0.0", - "nock": "^14.0.5", + "nock": "14.0.5", "null-loader": "^4.0.1", "pack-n-play": "^4.0.0", "puppeteer": "^24.0.0", diff --git a/packages/gcp-metadata/package.json b/packages/gcp-metadata/package.json index 09d23f750..556f71700 100644 --- a/packages/gcp-metadata/package.json +++ b/packages/gcp-metadata/package.json @@ -58,7 +58,6 @@ "chai": "^4.3.10", "cross-env": "^7.0.3", "gcbuild": "^1.3.39", - "gcx": "^2.0.27", "gts": "^6.0.2", "jsdoc": "^4.0.4", "jsdoc-fresh": "^5.0.0", diff --git a/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore b/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore index 27c5fe2d9..6b5cddb8f 100644 --- a/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore +++ b/packages/gcp-metadata/system-test/fixtures/hook/.gcloudignore @@ -1,19 +1,4 @@ -# This file specifies files that are *not* uploaded to Google Cloud Platform -# using gcloud. It follows the same syntax as .gitignore, with the addition of -# "#!include" directives (which insert the entries of the given .gitignore-style -# file at that point). -# -# For more information, run: -# $ gcloud topic gcloudignore -# -.gcloudignore -# If you would like to upload your .git directory, .gitignore file or files -# from your .gitignore file, remove the corresponding line -# below: +node_modules .git .gitignore - -node_modules -#!include:.gitignore - test/ diff --git a/packages/gcp-metadata/system-test/fixtures/hook/index.js b/packages/gcp-metadata/system-test/fixtures/hook/index.js index 684add2db..f4d634dd8 100644 --- a/packages/gcp-metadata/system-test/fixtures/hook/index.js +++ b/packages/gcp-metadata/system-test/fixtures/hook/index.js @@ -16,6 +16,16 @@ const gcpMetadata = require('gcp-metadata'); +// Log availability on startup to allow verification via logs +(async () => { + try { + const isAvailable = await gcpMetadata.isAvailable(); + console.log(`GCF_METADATA_CHECK: isAvailable=${isAvailable}`); + } catch (e) { + console.error(`GCF_METADATA_CHECK: error=${e.message}`); + } +})(); + exports.getMetadata = async (req, res) => { const isAvailable = await gcpMetadata.isAvailable(); const instance = await gcpMetadata.instance(); diff --git a/packages/gcp-metadata/system-test/system.ts b/packages/gcp-metadata/system-test/system.ts index 8ad5ef77c..3a219e59d 100644 --- a/packages/gcp-metadata/system-test/system.ts +++ b/packages/gcp-metadata/system-test/system.ts @@ -18,18 +18,17 @@ import assert from 'assert'; import {before, after, describe, it} from 'mocha'; import fs from 'fs'; import * as gcbuild from 'gcbuild'; -import {CloudFunctionsServiceClient} from '@google-cloud/functions'; +import {v2, CloudFunctionsServiceClient} from '@google-cloud/functions'; import * as path from 'path'; import {promisify} from 'util'; import {execSync} from 'child_process'; import {request} from 'gaxios'; -const loadGcx = () => import('gcx'); - const copy = promisify(fs.copyFile); const pkg = require('../../package.json'); // eslint-disable-line let gcf: CloudFunctionsServiceClient; +let gcfV2: v2.FunctionServiceClient; let projectId: string; const shortPrefix = 'gcloud-tests'; const randomUUID = () => @@ -41,7 +40,10 @@ describe('gcp metadata', () => { // pack up the gcp-metadata module and copy to the target dir await packModule(); gcf = new CloudFunctionsServiceClient(); + gcfV2 = new v2.FunctionServiceClient(); projectId = await gcf.auth.getProjectId(); + console.log(`Using Project ID: ${projectId}`); + console.log(`Function Name: ${fullPrefix}`); }); describe('cloud functions', () => { @@ -51,23 +53,60 @@ describe('gcp metadata', () => { // deploy the function to GCF await deployApp(); - // cloud functions now require authentication by default, see: - // https://cloud.google.com/functions/docs/release-notes - await gcf.setIamPolicy({ - resource: `projects/${projectId}/locations/us-central1/functions/${fullPrefix}`, - policy: { - bindings: [ - {members: ['allUsers'], role: 'roles/cloudfunctions.invoker'}, - ], - }, - }); }); it('should access the metadata service on GCF', async () => { - const url = `https://us-central1-${projectId}.cloudfunctions.net/${fullPrefix}`; - const res = await request<{isAvailable: boolean}>({url}); - console.dir(res.data); - assert.strictEqual(res.data.isAvailable, true); + // Fetch the function metadata + const name = `projects/${projectId}/locations/us-central1/functions/${fullPrefix}`; + const [func] = await gcfV2.getFunction({name}); + + // 2nd Gen URLs are stored in serviceConfig.uri + const url = func.serviceConfig?.uri; + + if (!url) { + throw new Error( + `Could not find URI for function: ${fullPrefix}. Is it a Gen 2 function?`, + ); + } + + console.log(`Verifying Gen 2 function via logs: ${fullPrefix}`); + + // Poll for the log entry + let found = false; + const maxRetries = 20; + const filter = `resource.type="cloud_run_revision" AND resource.labels.service_name="${fullPrefix}" AND textPayload:"GCF_METADATA_CHECK"`; + const cmd = `gcloud logging read '${filter}' --project=${projectId} --format="json" --limit=5`; + + console.log(`Polling for logs with command: ${cmd}`); + + for (let i = 0; i < maxRetries; i++) { + process.stdout.write('.'); + try { + const output = execSync(cmd).toString(); + const logs = JSON.parse(output); + if (logs && logs.length > 0) { + console.log('\nFound log entries:'); + console.dir(logs, {depth: null}); + const latestLog = logs[0].textPayload; + assert.ok( + latestLog.includes('isAvailable=true'), + `Metadata check failed: ${latestLog}`, + ); + found = true; + break; + } + } catch (e) { + console.error(`\nError reading logs: ${(e as any).message}`); + } + await new Promise(resolve => setTimeout(resolve, 5000)); + } + + if (!found) { + throw new Error( + `Could not find GCF_METADATA_CHECK log entry for ${fullPrefix} after ${maxRetries} retries.`, + ); + } + console.log('\nSuccessfully verified metadata access via logs.'); }); after(() => pruneFunctions(true)); @@ -98,12 +137,12 @@ describe('gcp metadata', () => { */ async function pruneFunctions(sessionOnly: boolean) { console.log('Pruning leaked functions...'); - const [fns] = await gcf.listFunctions({ + const [fns] = await gcfV2.listFunctions({ parent: `projects/${projectId}/locations/-`, }); await Promise.all( fns - .filter(fn => { + .filter((fn: any) => { if (sessionOnly) { return fn.name!.includes(fullPrefix); } @@ -112,8 +151,8 @@ async function pruneFunctions(sessionOnly: boolean) { const minutesSinceUpdate = (currentDate - updateDate) / 1000 / 60; return minutesSinceUpdate > 60 && fn.name!.includes(shortPrefix); }) - .map(async fn => { - await gcf.deleteFunction({name: fn.name}).catch(e => { + .map(async (fn: any) => { + await gcfV2.deleteFunction({name: fn.name}).catch((e: any) => { console.error(`There was a problem deleting function ${fn.name}.`); console.error(e); }); @@ -126,15 +165,40 @@ async function pruneFunctions(sessionOnly: boolean) { */ async function deployApp() { const targetDir = path.join(__dirname, '../../system-test/fixtures/hook'); - const gcx = await loadGcx(); - await gcx.deploy({ - name: fullPrefix, - entryPoint: 'getMetadata', - triggerHTTP: true, - runtime: 'nodejs20', - region: 'us-central1', - targetDir, - }); + const files = fs.readdirSync(targetDir); + console.log(`Files to package: ${files.join(', ')}`); + + console.log(`PATH: ${process.env.PATH}`); + try { + const whichGcloud = execSync('which gcloud').toString().trim(); + console.log(`Using gcloud at: ${whichGcloud}`); + } catch (e) { + console.error('gcloud CLI not found in PATH'); + } + + console.log( + `Deploying function ${fullPrefix} from ${targetDir} using gcloud...`, + ); + const cmd = + `gcloud functions deploy ${fullPrefix} ` + + '--gen2 ' + + '--region=us-central1 ' + + '--runtime=nodejs20 ' + + `--source=${targetDir} ` + + '--entry-point=getMetadata ' + + '--ingress-settings=internal-only ' + + '--allow-unauthenticated ' + + '--trigger-http ' + + `--project=${projectId} ` + + '--quiet'; + + try { + execSync(cmd, {stdio: 'inherit'}); + console.log(`Successfully deployed ${fullPrefix}`); + } catch (error) { + console.error(`Deployment failed: ${(error as any).message}`); + throw error; + } } /** diff --git a/packages/nodejs-proto-files/tools/prepublish.ts b/packages/nodejs-proto-files/tools/prepublish.ts index dd051fde8..52821d612 100644 --- a/packages/nodejs-proto-files/tools/prepublish.ts +++ b/packages/nodejs-proto-files/tools/prepublish.ts @@ -21,38 +21,60 @@ const protoFolders = ['google', 'grafeas', 'gapic']; // eslint-disable-next-line @typescript-eslint/no-var-requires const DecompressZip = require('decompress-zip'); -const extract = (input, opts, callback) => { - const output = Math.random() + '.zip'; - - (got as unknown as got.Got) - .stream(input) - .on('error', callback) - .pipe(fs.createWriteStream(output)) - .on('error', callback) - .on('finish', () => { - const unzipper = new DecompressZip(output); - - unzipper - .on('error', callback) - .extract({ - strip: opts.strip, - filter: file => { - if (opts.filter && !opts.filter(file)) return; - return path.extname(file.filename) === '.proto'; - }, - }) - .on('extract', () => { - fs.unlink(output, callback); - }); - }); +const extract = (input: string, opts: {strip?: number; filter?: (file: any) => boolean}, callback: (err?: Error | null) => void) => { + const output = Math.floor(Math.random() * 1000000) + '.zip'; + console.log(`Downloading ${input} to ${output}...`); + + const writeStream = fs.createWriteStream(output); + const stream = (got as unknown as got.Got).stream(input); + + stream.on('error', err => { + console.error(`Download error for ${input}: ${err.message}`); + writeStream.end(); + fs.unlink(output, () => {}); + callback(err); + }); + + stream.pipe(writeStream); + + writeStream.on('error', err => { + console.error(`Write error for ${output}: ${err.message}`); + fs.unlink(output, () => {}); + callback(err); + }); + + writeStream.on('finish', () => { + console.log(`Extracting ${output}...`); + const unzipper = new DecompressZip(output); + + unzipper + .on('error', err => { + console.error(`Extraction error for ${output}: ${err.message}`); + fs.unlink(output, () => {}); + callback(err); + }) + .extract({ + strip: opts.strip, + filter: file => { + if (opts.filter && !opts.filter(file)) return; + return path.extname(file.filename) === '.proto'; + }, + }) + .on('extract', () => { + console.log(`Finished extracting ${output}`); + fs.unlink(output, callback); + }); + }); }; const extractAsync = promisify(extract); const execAsync = promisify(require('child_process').exec); async function main() { + console.log(`Cleaning up old proto folders: ${protoFolders.join(', ')}`); await execAsync(`rm -rf ${protoFolders.join(' ')}`); + console.log('Fetching googleapis protos...'); await extractAsync( 'https://github.com/googleapis/googleapis/archive/master.zip', { @@ -60,13 +82,15 @@ async function main() { }, ); + console.log('Fetching protobuf protos...'); await extractAsync('https://github.com/google/protobuf/archive/main.zip', { strip: 2, filter: file => { + const parent = file.parent || path.dirname(file.filename); return ( - file.parent.indexOf('protobuf-main') === 0 && - file.parent.indexOf('protobuf-main/src/') === 0 && - file.parent.indexOf('/internal') === -1 && + parent.indexOf('protobuf-main') === 0 && + parent.indexOf('protobuf-main/src/') === 0 && + parent.indexOf('/internal') === -1 && file.filename.indexOf('unittest') === -1 && file.filename.indexOf('test') === -1 ); @@ -76,6 +100,17 @@ async function main() { await execAsync( '[ -d "overrides" ] && cp -R overrides/* google || echo "no overrides"', ); + + // Validation + for (const folder of protoFolders) { + if (!fs.existsSync(folder) || fs.readdirSync(folder).length === 0) { + throw new Error(`Failed to create or populate folder: ${folder}`); + } + } + console.log('Successfully prepared all proto files.'); } -main().catch(console.error); +main().catch(err => { + console.error('PREPUBLISH FAILED:', err); + process.exit(1); +});