robojerk/codeql-action

Remove fallback to gzip

Browse source 
Data shows it is unneeded
This commit is contained in:
Henry Mercer 2024-11-01 15:27:58 +00:00
parent 33f2dc57a4
commit 26077f3119
5 changed files with 6 additions and 294 deletions
Download patch file Download diff file Expand all files Collapse all files

123
.github/workflows/__zstd-bundle-fallback.yml generated vendored
View file

@ -1,123 +0,0 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# (cd pr-checks; pip install ruamel.yaml@0.17.31 && python3 sync.py)
# to regenerate this file.
name: PR Check - Zstandard bundle fallback
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- releases/v*
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
schedule:
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
zstd-bundle-fallback:
strategy:
fail-fast: false
matrix:
include:
- os: macos-latest
version: linked
- os: ubuntu-latest
version: linked
name: Zstandard bundle fallback
permissions:
contents: read
security-events: write
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Setup Python on MacOS
uses: actions/setup-python@v5
if: runner.os == 'macOS' && matrix.version == 'stable-v2.14.6'
with:
python-version: '3.11'
- name: Check out repository
uses: actions/checkout@v4
- name: Prepare test
id: prepare-test
uses: ./.github/actions/prepare-test
with:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check expected diagnostics
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.gz')) {
core.setFailed(
`Expected the tools URL to be a .tar.gz file, but found '${toolsUrl}'.`
);
}
const zstdFailureReason = downloadTelemetryNotifications[0].properties.attributes.zstdFailureReason;
console.log(`Found zstd failure reason: ${zstdFailureReason}`);
const expectedZstdFailureReason = 'Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.';
if (zstdFailureReason !== expectedZstdFailureReason) {
core.setFailed(
`Expected the zstd failure reason to be '${expectedZstdFailureReason}', but found '${zstdFailureReason}'.`
);
}
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_FORCE_ZSTD_FAILURE: true
CODEQL_ACTION_TEST_MODE: true

29
lib/setup-codeql.js generated
View file

@ -464,34 +464,7 @@ function getCanonicalToolcacheVersion(cliVersion, bundleVersion, logger) {
*/
async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger) {
const zstdAvailability = await tar.isZstdAvailable(logger);
let zstdFailureReason;
// If we think the installed version of tar supports zstd, try to use zstd,
// but be prepared to fall back to gzip in case we were wrong.
if (zstdAvailability.available) {
try {
// To facilitate testing the fallback, fail here if a testing environment variable is set.
if (process.env.CODEQL_ACTION_FORCE_ZSTD_FAILURE === "true") {
throw new Error("Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.");
}
return await setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger, zstdAvailability, true);
}
catch (e) {
zstdFailureReason = util.getErrorMessage(e) || "unknown error";
if (e instanceof actions_util_1.CommandInvocationError) {
zstdFailureReason += ` Full error: ${e.stderr}`;
logger.debug(`Invocation output the following to stderr: ${e.stderr}`);
}
logger.warning(`Failed to set up CodeQL tools with zstd. Falling back to gzipped version. Error: ${util.getErrorMessage(e)}`);
}
}
const result = await setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger, zstdAvailability, false);
if (result.toolsDownloadStatusReport && zstdFailureReason) {
result.toolsDownloadStatusReport.zstdFailureReason = zstdFailureReason;
}
return result;
}
async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger, zstdAvailability, useTarIfAvailable) {
const source = await getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, variant, useTarIfAvailable, logger);
const source = await getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, variant, zstdAvailability.available, logger);
let codeqlFolder;
let toolsVersion = source.toolsVersion;
let toolsDownloadStatusReport;

2
lib/setup-codeql.js.map
View file

File diff suppressed because one or more lines are too long

76
pr-checks/checks/zstd-bundle-fallback.yml
View file

@ -1,76 +0,0 @@
name: "Zstandard bundle fallback"
description: "Tests the fallback when downloading a Zstandard-compressed CodeQL Bundle fails"
versions:
- linked
operatingSystems:
- macos
- ubuntu
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_FORCE_ZSTD_FAILURE: true
steps:
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check expected diagnostics
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.gz')) {
core.setFailed(
`Expected the tools URL to be a .tar.gz file, but found '${toolsUrl}'.`
);
}
const zstdFailureReason = downloadTelemetryNotifications[0].properties.attributes.zstdFailureReason;
console.log(`Found zstd failure reason: ${zstdFailureReason}`);
const expectedZstdFailureReason = 'Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.';
if (zstdFailureReason !== expectedZstdFailureReason) {
core.setFailed(
`Expected the zstd failure reason to be '${expectedZstdFailureReason}', but found '${zstdFailureReason}'.`
);
}

70
src/setup-codeql.ts
View file

@ -7,7 +7,7 @@ import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as semver from "semver";
import { CommandInvocationError, isRunningLocalAction } from "./actions-util";
import { isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
import * as defaults from "./defaults.json";
import {
@ -634,7 +634,6 @@ export interface SetupCodeQLResult {
toolsSource: ToolsSource;
toolsVersion: string;
zstdAvailability: tar.ZstdAvailability;
zstdFailureReason?: string;
}
/**
@ -649,76 +648,15 @@ export async function setupCodeQLBundle(
variant: util.GitHubVariant,
defaultCliVersion: CodeQLDefaultVersionInfo,
logger: Logger,
): Promise<SetupCodeQLResult> {
const zstdAvailability = await tar.isZstdAvailable(logger);
let zstdFailureReason: string | undefined;
// If we think the installed version of tar supports zstd, try to use zstd,
// but be prepared to fall back to gzip in case we were wrong.
if (zstdAvailability.available) {
try {
// To facilitate testing the fallback, fail here if a testing environment variable is set.
if (process.env.CODEQL_ACTION_FORCE_ZSTD_FAILURE === "true") {
throw new Error(
"Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.",
);
}
return await setupCodeQLBundleWithCompressionMethod(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
logger,
zstdAvailability,
true,
);
} catch (e) {
zstdFailureReason = util.getErrorMessage(e) || "unknown error";
if (e instanceof CommandInvocationError) {
zstdFailureReason += ` Full error: ${e.stderr}`;
logger.debug(`Invocation output the following to stderr: ${e.stderr}`);
}
logger.warning(
`Failed to set up CodeQL tools with zstd. Falling back to gzipped version. Error: ${util.getErrorMessage(
e,
)}`,
);
}
}
const result = await setupCodeQLBundleWithCompressionMethod(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
logger,
zstdAvailability,
false,
);
if (result.toolsDownloadStatusReport && zstdFailureReason) {
result.toolsDownloadStatusReport.zstdFailureReason = zstdFailureReason;
}
return result;
}
async function setupCodeQLBundleWithCompressionMethod(
toolsInput: string | undefined,
apiDetails: api.GitHubApiDetails,
tempDir: string,
variant: util.GitHubVariant,
defaultCliVersion: CodeQLDefaultVersionInfo,
logger: Logger,
zstdAvailability: tar.ZstdAvailability,
useTarIfAvailable: boolean,
) {
const zstdAvailability = await tar.isZstdAvailable(logger);
const source = await getCodeQLSource(
toolsInput,
defaultCliVersion,
apiDetails,
variant,
useTarIfAvailable,
zstdAvailability.available,
logger,
);