robojerk/codeql-action

Merge pull request #2502 from github/henrymercer/zstd-experiment

Browse source 
Add a feature flag to use a bundle compressed using Zstandard when setting up the default tools
This commit is contained in:
Henry Mercer 2024-09-27 14:48:49 +01:00 • committed by GitHub
commit 2617ff2d3f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 713 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

130
.github/workflows/__zstd-bundle-fallback.yml generated vendored Normal file
View file

@ -0,0 +1,130 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# (cd pr-checks; pip install ruamel.yaml@0.17.31 && python3 sync.py)
# to regenerate this file.
name: PR Check - Zstandard bundle fallback
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- releases/v*
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
schedule:
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
zstd-bundle-fallback:
strategy:
fail-fast: false
matrix:
include:
- os: macos-latest
version: linked
- os: windows-latest
version: linked
- os: ubuntu-latest
version: linked
name: Zstandard bundle fallback
permissions:
contents: read
security-events: write
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Setup Python on MacOS
uses: actions/setup-python@v5
if: >-
runner.os == 'macOS' && (
matrix.version == 'stable-v2.13.5' ||
matrix.version == 'stable-v2.14.6')
with:
python-version: '3.11'
- name: Check out repository
uses: actions/checkout@v4
- name: Prepare test
id: prepare-test
uses: ./.github/actions/prepare-test
with:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check expected diagnostics
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.gz')) {
core.setFailed(
`Expected the tools URL to be a .tar.gz file, but found '${toolsUrl}'.`
);
}
const zstdFailureReason = downloadTelemetryNotifications[0].properties.attributes.zstdFailureReason;
console.log(`Found zstd failure reason: ${zstdFailureReason}`);
const expectedZstdFailureReason = 'Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.';
if (zstdFailureReason !== expectedZstdFailureReason) {
core.setFailed(
`Expected the zstd failure reason to be '${expectedZstdFailureReason}', but found '${zstdFailureReason}'.`
);
}
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_FORCE_ZSTD_FAILURE: true
CODEQL_ACTION_TEST_MODE: true

119
.github/workflows/__zstd-bundle.yml generated vendored Normal file
View file

@ -0,0 +1,119 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# (cd pr-checks; pip install ruamel.yaml@0.17.31 && python3 sync.py)
# to regenerate this file.
name: PR Check - Zstandard bundle
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- releases/v*
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
schedule:
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
zstd-bundle:
strategy:
fail-fast: false
matrix:
include:
- os: macos-latest
version: linked
- os: windows-latest
version: linked
- os: ubuntu-latest
version: linked
name: Zstandard bundle
permissions:
contents: read
security-events: write
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Setup Python on MacOS
uses: actions/setup-python@v5
if: >-
runner.os == 'macOS' && (
matrix.version == 'stable-v2.13.5' ||
matrix.version == 'stable-v2.14.6')
with:
python-version: '3.11'
- name: Check out repository
uses: actions/checkout@v4
- name: Prepare test
id: prepare-test
uses: ./.github/actions/prepare-test
with:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check diagnostic with expected tools URL appears in SARIF
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.zst')) {
core.setFailed(
`Expected the tools URL to be a .tar.zst file, but found ${toolsUrl}.`
);
}
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_TEST_MODE: true

2
CHANGELOG.md
View file

@ -6,7 +6,7 @@ Note that the only difference between `v2` and `v3` of the CodeQL Action is the
## [UNRELEASED]
No user facing changes.
- We are rolling out a feature in September/October 2024 that sets up CodeQL using a bundle compressed with [Zstandard](http://facebook.github.io/zstd/). Our aim is to improve the performance of setting up CodeQL. [#2502](https://github.com/github/codeql-action/pull/2502)
## 3.26.9 - 24 Sep 2024

5
lib/codeql.js generated
View file

@ -122,9 +122,9 @@ const CODEQL_VERSION_CACHE_CLEANUP = "2.17.1";
* version requirement. Must be set to true outside tests.
* @returns a { CodeQL, toolsVersion } object.
*/
async function setupCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger, checkVersion) {
async function setupCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger, checkVersion) {
try {
const { codeqlFolder, toolsDownloadStatusReport, toolsSource, toolsVersion, } = await setupCodeql.setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger);
const { codeqlFolder, toolsDownloadStatusReport, toolsSource, toolsVersion, zstdAvailability, } = await setupCodeql.setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger);
logger.debug(`Bundle download status report: ${JSON.stringify(toolsDownloadStatusReport)}`);
let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
if (process.platform === "win32") {
@ -139,6 +139,7 @@ async function setupCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliV
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
};
}
catch (e) {

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

20
lib/codeql.test.js generated
View file

@ -60,7 +60,7 @@ async function installIntoToolcache({ apiDetails = testing_utils_1.SAMPLE_DOTCOM
const url = (0, testing_utils_1.mockBundleDownloadApi)({ apiDetails, isPinned, tagName });
await codeql.setupCodeQL(cliVersion !== undefined ? undefined : url, apiDetails, tmpDir, util.GitHubVariant.GHES, cliVersion !== undefined
? { cliVersion, tagName }
: testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
: testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
}
function mockReleaseApi({ apiDetails = testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, assetNames, tagName, }) {
return (0, nock_1.default)(apiDetails.apiURL)
@ -97,7 +97,7 @@ function mockApiDetails(apiDetails) {
tagName: `codeql-bundle-${version}`,
isPinned: false,
});
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
@ -113,7 +113,7 @@ function mockApiDetails(apiDetails) {
tagName: `codeql-bundle-v2.14.0`,
isPinned: false,
});
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(toolcache.findAllVersions("CodeQL").length, 1);
t.assert(toolcache.find("CodeQL", `2.14.0`));
t.is(result.toolsVersion, `2.14.0`);
@ -132,7 +132,7 @@ function mockApiDetails(apiDetails) {
const url = (0, testing_utils_1.mockBundleDownloadApi)({
tagName: "codeql-bundle-20200610",
});
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
@ -158,7 +158,7 @@ for (const { tagName, expectedToolcacheVersion, } of EXPLICITLY_REQUESTED_BUNDLE
const url = (0, testing_utils_1.mockBundleDownloadApi)({
tagName,
});
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL(url, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.assert(toolcache.find("CodeQL", expectedToolcacheVersion));
t.deepEqual(result.toolsVersion, expectedToolcacheVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
@ -181,7 +181,7 @@ for (const toolcacheVersion of [
.withArgs("CodeQL", toolcacheVersion)
.returns("path/to/cached/codeql");
sinon.stub(toolcache, "findAllVersions").returns([toolcacheVersion]);
const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsVersion, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
@ -199,7 +199,7 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.GHES, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
}, (0, logging_1.getRunnerLogger)(true), false);
}, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
@ -221,7 +221,7 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.GHES, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
}, (0, logging_1.getRunnerLogger)(true), false);
}, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
@ -240,7 +240,7 @@ for (const toolcacheVersion of [
(0, testing_utils_1.mockBundleDownloadApi)({
tagName: defaults.bundleVersion,
});
const result = await codeql.setupCodeQL("latest", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL("latest", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
@ -262,7 +262,7 @@ for (const toolcacheVersion of [
platformSpecific: false,
tagName: "codeql-bundle-20230203",
});
const result = await codeql.setupCodeQL("https://github.com/codeql-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
const result = await codeql.setupCodeQL("https://github.com/codeql-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.true(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));

2
lib/codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

14
lib/feature-flags.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Features = exports.FEATURE_FLAGS_FILE_NAME = exports.featureConfig = exports.Feature = exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = void 0;
exports.Features = exports.FEATURE_FLAGS_FILE_NAME = exports.featureConfig = exports.Feature = exports.CODEQL_VERSION_ZSTD_BUNDLE = exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const semver = __importStar(require("semver"));
@ -37,6 +37,10 @@ const DEFAULT_VERSION_FEATURE_FLAG_SUFFIX = "_enabled";
* (Some earlier versions recognize the command-line flag, but they contain a bug which makes it unsafe to use).
*/
exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = "2.15.1";
/**
* The first version of the CodeQL Bundle that shipped with zstd-compressed bundles.
*/
exports.CODEQL_VERSION_ZSTD_BUNDLE = "2.19.0";
/**
* Feature enablement as returned by the GitHub API endpoint.
*
@ -51,6 +55,7 @@ var Feature;
Feature["DisableKotlinAnalysisEnabled"] = "disable_kotlin_analysis_enabled";
Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
Feature["ZstdBundle"] = "zstd_bundle";
})(Feature || (exports.Feature = Feature = {}));
exports.featureConfig = {
[Feature.CleanupTrapCaches]: {
@ -93,6 +98,13 @@ exports.featureConfig = {
legacyApi: true,
minimumVersion: undefined,
},
[Feature.ZstdBundle]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE",
// We haven't yet installed CodeQL when we check this feature flag, so we need to implement the
// version check separately.
minimumVersion: undefined,
},
};
exports.FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
/**

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

15
lib/init-action.js generated
View file

@ -40,7 +40,6 @@ const logging_1 = require("./logging");
const repository_1 = require("./repository");
const setup_codeql_1 = require("./setup-codeql");
const status_report_1 = require("./status-report");
const tar_1 = require("./tar");
const tools_features_1 = require("./tools-features");
const trap_caching_1 = require("./trap-caching");
const util_1 = require("./util");
@ -136,6 +135,7 @@ async function run() {
let toolsFeatureFlagsValid;
let toolsSource;
let toolsVersion;
let zstdAvailability;
const apiDetails = {
auth: (0, actions_util_1.getRequiredInput)("token"),
externalRepoAuth: (0, actions_util_1.getOptionalInput)("external-repository-token"),
@ -159,11 +159,12 @@ async function run() {
}
const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(gitHubVersion.type);
toolsFeatureFlagsValid = codeQLDefaultVersionInfo.toolsFeatureFlagsValid;
const initCodeQLResult = await (0, init_1.initCodeQL)((0, actions_util_1.getOptionalInput)("tools"), apiDetails, (0, actions_util_1.getTemporaryDirectory)(), gitHubVersion.type, codeQLDefaultVersionInfo, logger);
const initCodeQLResult = await (0, init_1.initCodeQL)((0, actions_util_1.getOptionalInput)("tools"), apiDetails, (0, actions_util_1.getTemporaryDirectory)(), gitHubVersion.type, codeQLDefaultVersionInfo, features, logger);
codeql = initCodeQLResult.codeql;
toolsDownloadStatusReport = initCodeQLResult.toolsDownloadStatusReport;
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
zstdAvailability = initCodeQLResult.zstdAvailability;
core.startGroup("Validating workflow");
if ((await (0, workflow_1.validateWorkflow)(codeql, logger)) === undefined) {
logger.info("Detected no issues with the code scanning workflow.");
@ -209,7 +210,9 @@ async function run() {
}
try {
(0, init_1.cleanupDatabaseClusterDirectory)(config, logger);
await logZstdAvailability(config, logger);
if (zstdAvailability) {
await recordZstdAvailability(config, zstdAvailability);
}
// Log CodeQL download telemetry, if appropriate
if (toolsDownloadStatusReport) {
(0, diagnostics_1.addDiagnostic)(config,
@ -392,14 +395,12 @@ function getTrapCachingEnabled() {
// On hosted runners, enable TRAP caching by default
return true;
}
async function logZstdAvailability(config, logger) {
// Log zstd availability
const zstdAvailableResult = await (0, tar_1.isZstdAvailable)(logger);
async function recordZstdAvailability(config, zstdAvailability) {
(0, diagnostics_1.addDiagnostic)(config,
// Arbitrarily choose the first language. We could also choose all languages, but that
// increases the risk of misinterpreting the data.
config.languages[0], (0, diagnostics_1.makeDiagnostic)("codeql-action/zstd-availability", "Zstandard availability", {
attributes: zstdAvailableResult,
attributes: zstdAvailability,
visibility: {
cliSummaryTable: false,
statusPage: false,

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

12
lib/init.js generated
View file

@ -40,12 +40,18 @@ const languages_1 = require("./languages");
const tools_features_1 = require("./tools-features");
const tracer_config_1 = require("./tracer-config");
const util = __importStar(require("./util"));
async function initCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger) {
async function initCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger) {
logger.startGroup("Setup CodeQL tools");
const { codeql, toolsDownloadStatusReport, toolsSource, toolsVersion } = await (0, codeql_1.setupCodeQL)(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger, true);
const { codeql, toolsDownloadStatusReport, toolsSource, toolsVersion, zstdAvailability, } = await (0, codeql_1.setupCodeQL)(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger, true);
await codeql.printVersion();
logger.endGroup();
return { codeql, toolsDownloadStatusReport, toolsSource, toolsVersion };
return {
codeql,
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
};
}
async function initConfig(inputs, codeql) {
const logger = inputs.logger;

2
lib/init.js.map
View file

@ -1 +1 @@
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAkBA,gCA2BC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AAhMD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAG1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,MAAc;IAOd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,EAAE,yBAAyB,EAAE,WAAW,EAAE,YAAY,EAAE,GACpE,MAAM,IAAA,oBAAW,EACf,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,MAAM,EACN,IAAI,CACL,CAAC;IACJ,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AAC1E,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AA/MD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAI1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

61
lib/setup-codeql.js generated
View file

@ -48,6 +48,7 @@ const api = __importStar(require("./api-client"));
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
const defaults = __importStar(require("./defaults.json"));
const feature_flags_1 = require("./feature-flags");
const tar = __importStar(require("./tar"));
const util = __importStar(require("./util"));
const util_1 = require("./util");
@ -60,7 +61,11 @@ var ToolsSource;
})(ToolsSource || (exports.ToolsSource = ToolsSource = {}));
exports.CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_BUNDLE_VERSION_ALIAS = ["linked", "latest"];
function getCodeQLBundleName() {
function getCodeQLBundleExtension(useZstd) {
return useZstd ? ".tar.zst" : ".tar.gz";
}
function getCodeQLBundleName(useZstd) {
const extension = getCodeQLBundleExtension(useZstd);
let platform;
if (process.platform === "win32") {
platform = "win64";
@ -72,9 +77,9 @@ function getCodeQLBundleName() {
platform = "osx64";
}
else {
return "codeql-bundle.tar.gz";
return `codeql-bundle${extension}`;
}
return `codeql-bundle-${platform}.tar.gz`;
return `codeql-bundle-${platform}${extension}`;
}
function getCodeQLActionRepository(logger) {
if ((0, actions_util_1.isRunningLocalAction)()) {
@ -86,7 +91,7 @@ function getCodeQLActionRepository(logger) {
}
return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
}
async function getCodeQLBundleDownloadURL(tagName, apiDetails, logger) {
async function getCodeQLBundleDownloadURL(tagName, apiDetails, useZstd, logger) {
const codeQLActionRepository = getCodeQLActionRepository(logger);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
@ -101,7 +106,7 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, logger) {
const uniqueDownloadSources = potentialDownloadSources.filter((source, index, self) => {
return !self.slice(0, index).some((other) => (0, fast_deep_equal_1.default)(source, other));
});
const codeQLBundleName = getCodeQLBundleName();
const codeQLBundleName = getCodeQLBundleName(useZstd);
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
@ -193,7 +198,7 @@ async function findOverridingToolsInCache(humanReadableVersion, logger) {
}
return undefined;
}
async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, variant, logger) {
async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, variant, tarSupportsZstd, features, logger) {
if (toolsInput &&
!CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput) &&
!toolsInput.startsWith("http")) {
@ -335,7 +340,8 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
}
}
if (!url) {
url = await getCodeQLBundleDownloadURL(tagName, apiDetails, logger);
url = await getCodeQLBundleDownloadURL(tagName, apiDetails, cliVersion !== undefined &&
(await useZstdBundle(cliVersion, features, tarSupportsZstd)), logger);
}
if (cliVersion) {
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url}.`);
@ -471,8 +477,32 @@ function getCanonicalToolcacheVersion(cliVersion, bundleVersion, logger) {
*
* @returns the path to the extracted bundle, and the version of the tools
*/
async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger) {
const source = await getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, variant, logger);
async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger) {
const zstdAvailability = await tar.isZstdAvailable(logger);
let zstdFailureReason;
// If we think the installed version of tar supports zstd, try to use zstd,
// but be prepared to fall back to gzip in case we were wrong.
if (zstdAvailability.available) {
try {
// To facilitate testing the fallback, fail here if a testing environment variable is set.
if (process.env.CODEQL_ACTION_FORCE_ZSTD_FAILURE === "true") {
throw new Error("Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.");
}
return await setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger, zstdAvailability, true);
}
catch (e) {
zstdFailureReason = util.getErrorMessage(e) || "unknown error";
logger.warning(`Failed to set up CodeQL tools with zstd. Falling back to gzipped version. Error: ${util.getErrorMessage(e)}`);
}
}
const result = await setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger, zstdAvailability, false);
if (result.toolsDownloadStatusReport && zstdFailureReason) {
result.toolsDownloadStatusReport.zstdFailureReason = zstdFailureReason;
}
return result;
}
async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, features, logger, zstdAvailability, useTarIfAvailable) {
const source = await getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, variant, useTarIfAvailable, features, logger);
let codeqlFolder;
let toolsVersion = source.toolsVersion;
let toolsDownloadStatusReport;
@ -500,7 +530,13 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau
default:
util.assertNever(source);
}
return { codeqlFolder, toolsDownloadStatusReport, toolsSource, toolsVersion };
return {
codeqlFolder,
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
};
}
async function cleanUpGlob(glob, name, logger) {
logger.debug(`Cleaning up ${name}.`);
@ -525,4 +561,9 @@ function sanitizeUrlForStatusReport(url) {
? url
: "sanitized-value";
}
async function useZstdBundle(cliVersion, features, tarSupportsZstd) {
return (tarSupportsZstd &&
semver.gte(cliVersion, feature_flags_1.CODEQL_VERSION_ZSTD_BUNDLE) &&
!!(await features.getValue(feature_flags_1.Feature.ZstdBundle)));
}
//# sourceMappingURL=setup-codeql.js.map

2
lib/setup-codeql.js.map
View file

File diff suppressed because one or more lines are too long

10
lib/setup-codeql.test.js generated
View file

@ -79,7 +79,7 @@ ava_1.default.beforeEach(() => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const tagName = "codeql-bundle-v1.2.3";
(0, testing_utils_1.mockBundleDownloadApi)({ tagName });
const source = await setupCodeql.getCodeQLSource(`https://github.com/github/codeql-action/releases/download/${tagName}/codeql-bundle-linux64.tar.gz`, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, util_1.GitHubVariant.DOTCOM, (0, logging_1.getRunnerLogger)(true));
const source = await setupCodeql.getCodeQLSource(`https://github.com/github/codeql-action/releases/download/${tagName}/codeql-bundle-linux64.tar.gz`, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, util_1.GitHubVariant.DOTCOM, false, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
t.is(source.sourceType, "download");
t.is(source["cliVersion"], "1.2.3");
});
@ -87,7 +87,7 @@ ava_1.default.beforeEach(() => {
(0, ava_1.default)("getCodeQLSource correctly returns bundled CLI version when tools == linked", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const source = await setupCodeql.getCodeQLSource("linked", testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, util_1.GitHubVariant.DOTCOM, (0, logging_1.getRunnerLogger)(true));
const source = await setupCodeql.getCodeQLSource("linked", testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, util_1.GitHubVariant.DOTCOM, false, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
t.is(source.toolsVersion, testing_utils_1.LINKED_CLI_VERSION.cliVersion);
t.is(source.sourceType, "download");
});
@ -97,7 +97,7 @@ ava_1.default.beforeEach(() => {
const logger = (0, testing_utils_1.getRecordingLogger)(loggedMessages);
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const source = await setupCodeql.getCodeQLSource("latest", testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, util_1.GitHubVariant.DOTCOM, logger);
const source = await setupCodeql.getCodeQLSource("latest", testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, util_1.GitHubVariant.DOTCOM, false, (0, testing_utils_1.createFeatures)([]), logger);
// First, ensure that the CLI version is the linked version, so that backwards
// compatibility is maintained.
t.is(source.toolsVersion, testing_utils_1.LINKED_CLI_VERSION.cliVersion);
@ -125,7 +125,7 @@ ava_1.default.beforeEach(() => {
});
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const result = await setupCodeql.setupCodeQLBundle("linked", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, "tmp/codeql_action_test/", util_1.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, logger);
const result = await setupCodeql.setupCodeQLBundle("linked", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, "tmp/codeql_action_test/", util_1.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), logger);
// Basic sanity check that the version we got back is indeed
// the linked (default) CLI version.
t.is(result.toolsVersion, testing_utils_1.LINKED_CLI_VERSION.cliVersion);
@ -154,7 +154,7 @@ ava_1.default.beforeEach(() => {
});
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const result = await setupCodeql.setupCodeQLBundle(bundleUrl, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, "tmp/codeql_action_test/", util_1.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, logger);
const result = await setupCodeql.setupCodeQLBundle(bundleUrl, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, "tmp/codeql_action_test/", util_1.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), logger);
// Basic sanity check that the version we got back is indeed the version that the
// bundle contains..
t.is(result.toolsVersion, expectedVersion);

2
lib/setup-codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/tar.js.map
View file

@ -1 +1 @@
{"version":3,"file":"tar.js","sourceRoot":"","sources":["../src/tar.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,0CA4BC;AAID,0BAeC;AAED,wDAKC;AAtGD,6DAA0D;AAC1D,+DAAiD;AACjD,uDAAmD;AAGnD,iCAAqC;AAErC,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAC7C,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAO5C,KAAK,UAAU,aAAa;IAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAS,EAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,uBAAU,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE;QACxD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IACV,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,oEAAoE;IACpE,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,gBAAgB,OAAO,GAAG,CAAC,CAAC;QACrD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,OAAO,IAAI,4BAA4B;oBAClD,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,OAAO,IAAI,4BAA4B;oBAClD,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ;gBACE,IAAA,kBAAW,EAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,oFAAoF;YAClF,6BAA6B,CAAC,EAAE,CACnC,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;AACH,CAAC;AAIM,KAAK,UAAU,OAAO,CAC3B,IAAY,EACZ,iBAAoC;IAEpC,QAAQ,iBAAiB,EAAE,CAAC;QAC1B,KAAK,MAAM;YACT,oEAAoE;YACpE,sEAAsE;YACtE,yCAAyC;YACzC,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1C,KAAK,MAAM;YACT,gEAAgE;YAChE,sBAAsB;YACtB,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,SAAgB,sBAAsB,CAAC,IAAY;IACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
{"version":3,"file":"tar.js","sourceRoot":"","sources":["../src/tar.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,0CA4BC;AAID,0BAeC;AAED,wDAKC;AA3GD,6DAA0D;AAC1D,+DAAiD;AACjD,uDAAmD;AAGnD,iCAAqC;AAErC,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAC7C,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAO5C,KAAK,UAAU,aAAa;IAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAS,EAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,uBAAU,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE;QACxD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IACV,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,oEAAoE;IACpE,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAOM,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,gBAAgB,OAAO,GAAG,CAAC,CAAC;QACrD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,OAAO,IAAI,4BAA4B;oBAClD,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,OAAO,IAAI,4BAA4B;oBAClD,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ;gBACE,IAAA,kBAAW,EAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,oFAAoF;YAClF,6BAA6B,CAAC,EAAE,CACnC,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;AACH,CAAC;AAIM,KAAK,UAAU,OAAO,CAC3B,IAAY,EACZ,iBAAoC;IAEpC,QAAQ,iBAAiB,EAAE,CAAC;QAC1B,KAAK,MAAM;YACT,oEAAoE;YACpE,sEAAsE;YACtE,yCAAyC;YACzC,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1C,KAAK,MAAM;YACT,gEAAgE;YAChE,sBAAsB;YACtB,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,SAAgB,sBAAsB,CAAC,IAAY;IACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

2
lib/upload-lib.js generated
View file

@ -173,7 +173,7 @@ async function combineSarifFilesUsingCLI(sarifFiles, gitHubVersion, features, lo
};
const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(gitHubVersion.type);
const initCodeQLResult = await (0, init_1.initCodeQL)(undefined, // There is no tools input on the upload action
apiDetails, tempDir, gitHubVersion.type, codeQLDefaultVersionInfo, logger);
apiDetails, tempDir, gitHubVersion.type, codeQLDefaultVersionInfo, features, logger);
codeQL = initCodeQLResult.codeql;
}
if (!(await codeQL.supportsFeature(tools_features_1.ToolsFeature.SarifMergeRunsFromEqualCategory))) {

2
lib/upload-lib.js.map
View file

File diff suppressed because one or more lines are too long

77
pr-checks/checks/zstd-bundle-fallback.yml Normal file
View file

@ -0,0 +1,77 @@
name: "Zstandard bundle fallback"
description: "Tests the fallback when downloading a Zstandard-compressed CodeQL Bundle fails"
versions:
- linked
operatingSystems:
- macos
- windows
- ubuntu
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_FORCE_ZSTD_FAILURE: true
steps:
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check expected diagnostics
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.gz')) {
core.setFailed(
`Expected the tools URL to be a .tar.gz file, but found '${toolsUrl}'.`
);
}
const zstdFailureReason = downloadTelemetryNotifications[0].properties.attributes.zstdFailureReason;
console.log(`Found zstd failure reason: ${zstdFailureReason}`);
const expectedZstdFailureReason = 'Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.';
if (zstdFailureReason !== expectedZstdFailureReason) {
core.setFailed(
`Expected the zstd failure reason to be '${expectedZstdFailureReason}', but found '${zstdFailureReason}'.`
);
}

66
pr-checks/checks/zstd-bundle.yml Normal file
View file

@ -0,0 +1,66 @@
name: "Zstandard bundle"
description: "Tests the feature flag that downloads a Zstandard-compressed CodeQL Bundle by default"
versions:
- linked
operatingSystems:
- macos
- windows
- ubuntu
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
steps:
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check diagnostic with expected tools URL appears in SARIF
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.zst')) {
core.setFailed(
`Expected the tools URL to be a .tar.zst file, but found ${toolsUrl}.`
);
}

5
pr-checks/sync.py
View file

@ -65,7 +65,10 @@ for file in (this_dir / 'checks').glob('*.yml'):
matrix = []
excludedOsesAndVersions = checkSpecification.get('excludeOsAndVersionCombination', [])
for version in checkSpecification.get('versions', defaultTestVersions):
runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"]
if version == "latest":
raise ValueError('Did not recognize "version: latest". Did you mean "version: linked"?')
runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"]
operatingSystems = checkSpecification.get('operatingSystems', ["ubuntu", "macos", "windows"])
for operatingSystem in operatingSystems:

20
src/codeql.test.ts
View file

@ -68,6 +68,8 @@ async function installIntoToolcache({
cliVersion !== undefined
? { cliVersion, tagName }
: SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -127,6 +129,8 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -156,6 +160,8 @@ test("caches semantically versioned bundles using their semantic version number"
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -189,6 +195,8 @@ test("downloads an explicitly requested bundle even if a different version is ca
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -233,6 +241,8 @@ for (const {
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -271,6 +281,8 @@ for (const toolcacheVersion of [
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -301,6 +313,8 @@ test(`uses a cached bundle when no tools input is given on GHES`, async (t) => {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
},
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -335,6 +349,8 @@ test(`downloads bundle if only an unpinned version is cached on GHES`, async (t)
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
},
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -368,6 +384,8 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);
@ -404,6 +422,8 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
tmpDir,
util.GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
getRunnerLogger(true),
false,
);

6
src/codeql.ts
View file

@ -28,6 +28,7 @@ import {
import { Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
@ -351,6 +352,7 @@ export async function setupCodeQL(
tempDir: string,
variant: util.GitHubVariant,
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
checkVersion: boolean,
): Promise<{
@ -358,6 +360,7 @@ export async function setupCodeQL(
toolsDownloadStatusReport?: setupCodeql.ToolsDownloadStatusReport;
toolsSource: setupCodeql.ToolsSource;
toolsVersion: string;
zstdAvailability: ZstdAvailability;
}> {
try {
const {
@ -365,12 +368,14 @@ export async function setupCodeQL(
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
} = await setupCodeql.setupCodeQLBundle(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
features,
logger,
);
@ -395,6 +400,7 @@ export async function setupCodeQL(
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
};
} catch (e) {
throw new Error(

13
src/feature-flags.ts
View file

@ -20,6 +20,11 @@ const DEFAULT_VERSION_FEATURE_FLAG_SUFFIX = "_enabled";
*/
export const CODEQL_VERSION_FINE_GRAINED_PARALLELISM = "2.15.1";
/**
* The first version of the CodeQL Bundle that shipped with zstd-compressed bundles.
*/
export const CODEQL_VERSION_ZSTD_BUNDLE = "2.19.0";
export interface CodeQLDefaultVersionInfo {
cliVersion: string;
tagName: string;
@ -47,6 +52,7 @@ export enum Feature {
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
ZstdBundle = "zstd_bundle",
}
export const featureConfig: Record<
@ -120,6 +126,13 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: undefined,
},
[Feature.ZstdBundle]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE",
// We haven't yet installed CodeQL when we check this feature flag, so we need to implement the
// version check separately.
minimumVersion: undefined,
},
};
/**

18
src/init-action.ts
View file

@ -42,7 +42,7 @@ import {
getActionsStatus,
sendStatusReport,
} from "./status-report";
import { isZstdAvailable } from "./tar";
import { ZstdAvailability } from "./tar";
import { ToolsFeature } from "./tools-features";
import { getTotalCacheSize } from "./trap-caching";
import {
@ -255,6 +255,7 @@ async function run() {
let toolsFeatureFlagsValid: boolean | undefined;
let toolsSource: ToolsSource;
let toolsVersion: string;
let zstdAvailability: ZstdAvailability | undefined;
const apiDetails = {
auth: getRequiredInput("token"),
@ -308,12 +309,14 @@ async function run() {
getTemporaryDirectory(),
gitHubVersion.type,
codeQLDefaultVersionInfo,
features,
logger,
);
codeql = initCodeQLResult.codeql;
toolsDownloadStatusReport = initCodeQLResult.toolsDownloadStatusReport;
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
zstdAvailability = initCodeQLResult.zstdAvailability;
core.startGroup("Validating workflow");
if ((await validateWorkflow(codeql, logger)) === undefined) {
@ -377,7 +380,9 @@ async function run() {
try {
cleanupDatabaseClusterDirectory(config, logger);
await logZstdAvailability(config, logger);
if (zstdAvailability) {
await recordZstdAvailability(config, zstdAvailability);
}
// Log CodeQL download telemetry, if appropriate
if (toolsDownloadStatusReport) {
@ -674,9 +679,10 @@ function getTrapCachingEnabled(): boolean {
return true;
}
async function logZstdAvailability(config: configUtils.Config, logger: Logger) {
// Log zstd availability
const zstdAvailableResult = await isZstdAvailable(logger);
async function recordZstdAvailability(
config: configUtils.Config,
zstdAvailability: ZstdAvailability,
) {
addDiagnostic(
config,
// Arbitrarily choose the first language. We could also choose all languages, but that
@ -686,7 +692,7 @@ async function logZstdAvailability(config: configUtils.Config, logger: Logger) {
"codeql-action/zstd-availability",
"Zstandard availability",
{
attributes: zstdAvailableResult,
attributes: zstdAvailability,
visibility: {
cliSummaryTable: false,
statusPage: false,

39
src/init.ts
View file

@ -8,10 +8,11 @@ import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo } from "./feature-flags";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsFeature } from "./tools-features";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
@ -22,27 +23,41 @@ export async function initCodeQL(
tempDir: string,
variant: util.GitHubVariant,
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
): Promise<{
codeql: CodeQL;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: ToolsSource;
toolsVersion: string;
zstdAvailability: ZstdAvailability;
}> {
logger.startGroup("Setup CodeQL tools");
const { codeql, toolsDownloadStatusReport, toolsSource, toolsVersion } =
await setupCodeQL(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
logger,
true,
);
const {
codeql,
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
} = await setupCodeQL(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
features,
logger,
true,
);
await codeql.printVersion();
logger.endGroup();
return { codeql, toolsDownloadStatusReport, toolsSource, toolsVersion };
return {
codeql,
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
};
}
export async function initConfig(

9
src/setup-codeql.test.ts
View file

@ -11,6 +11,7 @@ import {
LoggedMessage,
SAMPLE_DEFAULT_CLI_VERSION,
SAMPLE_DOTCOM_API_DETAILS,
createFeatures,
getRecordingLogger,
mockBundleDownloadApi,
setupActionsVars,
@ -89,6 +90,8 @@ test("getCodeQLSource sets CLI version for a semver tagged bundle", async (t) =>
SAMPLE_DEFAULT_CLI_VERSION,
SAMPLE_DOTCOM_API_DETAILS,
GitHubVariant.DOTCOM,
false,
createFeatures([]),
getRunnerLogger(true),
);
@ -105,6 +108,8 @@ test("getCodeQLSource correctly returns bundled CLI version when tools == linked
SAMPLE_DEFAULT_CLI_VERSION,
SAMPLE_DOTCOM_API_DETAILS,
GitHubVariant.DOTCOM,
false,
createFeatures([]),
getRunnerLogger(true),
);
@ -124,6 +129,8 @@ test("getCodeQLSource correctly returns bundled CLI version when tools == latest
SAMPLE_DEFAULT_CLI_VERSION,
SAMPLE_DOTCOM_API_DETAILS,
GitHubVariant.DOTCOM,
false,
createFeatures([]),
logger,
);
@ -170,6 +177,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
"tmp/codeql_action_test/",
GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
logger,
);
@ -218,6 +226,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
"tmp/codeql_action_test/",
GitHubVariant.DOTCOM,
SAMPLE_DEFAULT_CLI_VERSION,
createFeatures([]),
logger,
);

120
src/setup-codeql.ts
View file

@ -15,7 +15,12 @@ import * as api from "./api-client";
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
import * as defaults from "./defaults.json";
import { CodeQLDefaultVersionInfo } from "./feature-flags";
import {
CODEQL_VERSION_ZSTD_BUNDLE,
CodeQLDefaultVersionInfo,
Feature,
FeatureEnablement,
} from "./feature-flags";
import { Logger } from "./logging";
import * as tar from "./tar";
import * as util from "./util";
@ -32,7 +37,13 @@ export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_BUNDLE_VERSION_ALIAS: string[] = ["linked", "latest"];
function getCodeQLBundleName(): string {
function getCodeQLBundleExtension(useZstd: boolean): string {
return useZstd ? ".tar.zst" : ".tar.gz";
}
function getCodeQLBundleName(useZstd: boolean): string {
const extension = getCodeQLBundleExtension(useZstd);
let platform: string;
if (process.platform === "win32") {
platform = "win64";
@ -41,9 +52,9 @@ function getCodeQLBundleName(): string {
} else if (process.platform === "darwin") {
platform = "osx64";
} else {
return "codeql-bundle.tar.gz";
return `codeql-bundle${extension}`;
}
return `codeql-bundle-${platform}.tar.gz`;
return `codeql-bundle-${platform}${extension}`;
}
export function getCodeQLActionRepository(logger: Logger): string {
@ -63,6 +74,7 @@ export function getCodeQLActionRepository(logger: Logger): string {
async function getCodeQLBundleDownloadURL(
tagName: string,
apiDetails: api.GitHubApiDetails,
useZstd: boolean,
logger: Logger,
): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(logger);
@ -81,7 +93,7 @@ async function getCodeQLBundleDownloadURL(
return !self.slice(0, index).some((other) => deepEqual(source, other));
},
);
const codeQLBundleName = getCodeQLBundleName();
const codeQLBundleName = getCodeQLBundleName(useZstd);
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
@ -231,6 +243,8 @@ export async function getCodeQLSource(
defaultCliVersion: CodeQLDefaultVersionInfo,
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
tarSupportsZstd: boolean,
features: FeatureEnablement,
logger: Logger,
): Promise<CodeQLToolsSource> {
if (
@ -424,7 +438,13 @@ export async function getCodeQLSource(
}
if (!url) {
url = await getCodeQLBundleDownloadURL(tagName!, apiDetails, logger);
url = await getCodeQLBundleDownloadURL(
tagName!,
apiDetails,
cliVersion !== undefined &&
(await useZstdBundle(cliVersion, features, tarSupportsZstd)),
logger,
);
}
if (cliVersion) {
@ -467,6 +487,7 @@ export interface ToolsDownloadStatusReport {
downloadDurationMs: number;
extractionDurationMs: number;
toolsUrl: string;
zstdFailureReason?: string;
}
// Exported using `export const` for testing purposes. Specifically, we want to
@ -635,6 +656,8 @@ export interface SetupCodeQLResult {
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: ToolsSource;
toolsVersion: string;
zstdAvailability: tar.ZstdAvailability;
zstdFailureReason?: string;
}
/**
@ -648,13 +671,78 @@ export async function setupCodeQLBundle(
tempDir: string,
variant: util.GitHubVariant,
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
): Promise<SetupCodeQLResult> {
const zstdAvailability = await tar.isZstdAvailable(logger);
let zstdFailureReason: string | undefined;
// If we think the installed version of tar supports zstd, try to use zstd,
// but be prepared to fall back to gzip in case we were wrong.
if (zstdAvailability.available) {
try {
// To facilitate testing the fallback, fail here if a testing environment variable is set.
if (process.env.CODEQL_ACTION_FORCE_ZSTD_FAILURE === "true") {
throw new Error(
"Failing since CODEQL_ACTION_FORCE_ZSTD_FAILURE is true.",
);
}
return await setupCodeQLBundleWithCompressionMethod(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
features,
logger,
zstdAvailability,
true,
);
} catch (e) {
zstdFailureReason = util.getErrorMessage(e) || "unknown error";
logger.warning(
`Failed to set up CodeQL tools with zstd. Falling back to gzipped version. Error: ${util.getErrorMessage(
e,
)}`,
);
}
}
const result = await setupCodeQLBundleWithCompressionMethod(
toolsInput,
apiDetails,
tempDir,
variant,
defaultCliVersion,
features,
logger,
zstdAvailability,
false,
);
if (result.toolsDownloadStatusReport && zstdFailureReason) {
result.toolsDownloadStatusReport.zstdFailureReason = zstdFailureReason;
}
return result;
}
async function setupCodeQLBundleWithCompressionMethod(
toolsInput: string | undefined,
apiDetails: api.GitHubApiDetails,
tempDir: string,
variant: util.GitHubVariant,
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
zstdAvailability: tar.ZstdAvailability,
useTarIfAvailable: boolean,
) {
const source = await getCodeQLSource(
toolsInput,
defaultCliVersion,
apiDetails,
variant,
useTarIfAvailable,
features,
logger,
);
@ -694,7 +782,13 @@ export async function setupCodeQLBundle(
default:
util.assertNever(source);
}
return { codeqlFolder, toolsDownloadStatusReport, toolsSource, toolsVersion };
return {
codeqlFolder,
toolsDownloadStatusReport,
toolsSource,
toolsVersion,
zstdAvailability,
};
}
async function cleanUpGlob(glob: string, name: string, logger: Logger) {
@ -722,3 +816,15 @@ function sanitizeUrlForStatusReport(url: string): string {
? url
: "sanitized-value";
}
async function useZstdBundle(
cliVersion: string,
features: FeatureEnablement,
tarSupportsZstd: boolean,
): Promise<boolean> {
return (
tarSupportsZstd &&
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE) &&
!!(await features.getValue(Feature.ZstdBundle))
);
}

7
src/tar.ts
View file

@ -46,9 +46,14 @@ async function getTarVersion(): Promise<TarVersion> {
}
}
export interface ZstdAvailability {
available: boolean;
version?: TarVersion;
}
export async function isZstdAvailable(
logger: Logger,
): Promise<{ available: boolean; version?: TarVersion }> {
): Promise<ZstdAvailability> {
try {
const tarVersion = await getTarVersion();
const { type, version } = tarVersion;

1
src/upload-lib.ts
View file

@ -221,6 +221,7 @@ async function combineSarifFilesUsingCLI(
tempDir,
gitHubVersion.type,
codeQLDefaultVersionInfo,
features,
logger,
);