robojerk/codeql-action

Merge pull request #2538 from github/henrymercer/zstd-stream

Browse source 
Support streaming the download and extraction of zstd bundles
This commit is contained in:
Henry Mercer 2024-10-14 20:16:34 +01:00 • committed by GitHub
commit 563627499b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
51 changed files with 1971 additions and 209 deletions
Download patch file Download diff file Expand all files Collapse all files

118
.github/workflows/__zstd-bundle-streaming.yml generated vendored Normal file
View file

@ -0,0 +1,118 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# (cd pr-checks; pip install ruamel.yaml@0.17.31 && python3 sync.py)
# to regenerate this file.
name: PR Check - Zstandard bundle (streaming)
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- releases/v*
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
schedule:
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
zstd-bundle-streaming:
strategy:
fail-fast: false
matrix:
include:
- os: macos-latest
version: linked
- os: ubuntu-latest
version: linked
name: Zstandard bundle (streaming)
permissions:
contents: read
security-events: write
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Setup Python on MacOS
uses: actions/setup-python@v5
if: >-
runner.os == 'macOS' && (
matrix.version == 'stable-v2.13.5' ||
matrix.version == 'stable-v2.14.6')
with:
python-version: '3.11'
- name: Check out repository
uses: actions/checkout@v4
- name: Prepare test
id: prepare-test
uses: ./.github/actions/prepare-test
with:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check diagnostic with expected tools URL appears in SARIF
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.zst')) {
core.setFailed(
`Expected the tools URL to be a .tar.zst file, but found ${toolsUrl}.`
);
}
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION: true
CODEQL_ACTION_TEST_MODE: true

4
lib/actions-util.js generated
View file

@ -468,7 +468,9 @@ const MAX_STDERR_BUFFER_SIZE = 20000;
async function runTool(cmd, args = [], opts = {}) {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
if (!opts.noStreamStdout) {
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
}
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {

2
lib/actions-util.js.map
View file

File diff suppressed because one or more lines are too long

4
lib/codeql.js generated
View file

@ -241,7 +241,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
async getVersion() {
let result = util.getCachedCodeQlVersion();
if (result === undefined) {
const output = await runCli(cmd, ["version", "--format=json"]);
const output = await runCli(cmd, ["version", "--format=json"], {
noStreamStdout: true,
});
try {
result = JSON.parse(output);
}

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

32
lib/codeql.test.js generated
View file

@ -101,7 +101,6 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
}
t.is(toolcache.findAllVersions("CodeQL").length, 2);
});
@ -118,7 +117,9 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", `2.14.0`));
t.is(result.toolsVersion, `2.14.0`);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
(0, ava_1.default)("downloads an explicitly requested bundle even if a different version is cached", async (t) => {
@ -136,7 +137,9 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [
@ -184,7 +187,9 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsVersion, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
});
});
}
@ -202,7 +207,9 @@ for (const toolcacheVersion of [
}, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
});
@ -224,7 +231,9 @@ for (const toolcacheVersion of [
}, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
@ -243,7 +252,9 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL("latest", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
@ -265,13 +276,22 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL("https://github.com/codeql-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.true(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
t.is(cachedVersions[0], "0.0.0-20230203");
t.false(releasesApiMock.isDone());
});
});
function assertDurationsInteger(t, statusReport) {
t.assert(Number.isInteger(statusReport?.combinedDurationMs));
if (statusReport.downloadDurationMs !== undefined) {
t.assert(Number.isInteger(statusReport?.downloadDurationMs));
t.assert(Number.isInteger(statusReport?.extractionDurationMs));
}
}
(0, ava_1.default)("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);
t.deepEqual(codeql.getExtraOptions({ foo: [42] }, ["foo"], []), ["42"]);

2
lib/codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/feature-flags.js generated
View file

@ -59,6 +59,7 @@ var Feature;
Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
Feature["ZstdBundle"] = "zstd_bundle";
Feature["ZstdBundleStreamingExtraction"] = "zstd_bundle_streaming_extraction";
})(Feature || (exports.Feature = Feature = {}));
exports.featureConfig = {
[Feature.ArtifactV4Upgrade]: {
@ -119,6 +120,11 @@ exports.featureConfig = {
// version check separately.
minimumVersion: undefined,
},
[Feature.ZstdBundleStreamingExtraction]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
minimumVersion: undefined,
},
};
exports.FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
/**

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/init-action.js generated
View file

@ -58,7 +58,7 @@ async function sendCompletedStatusReport(startedAt, config, configFile, toolsDow
workflow_languages: workflowLanguages || "",
};
const initToolsDownloadFields = {};
if (toolsDownloadStatusReport !== undefined) {
if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadStatusReport.downloadDurationMs;
}

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/init.js.map
View file

@ -1 +1 @@
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AA/MD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAI1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AAhND,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAK1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

13
lib/logging.js generated
View file

@ -26,6 +26,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
exports.getActionsLogger = getActionsLogger;
exports.getRunnerLogger = getRunnerLogger;
exports.withGroup = withGroup;
exports.formatDuration = formatDuration;
const core = __importStar(require("@actions/core"));
function getActionsLogger() {
return core;
@ -54,4 +55,16 @@ function withGroup(groupName, f) {
core.endGroup();
}
}
/** Format a duration for use in logs. */
function formatDuration(durationMs) {
if (durationMs < 1000) {
return `${durationMs}ms`;
}
if (durationMs < 60 * 1000) {
return `${(durationMs / 1000).toFixed(1)}s`;
}
const minutes = Math.floor(durationMs / (60 * 1000));
const seconds = Math.floor((durationMs % (60 * 1000)) / 1000);
return `${minutes}m${seconds}s`;
}
//# sourceMappingURL=logging.js.map

2
lib/logging.js.map
View file

@ -1 +1 @@
{"version":3,"file":"logging.js","sourceRoot":"","sources":["../src/logging.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAcA,4CAEC;AAED,0CAcC;AAED,8BAOC;AAzCD,oDAAsC;AActC,SAAgB,gBAAgB;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,eAAe,CAAC,SAAkB;IAChD,OAAO;QACL,sCAAsC;QACtC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,SAAS;QAClD,sCAAsC;QACtC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,sCAAsC;QACtC,OAAO,EAAE,OAAO,CAAC,IAAI;QACrB,sCAAsC;QACtC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;QAC3B,QAAQ,EAAE,GAAG,EAAE,CAAC,SAAS;KAC1B,CAAC;AACJ,CAAC;AAED,SAAgB,SAAS,CAAI,SAAiB,EAAE,CAAU;IACxD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3B,IAAI,CAAC;QACH,OAAO,CAAC,EAAE,CAAC;IACb,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;AACH,CAAC"}
{"version":3,"file":"logging.js","sourceRoot":"","sources":["../src/logging.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAcA,4CAEC;AAED,0CAcC;AAED,8BAOC;AAGD,wCAWC;AAvDD,oDAAsC;AActC,SAAgB,gBAAgB;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,eAAe,CAAC,SAAkB;IAChD,OAAO;QACL,sCAAsC;QACtC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,SAAS;QAClD,sCAAsC;QACtC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,sCAAsC;QACtC,OAAO,EAAE,OAAO,CAAC,IAAI;QACrB,sCAAsC;QACtC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;QAC3B,QAAQ,EAAE,GAAG,EAAE,CAAC,SAAS;KAC1B,CAAC;AACJ,CAAC;AAED,SAAgB,SAAS,CAAI,SAAiB,EAAE,CAAU;IACxD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3B,IAAI,CAAC;QACH,OAAO,CAAC,EAAE,CAAC;IACb,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED,yCAAyC;AACzC,SAAgB,cAAc,CAAC,UAAkB;IAC/C,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;QACtB,OAAO,GAAG,UAAU,IAAI,CAAC;IAC3B,CAAC;IAED,IAAI,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC9D,OAAO,GAAG,OAAO,IAAI,OAAO,GAAG,CAAC;AAClC,CAAC"}

56
lib/setup-codeql.js generated
View file

@ -40,15 +40,13 @@ const perf_hooks_1 = require("perf_hooks");
const toolcache = __importStar(require("@actions/tool-cache"));
const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
const semver = __importStar(require("semver"));
const uuid_1 = require("uuid");
const actions_util_1 = require("./actions-util");
const api = __importStar(require("./api-client"));
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
const defaults = __importStar(require("./defaults.json"));
const feature_flags_1 = require("./feature-flags");
const logging_1 = require("./logging");
const tar = __importStar(require("./tar"));
const tools_download_1 = require("./tools-download");
const util = __importStar(require("./util"));
const util_1 = require("./util");
var ToolsSource;
@ -343,10 +341,10 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
(await useZstdBundle(cliVersion, features, tarSupportsZstd)), logger);
}
if (cliVersion) {
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url}.`);
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url} .`);
}
else {
logger.info(`Using CodeQL CLI sourced from ${url}.`);
logger.info(`Using CodeQL CLI sourced from ${url} .`);
}
return {
bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger),
@ -372,7 +370,7 @@ async function tryGetFallbackToolcacheVersion(cliVersion, tagName, logger) {
}
// Exported using `export const` for testing purposes. Specifically, we want to
// be able to stub this function and have other functions in this file use that stub.
const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, logger) {
const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, features, logger) {
const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers = {
@ -394,56 +392,29 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
else {
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
const dest = path.join(tempDir, (0, uuid_1.v4)());
const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
const toolsDownloadStart = perf_hooks_1.performance.now();
const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
let extractedBundlePath;
let extractionDurationMs;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = perf_hooks_1.performance.now();
extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
}
finally {
await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
}
const { extractedBundlePath, statusReport } = await (0, tools_download_1.downloadAndExtract)(codeqlURL, authorization, { "User-Agent": "CodeQL Action", ...headers }, tarVersion, tempDir, features, logger);
const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`);
return {
codeqlFolder: extractedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? "unknown",
};
}
logger.debug("Caching CodeQL bundle.");
const toolcacheVersion = getCanonicalToolcacheVersion(maybeCliVersion, bundleVersion, logger);
const toolcacheStart = perf_hooks_1.performance.now();
const toolcachedBundlePath = await toolcache.cacheDir(extractedBundlePath, "CodeQL", toolcacheVersion);
logger.info(`Added CodeQL bundle to the tool cache (${(0, logging_1.formatDuration)(perf_hooks_1.performance.now() - toolcacheStart)}).`);
// Defensive check: we expect `cacheDir` to copy the bundle to a new location.
if (toolcachedBundlePath !== extractedBundlePath) {
await (0, util_1.cleanUpGlob)(extractedBundlePath, "CodeQL bundle from temporary directory", logger);
}
return {
codeqlFolder: toolcachedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? toolcacheVersion,
};
};
@ -529,7 +500,7 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
toolsSource = ToolsSource.Toolcache;
break;
case "download": {
const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, logger);
const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, features, logger);
toolsVersion = result.toolsVersion;
codeqlFolder = result.codeqlFolder;
toolsDownloadStatusReport = result.statusReport;
@ -547,11 +518,6 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
zstdAvailability,
};
}
function sanitizeUrlForStatusReport(url) {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
? url
: "sanitized-value";
}
async function useZstdBundle(cliVersion, features, tarSupportsZstd) {
return (
// In testing, gzip performs better than zstd on Windows.

2
lib/setup-codeql.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/setup-codeql.test.js generated
View file

@ -116,9 +116,11 @@ ava_1.default.beforeEach(() => {
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
streamExtraction: false,
toolsUrl: "toolsUrl",
},
toolsVersion: testing_utils_1.LINKED_CLI_VERSION.cliVersion,
@ -145,9 +147,11 @@ ava_1.default.beforeEach(() => {
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
streamExtraction: false,
toolsUrl: bundleUrl,
},
toolsVersion: expectedVersion,
@ -159,7 +163,7 @@ ava_1.default.beforeEach(() => {
// bundle contains..
t.is(result.toolsVersion, expectedVersion);
// Ensure message logging CodeQL CLI version was present in user logs.
const expected_message = `Using CodeQL CLI version 2.16.0 sourced from ${bundleUrl}.`;
const expected_message = `Using CodeQL CLI version 2.16.0 sourced from ${bundleUrl} .`;
t.assert(loggedMessages.some((msg) => typeof msg.message === "string" &&
msg.message.includes(expected_message)));
});

2
lib/setup-codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

47
lib/tar.js generated
View file

@ -30,6 +30,7 @@ exports.isZstdAvailable = isZstdAvailable;
exports.extract = extract;
exports.extractTarZst = extractTarZst;
exports.inferCompressionMethod = inferCompressionMethod;
const child_process_1 = require("child_process");
const fs = __importStar(require("fs"));
const path_1 = __importDefault(require("path"));
const toolrunner_1 = require("@actions/exec/lib/toolrunner");
@ -122,7 +123,7 @@ async function extract(tarPath, compressionMethod, tarVersion, logger) {
if (!tarVersion) {
throw new Error("Could not determine tar version, which is required to extract a Zstandard archive.");
}
return await extractTarZst(tarPath, tarVersion, logger);
return await extractTarZst(fs.createReadStream(tarPath), tarVersion, logger);
}
}
/**
@ -132,37 +133,45 @@ async function extract(tarPath, compressionMethod, tarVersion, logger) {
* @param dest destination directory. Optional.
* @returns path to the destination directory
*/
async function extractTarZst(file, tarVersion, logger) {
if (!file) {
throw new Error("parameter 'file' is required");
}
// Create dest
async function extractTarZst(tarStream, tarVersion, logger) {
const dest = await createExtractFolder();
try {
// Initialize args
const args = ["-x", "-v"];
let destArg = dest;
let fileArg = file;
if (process.platform === "win32" && tarVersion.type === "gnu") {
args.push("--force-local");
destArg = dest.replace(/\\/g, "/");
// Technically only the dest needs to have `/` but for aesthetic consistency
// convert slashes in the file arg too.
fileArg = file.replace(/\\/g, "/");
}
const args = ["-x", "--zstd"];
if (tarVersion.type === "gnu") {
// Suppress warnings when using GNU tar to extract archives created by BSD tar
args.push("--warning=no-unknown-keyword");
args.push("--overwrite");
}
args.push("-C", destArg, "-f", fileArg);
await (0, actions_util_1.runTool)(`tar`, args);
args.push("-f", "-", "-C", dest);
process.stdout.write(`[command]tar ${args.join(" ")}\n`);
const tarProcess = (0, child_process_1.spawn)("tar", args, { stdio: "pipe" });
let stdout = "";
tarProcess.stdout?.on("data", (data) => {
stdout += data.toString();
process.stdout.write(data);
});
let stderr = "";
tarProcess.stderr?.on("data", (data) => {
stderr += data.toString();
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
});
tarStream.pipe(tarProcess.stdin);
await new Promise((resolve, reject) => {
tarProcess.on("exit", (code) => {
if (code !== 0) {
reject(new actions_util_1.CommandInvocationError("tar", args, code ?? undefined, stdout, stderr));
}
resolve();
});
});
return dest;
}
catch (e) {
await (0, util_1.cleanUpGlob)(dest, "extraction destination directory", logger);
throw e;
}
return dest;
}
async function createExtractFolder() {
const dest = path_1.default.join((0, actions_util_1.getTemporaryDirectory)(), (0, uuid_1.v4)());

2
lib/tar.js.map
View file

@ -1 +1 @@
{"version":3,"file":"tar.js","sourceRoot":"","sources":["../src/tar.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,0CA+BC;AAID,0BAmBC;AASD,sCAyCC;AAQD,wDAKC;AA9LD,uCAAyB;AACzB,gDAAwB;AAExB,6DAA0D;AAC1D,+DAAiD;AACjD,uDAAmD;AACnD,+BAAoC;AAEpC,iDAAgE;AAEhE,iCAAkD;AAElD,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAC7C,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAO5C,KAAK,UAAU,kBAAkB,CAC/B,MAAc,EACd,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAA,sBAAS,EAAC,MAAM,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa;IAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAS,EAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,uBAAU,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE;QACxD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IACV,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,oEAAoE;IACpE,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAQM,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,MAAM,eAAe,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,gBAAgB,OAAO,GAAG,CAAC,CAAC;QACrD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,eAAe,IAAI,OAAO,IAAI,4BAA4B;oBACrE,eAAe;oBACf,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,eAAe,IAAI,OAAO,IAAI,4BAA4B;oBACrE,eAAe;oBACf,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ;gBACE,IAAA,kBAAW,EAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,OAAO,CACZ,gFAAgF;YAC9E,6BAA6B,CAAC,EAAE,CACnC,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;IAC/C,CAAC;AACH,CAAC;AAIM,KAAK,UAAU,OAAO,CAC3B,OAAe,EACf,iBAAoC,EACpC,UAAkC,EAClC,MAAc;IAEd,QAAQ,iBAAiB,EAAE,CAAC;QAC1B,KAAK,MAAM;YACT,yEAAyE;YACzE,mCAAmC;YACnC,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC7C,KAAK,MAAM;YACT,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,aAAa,CACjC,IAAY,EACZ,UAAsB,EACtB,MAAc;IAEd,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IAED,cAAc;IACd,MAAM,IAAI,GAAG,MAAM,mBAAmB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,kBAAkB;QAClB,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAE1B,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC9D,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC3B,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAEnC,4EAA4E;YAC5E,uCAAuC;YACvC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC9B,8EAA8E;YAC9E,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAA,sBAAO,EAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAA,kBAAW,EAAC,IAAI,EAAE,kCAAkC,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,CAAC,CAAC;IACV,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,MAAM,IAAI,GAAG,cAAI,CAAC,IAAI,CAAC,IAAA,oCAAqB,GAAE,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,sBAAsB,CAAC,OAAe;IACpD,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
{"version":3,"file":"tar.js","sourceRoot":"","sources":["../src/tar.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2EA,0CA+BC;AAID,0BAuBC;AASD,sCA2DC;AAQD,wDAKC;AAtND,iDAAsC;AACtC,uCAAyB;AACzB,gDAAwB;AAGxB,6DAA0D;AAC1D,+DAAiD;AACjD,uDAAmD;AACnD,+BAAoC;AAEpC,iDAA+E;AAE/E,iCAAkD;AAElD,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAC7C,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAO5C,KAAK,UAAU,kBAAkB,CAC/B,MAAc,EACd,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAA,sBAAS,EAAC,MAAM,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa;IAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAS,EAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,uBAAU,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE;QACxD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IACV,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,oEAAoE;IACpE,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAQM,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,MAAM,eAAe,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,gBAAgB,OAAO,GAAG,CAAC,CAAC;QACrD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,eAAe,IAAI,OAAO,IAAI,4BAA4B;oBACrE,eAAe;oBACf,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,eAAe,IAAI,OAAO,IAAI,4BAA4B;oBACrE,eAAe;oBACf,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ;gBACE,IAAA,kBAAW,EAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,OAAO,CACZ,gFAAgF;YAC9E,6BAA6B,CAAC,EAAE,CACnC,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;IAC/C,CAAC;AACH,CAAC;AAIM,KAAK,UAAU,OAAO,CAC3B,OAAe,EACf,iBAAoC,EACpC,UAAkC,EAClC,MAAc;IAEd,QAAQ,iBAAiB,EAAE,CAAC;QAC1B,KAAK,MAAM;YACT,yEAAyE;YACzE,mCAAmC;YACnC,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC7C,KAAK,MAAM;YACT,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,aAAa,CACxB,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAC5B,UAAU,EACV,MAAM,CACP,CAAC;IACN,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,aAAa,CACjC,SAA0B,EAC1B,UAAsB,EACtB,MAAc;IAEd,MAAM,IAAI,GAAG,MAAM,mBAAmB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,kBAAkB;QAClB,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAE9B,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC9B,8EAA8E;YAC9E,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAEjC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAG,IAAA,qBAAK,EAAC,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YAC7C,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YAC7C,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAEjC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC7B,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CACJ,IAAI,qCAAsB,CACxB,KAAK,EACL,IAAI,EACJ,IAAI,IAAI,SAAS,EACjB,MAAM,EACN,MAAM,CACP,CACF,CAAC;gBACJ,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAA,kBAAW,EAAC,IAAI,EAAE,kCAAkC,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,MAAM,IAAI,GAAG,cAAI,CAAC,IAAI,CAAC,IAAA,oCAAqB,GAAE,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,sBAAsB,CAAC,OAAe;IACpD,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

110
lib/tools-download.js generated Normal file
View file

@ -0,0 +1,110 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadAndExtract = downloadAndExtract;
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const toolcache = __importStar(require("@actions/tool-cache"));
const follow_redirects_1 = require("follow-redirects");
const uuid_1 = require("uuid");
const feature_flags_1 = require("./feature-flags");
const logging_1 = require("./logging");
const tar = __importStar(require("./tar"));
const util_1 = require("./util");
function makeDownloadFirstToolsDownloadDurations(downloadDurationMs, extractionDurationMs) {
return {
combinedDurationMs: downloadDurationMs + extractionDurationMs,
downloadDurationMs,
extractionDurationMs,
streamExtraction: false,
};
}
function makeStreamedToolsDownloadDurations(combinedDurationMs) {
return {
combinedDurationMs,
downloadDurationMs: undefined,
extractionDurationMs: undefined,
streamExtraction: true,
};
}
async function downloadAndExtract(codeqlURL, authorization, headers, tarVersion, tempDir, features, logger) {
logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
if (compressionMethod === "zstd" &&
(await features.getValue(feature_flags_1.Feature.ZstdBundleStreamingExtraction))) {
logger.info(`Streaming the extraction of the CodeQL bundle.`);
const toolsInstallStart = perf_hooks_1.performance.now();
const extractedBundlePath = await downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger);
const combinedDurationMs = Math.round(perf_hooks_1.performance.now() - toolsInstallStart);
logger.info(`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${(0, logging_1.formatDuration)(combinedDurationMs)}).`);
return {
extractedBundlePath,
statusReport: {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeStreamedToolsDownloadDurations(combinedDurationMs),
},
};
}
const dest = path.join(tempDir, (0, uuid_1.v4)());
const toolsDownloadStart = perf_hooks_1.performance.now();
const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, headers);
const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.info(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${(0, logging_1.formatDuration)(downloadDurationMs)}).`);
let extractedBundlePath;
let extractionDurationMs;
try {
logger.info("Extracting CodeQL bundle.");
const extractionStart = perf_hooks_1.performance.now();
extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
logger.info(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${(0, logging_1.formatDuration)(extractionDurationMs)}).`);
}
finally {
await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
}
return {
extractedBundlePath,
statusReport: {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeDownloadFirstToolsDownloadDurations(downloadDurationMs, extractionDurationMs),
},
};
}
async function downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger) {
headers = Object.assign({ "User-Agent": "CodeQL Action", authorization }, headers);
const response = await new Promise((resolve) => follow_redirects_1.https.get(codeqlURL, { headers }, (r) => resolve(r)));
if (response.statusCode !== 200) {
throw new Error(`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`);
}
return await tar.extractTarZst(response, tarVersion, logger);
}
function sanitizeUrlForStatusReport(url) {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
? url
: "sanitized-value";
}
//# sourceMappingURL=tools-download.js.map

1
lib/tools-download.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,gDAsGC;AAzKD,2CAA6B;AAC7B,2CAAyC;AAEzC,+DAAiD;AACjD,uDAAyC;AACzC,+BAAoC;AAEpC,mDAA6D;AAC7D,uCAAmD;AACnD,2CAA6B;AAC7B,iCAAqC;AAarC,SAAS,uCAAuC,CAC9C,kBAA0B,EAC1B,oBAA4B;IAE5B,OAAO;QACL,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;QAC7D,kBAAkB;QAClB,oBAAoB;QACpB,gBAAgB,EAAE,KAAK;KACxB,CAAC;AACJ,CAAC;AAaD,SAAS,kCAAkC,CACzC,kBAA0B;IAE1B,OAAO;QACL,kBAAkB;QAClB,kBAAkB,EAAE,SAAS;QAC7B,oBAAoB,EAAE,SAAS;QAC/B,gBAAgB,EAAE,IAAI;KACvB,CAAC;AACJ,CAAC;AAYM,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,OAAe,EACf,QAA2B,EAC3B,MAAc;IAKd,MAAM,CAAC,IAAI,CACT,iCAAiC,SAAS,2BAA2B,CACtE,CAAC;IAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,IACE,iBAAiB,KAAK,MAAM;QAC5B,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,uBAAO,CAAC,6BAA6B,CAAC,CAAC,EAChE,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAE9D,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,mBAAmB,GAAG,MAAM,mCAAmC,CACnE,SAAS,EACT,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;QACF,MAAM,CAAC,IAAI,CACT,wDAAwD,mBAAmB,KAAK,IAAA,wBAAc,EAC5F,kBAAkB,CACnB,IAAI,CACN,CAAC;QAEF,OAAO;YACL,mBAAmB;YACnB,YAAY,EAAE;gBACZ,iBAAiB;gBACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;gBAC/C,GAAG,kCAAkC,CAAC,kBAAkB,CAAC;aAC1D;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,IAAA,wBAAc,EAC5E,kBAAkB,CACnB,IAAI,CACN,CAAC;IAEF,IAAI,mBAA2B,CAAC;IAChC,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,mBAAmB,GAAG,MAAM,GAAG,CAAC,OAAO,CACrC,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,mBAAmB,KAAK,IAAA,wBAAc,EAC5E,oBAAoB,CACrB,IAAI,CACN,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,YAAY,EAAE;YACZ,iBAAiB;YACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;YAC/C,GAAG,uCAAuC,CACxC,kBAAkB,EAClB,oBAAoB,CACrB;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,EAChD,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,EAAE,CAC9D,wBAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CACrD,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}

30
node_modules/.package-lock.json generated vendored
View file

@ -1236,6 +1236,16 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/follow-redirects": {
"version": "1.14.4",
"resolved": "https://registry.npmjs.org/@types/follow-redirects/-/follow-redirects-1.14.4.tgz",
"integrity": "sha512-GWXfsD0Jc1RWiFmMuMFCpXMzi9L7oPDVwxUnZdg89kDNnqsRfUKXEtUYtA98A6lig1WXH/CYY/fvPW9HuN5fTA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/get-folder-size": {
"version": "2.0.0",
"dev": true,
@ -3996,6 +4006,26 @@
"dev": true,
"license": "ISC"
},
"node_modules/follow-redirects": {
"version": "1.15.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz",
"integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"license": "MIT",
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/for-each": {
"version": "0.3.3",
"dev": true,

21
node_modules/@types/follow-redirects/LICENSE generated vendored Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

15
node_modules/@types/follow-redirects/README.md generated vendored Normal file
View file

@ -0,0 +1,15 @@
# Installation
> `npm install --save @types/follow-redirects`
# Summary
This package contains type definitions for follow-redirects (https://github.com/follow-redirects/follow-redirects).
# Details
Files were exported from https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/follow-redirects.
### Additional Details
* Last updated: Tue, 07 Nov 2023 03:09:37 GMT
* Dependencies: [@types/node](https://npmjs.com/package/@types/node)
# Credits
These definitions were written by [Emily Klassen](https://github.com/forivall), [Claas Ahlrichs](https://github.com/claasahl), and [Piotr Błażejewicz](https://github.com/peterblazejewicz).

3
node_modules/@types/follow-redirects/http.d.ts generated vendored Normal file
View file

@ -0,0 +1,3 @@
import { http } from ".";
export = http;

3
node_modules/@types/follow-redirects/https.d.ts generated vendored Normal file
View file

@ -0,0 +1,3 @@
import { https } from ".";
export = https;

147
node_modules/@types/follow-redirects/index.d.ts generated vendored Normal file
View file

@ -0,0 +1,147 @@
/// <reference types="node" />
import * as coreHttp from "http";
import * as coreHttps from "https";
import { Writable } from "stream";
import { URL } from "url";
export interface WrappableRequest {
getHeader?(...args: any[]): any;
setHeader(...args: any[]): any;
removeHeader(...args: any[]): any;
abort?(...args: any[]): any;
flushHeaders?(...args: any[]): any;
setNoDelay?(...args: any[]): any;
setSocketKeepAlive?(...args: any[]): any;
setTimeout?(...args: any[]): any;
}
export interface WrappableResponse {
statusCode?: number | undefined;
headers: {
location?: string | undefined;
};
destroy(): any;
}
export interface Scheme<Options, Request extends WrappableRequest, Response> {
request(options: Options, callback?: (res: Response) => any): Request;
}
export interface RedirectableRequest<Request extends WrappableRequest, Response> extends Writable {
setHeader: Request["setHeader"];
removeHeader: Request["removeHeader"];
abort: Request["abort"];
flushHeaders: Request["flushHeaders"];
getHeader: Request["getHeader"];
setNoDelay: Request["setNoDelay"];
setSocketKeepAlive: Request["setSocketKeepAlive"];
setTimeout: Request["setTimeout"];
addListener(event: string, listener: (...args: any[]) => void): this;
addListener(event: "response", listener: (response: Response) => void): this;
addListener(event: "error", listener: (err: Error) => void): this;
emit(event: string | symbol, ...args: any[]): boolean;
emit(event: "response", response: Response): boolean;
emit(event: "error", err: Error): boolean;
on(event: string, listener: (...args: any[]) => void): this;
on(event: "response", listener: (response: Response) => void): this;
on(event: "error", listener: (err: Error) => void): this;
once(event: string, listener: (...args: any[]) => void): this;
once(event: "response", listener: (response: Response) => void): this;
once(event: "error", listener: (err: Error) => void): this;
prependListener(event: string, listener: (...args: any[]) => void): this;
prependListener(event: "response", listener: (response: Response) => void): this;
prependListener(event: "error", listener: (err: Error) => void): this;
prependOnceListener(event: string, listener: (...args: any[]) => void): this;
prependOnceListener(event: "response", listener: (response: Response) => void): this;
prependOnceListener(event: "error", listener: (err: Error) => void): this;
}
export interface RedirectScheme<Options, Request extends WrappableRequest, Response> {
/**
* This function has two overloads:
* ```typescript
* request(options: string | URL | Options, callback)
* request(url: string | URL, options: Options, callback)
* ```
*/
request(
url: string | URL | (Options & FollowOptions<Options>),
options?: (Options & FollowOptions<Options>) | ((res: Response & FollowResponse) => void),
callback?: (res: Response & FollowResponse) => void,
): RedirectableRequest<Request, Response>;
/**
* This function has two overloads:
* ```typescript
* get(options: string | URL | Options, callback)
* get(url: string | URL, options: Options, callback)
* ```
*/
get(
url: string | URL | (Options & FollowOptions<Options>),
options?: (Options & FollowOptions<Options>) | ((res: Response & FollowResponse) => void),
callback?: (res: Response & FollowResponse) => void,
): RedirectableRequest<Request, Response>;
}
export type Override<T, U> = Pick<T, Exclude<keyof T, keyof U>> & U;
export interface FollowOptions<Options> {
followRedirects?: boolean | undefined;
maxRedirects?: number | undefined;
maxBodyLength?: number | undefined;
beforeRedirect?:
| ((options: Options & FollowOptions<Options>, responseDetails: ResponseDetails) => void)
| undefined;
agents?:
| {
http?: coreHttp.Agent | undefined;
https?: coreHttps.Agent | undefined;
}
| undefined;
trackRedirects?: boolean | undefined;
}
export interface FollowResponse {
responseUrl: string;
redirects: Redirect[];
}
export interface Redirect {
url: string;
headers: coreHttp.IncomingHttpHeaders;
statusCode: number;
}
export interface ResponseDetails {
headers: coreHttp.IncomingHttpHeaders;
}
export const http: Override<
typeof coreHttp,
RedirectScheme<coreHttp.RequestOptions, coreHttp.ClientRequest, coreHttp.IncomingMessage>
>;
export const https: Override<
typeof coreHttps,
RedirectScheme<coreHttps.RequestOptions, coreHttp.ClientRequest, coreHttp.IncomingMessage>
>;
export type WrappedScheme<T extends Scheme<any, any, any>> = Override<
T,
RedirectScheme<
T extends Scheme<infer Options, any, any> ? Options : never,
T extends Scheme<any, infer Request, any> ? Request : never,
T extends Scheme<any, any, infer Response> ? Response : never
>
>;
export function wrap<T extends { [key: string]: Scheme<any, any, any> }>(
protocols: T,
): {
[K in keyof T]: WrappedScheme<T[K]>;
};

37
node_modules/@types/follow-redirects/package.json generated vendored Normal file
View file

@ -0,0 +1,37 @@
{
"name": "@types/follow-redirects",
"version": "1.14.4",
"description": "TypeScript definitions for follow-redirects",
"homepage": "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/follow-redirects",
"license": "MIT",
"contributors": [
{
"name": "Emily Klassen",
"githubUsername": "forivall",
"url": "https://github.com/forivall"
},
{
"name": "Claas Ahlrichs",
"githubUsername": "claasahl",
"url": "https://github.com/claasahl"
},
{
"name": "Piotr Błażejewicz",
"githubUsername": "peterblazejewicz",
"url": "https://github.com/peterblazejewicz"
}
],
"main": "",
"types": "index.d.ts",
"repository": {
"type": "git",
"url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git",
"directory": "types/follow-redirects"
},
"scripts": {},
"dependencies": {
"@types/node": "*"
},
"typesPublisherContentHash": "aae4331775f0d12b6291656304a64b4fc56d23a148d8ded8decf38cc252a2a2e",
"typeScriptVersion": "4.5"
}

18
node_modules/follow-redirects/LICENSE generated vendored Normal file
View file

@ -0,0 +1,18 @@
Copyright 2014–present Olivier Lalonde <olalonde@gmail.com>, James Talmage <james@talmage.io>, Ruben Verborgh
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

155
node_modules/follow-redirects/README.md generated vendored Normal file
View file

@ -0,0 +1,155 @@
## Follow Redirects
Drop-in replacement for Node's `http` and `https` modules that automatically follows redirects.
[![npm version](https://img.shields.io/npm/v/follow-redirects.svg)](https://www.npmjs.com/package/follow-redirects)
[![Build Status](https://github.com/follow-redirects/follow-redirects/workflows/CI/badge.svg)](https://github.com/follow-redirects/follow-redirects/actions)
[![Coverage Status](https://coveralls.io/repos/follow-redirects/follow-redirects/badge.svg?branch=master)](https://coveralls.io/r/follow-redirects/follow-redirects?branch=master)
[![npm downloads](https://img.shields.io/npm/dm/follow-redirects.svg)](https://www.npmjs.com/package/follow-redirects)
[![Sponsor on GitHub](https://img.shields.io/static/v1?label=Sponsor&message=%F0%9F%92%96&logo=GitHub)](https://github.com/sponsors/RubenVerborgh)
`follow-redirects` provides [request](https://nodejs.org/api/http.html#http_http_request_options_callback) and [get](https://nodejs.org/api/http.html#http_http_get_options_callback)
methods that behave identically to those found on the native [http](https://nodejs.org/api/http.html#http_http_request_options_callback) and [https](https://nodejs.org/api/https.html#https_https_request_options_callback)
modules, with the exception that they will seamlessly follow redirects.
```javascript
const { http, https } = require('follow-redirects');
http.get('http://bit.ly/900913', response => {
response.on('data', chunk => {
console.log(chunk);
});
}).on('error', err => {
console.error(err);
});
```
You can inspect the final redirected URL through the `responseUrl` property on the `response`.
If no redirection happened, `responseUrl` is the original request URL.
```javascript
const request = https.request({
host: 'bitly.com',
path: '/UHfDGO',
}, response => {
console.log(response.responseUrl);
// 'http://duckduckgo.com/robots.txt'
});
request.end();
```
## Options
### Global options
Global options are set directly on the `follow-redirects` module:
```javascript
const followRedirects = require('follow-redirects');
followRedirects.maxRedirects = 10;
followRedirects.maxBodyLength = 20 * 1024 * 1024; // 20 MB
```
The following global options are supported:
- `maxRedirects` (default: `21`) – sets the maximum number of allowed redirects; if exceeded, an error will be emitted.
- `maxBodyLength` (default: 10MB) – sets the maximum size of the request body; if exceeded, an error will be emitted.
### Per-request options
Per-request options are set by passing an `options` object:
```javascript
const url = require('url');
const { http, https } = require('follow-redirects');
const options = url.parse('http://bit.ly/900913');
options.maxRedirects = 10;
options.beforeRedirect = (options, response, request) => {
// Use this to adjust the request options upon redirecting,
// to inspect the latest response headers,
// or to cancel the request by throwing an error
// response.headers = the redirect response headers
// response.statusCode = the redirect response code (eg. 301, 307, etc.)
// request.url = the requested URL that resulted in a redirect
// request.headers = the headers in the request that resulted in a redirect
// request.method = the method of the request that resulted in a redirect
if (options.hostname === "example.com") {
options.auth = "user:password";
}
};
http.request(options);
```
In addition to the [standard HTTP](https://nodejs.org/api/http.html#http_http_request_options_callback) and [HTTPS options](https://nodejs.org/api/https.html#https_https_request_options_callback),
the following per-request options are supported:
- `followRedirects` (default: `true`) – whether redirects should be followed.
- `maxRedirects` (default: `21`) – sets the maximum number of allowed redirects; if exceeded, an error will be emitted.
- `maxBodyLength` (default: 10MB) – sets the maximum size of the request body; if exceeded, an error will be emitted.
- `beforeRedirect` (default: `undefined`) – optionally change the request `options` on redirects, or abort the request by throwing an error.
- `agents` (default: `undefined`) – sets the `agent` option per protocol, since HTTP and HTTPS use different agents. Example value: `{ http: new http.Agent(), https: new https.Agent() }`
- `trackRedirects` (default: `false`) – whether to store the redirected response details into the `redirects` array on the response object.
### Advanced usage
By default, `follow-redirects` will use the Node.js default implementations
of [`http`](https://nodejs.org/api/http.html)
and [`https`](https://nodejs.org/api/https.html).
To enable features such as caching and/or intermediate request tracking,
you might instead want to wrap `follow-redirects` around custom protocol implementations:
```javascript
const { http, https } = require('follow-redirects').wrap({
http: require('your-custom-http'),
https: require('your-custom-https'),
});
```
Such custom protocols only need an implementation of the `request` method.
## Browser Usage
Due to the way the browser works,
the `http` and `https` browser equivalents perform redirects by default.
By requiring `follow-redirects` this way:
```javascript
const http = require('follow-redirects/http');
const https = require('follow-redirects/https');
```
you can easily tell webpack and friends to replace
`follow-redirect` by the built-in versions:
```json
{
"follow-redirects/http" : "http",
"follow-redirects/https" : "https"
}
```
## Contributing
Pull Requests are always welcome. Please [file an issue](https://github.com/follow-redirects/follow-redirects/issues)
detailing your proposal before you invest your valuable time. Additional features and bug fixes should be accompanied
by tests. You can run the test suite locally with a simple `npm test` command.
## Debug Logging
`follow-redirects` uses the excellent [debug](https://www.npmjs.com/package/debug) for logging. To turn on logging
set the environment variable `DEBUG=follow-redirects` for debug output from just this module. When running the test
suite it is sometimes advantageous to set `DEBUG=*` to see output from the express server as well.
## Authors
- [Ruben Verborgh](https://ruben.verborgh.org/)
- [Olivier Lalonde](mailto:olalonde@gmail.com)
- [James Talmage](mailto:james@talmage.io)
## License
[MIT License](https://github.com/follow-redirects/follow-redirects/blob/master/LICENSE)

15
node_modules/follow-redirects/debug.js generated vendored Normal file
View file

@ -0,0 +1,15 @@
var debug;
module.exports = function () {
if (!debug) {
try {
/* eslint global-require: off */
debug = require("debug")("follow-redirects");
}
catch (error) { /* */ }
if (typeof debug !== "function") {
debug = function () { /* */ };
}
}
debug.apply(null, arguments);
};

1
node_modules/follow-redirects/http.js generated vendored Normal file
View file

@ -0,0 +1 @@
module.exports = require("./").http;

1
node_modules/follow-redirects/https.js generated vendored Normal file
View file

@ -0,0 +1 @@
module.exports = require("./").https;

686
node_modules/follow-redirects/index.js generated vendored Normal file
View file

@ -0,0 +1,686 @@
var url = require("url");
var URL = url.URL;
var http = require("http");
var https = require("https");
var Writable = require("stream").Writable;
var assert = require("assert");
var debug = require("./debug");
// Preventive platform detection
// istanbul ignore next
(function detectUnsupportedEnvironment() {
var looksLikeNode = typeof process !== "undefined";
var looksLikeBrowser = typeof window !== "undefined" && typeof document !== "undefined";
var looksLikeV8 = isFunction(Error.captureStackTrace);
if (!looksLikeNode && (looksLikeBrowser || !looksLikeV8)) {
console.warn("The follow-redirects package should be excluded from browser builds.");
}
}());
// Whether to use the native URL object or the legacy url module
var useNativeURL = false;
try {
assert(new URL(""));
}
catch (error) {
useNativeURL = error.code === "ERR_INVALID_URL";
}
// URL fields to preserve in copy operations
var preservedUrlFields = [
"auth",
"host",
"hostname",
"href",
"path",
"pathname",
"port",
"protocol",
"query",
"search",
"hash",
];
// Create handlers that pass events from native requests
var events = ["abort", "aborted", "connect", "error", "socket", "timeout"];
var eventHandlers = Object.create(null);
events.forEach(function (event) {
eventHandlers[event] = function (arg1, arg2, arg3) {
this._redirectable.emit(event, arg1, arg2, arg3);
};
});
// Error types with codes
var InvalidUrlError = createErrorType(
"ERR_INVALID_URL",
"Invalid URL",
TypeError
);
var RedirectionError = createErrorType(
"ERR_FR_REDIRECTION_FAILURE",
"Redirected request failed"
);
var TooManyRedirectsError = createErrorType(
"ERR_FR_TOO_MANY_REDIRECTS",
"Maximum number of redirects exceeded",
RedirectionError
);
var MaxBodyLengthExceededError = createErrorType(
"ERR_FR_MAX_BODY_LENGTH_EXCEEDED",
"Request body larger than maxBodyLength limit"
);
var WriteAfterEndError = createErrorType(
"ERR_STREAM_WRITE_AFTER_END",
"write after end"
);
// istanbul ignore next
var destroy = Writable.prototype.destroy || noop;
// An HTTP(S) request that can be redirected
function RedirectableRequest(options, responseCallback) {
// Initialize the request
Writable.call(this);
this._sanitizeOptions(options);
this._options = options;
this._ended = false;
this._ending = false;
this._redirectCount = 0;
this._redirects = [];
this._requestBodyLength = 0;
this._requestBodyBuffers = [];
// Attach a callback if passed
if (responseCallback) {
this.on("response", responseCallback);
}
// React to responses of native requests
var self = this;
this._onNativeResponse = function (response) {
try {
self._processResponse(response);
}
catch (cause) {
self.emit("error", cause instanceof RedirectionError ?
cause : new RedirectionError({ cause: cause }));
}
};
// Perform the first request
this._performRequest();
}
RedirectableRequest.prototype = Object.create(Writable.prototype);
RedirectableRequest.prototype.abort = function () {
destroyRequest(this._currentRequest);
this._currentRequest.abort();
this.emit("abort");
};
RedirectableRequest.prototype.destroy = function (error) {
destroyRequest(this._currentRequest, error);
destroy.call(this, error);
return this;
};
// Writes buffered data to the current native request
RedirectableRequest.prototype.write = function (data, encoding, callback) {
// Writing is not allowed if end has been called
if (this._ending) {
throw new WriteAfterEndError();
}
// Validate input and shift parameters if necessary
if (!isString(data) && !isBuffer(data)) {
throw new TypeError("data should be a string, Buffer or Uint8Array");
}
if (isFunction(encoding)) {
callback = encoding;
encoding = null;
}
// Ignore empty buffers, since writing them doesn't invoke the callback
// https://github.com/nodejs/node/issues/22066
if (data.length === 0) {
if (callback) {
callback();
}
return;
}
// Only write when we don't exceed the maximum body length
if (this._requestBodyLength + data.length <= this._options.maxBodyLength) {
this._requestBodyLength += data.length;
this._requestBodyBuffers.push({ data: data, encoding: encoding });
this._currentRequest.write(data, encoding, callback);
}
// Error when we exceed the maximum body length
else {
this.emit("error", new MaxBodyLengthExceededError());
this.abort();
}
};
// Ends the current native request
RedirectableRequest.prototype.end = function (data, encoding, callback) {
// Shift parameters if necessary
if (isFunction(data)) {
callback = data;
data = encoding = null;
}
else if (isFunction(encoding)) {
callback = encoding;
encoding = null;
}
// Write data if needed and end
if (!data) {
this._ended = this._ending = true;
this._currentRequest.end(null, null, callback);
}
else {
var self = this;
var currentRequest = this._currentRequest;
this.write(data, encoding, function () {
self._ended = true;
currentRequest.end(null, null, callback);
});
this._ending = true;
}
};
// Sets a header value on the current native request
RedirectableRequest.prototype.setHeader = function (name, value) {
this._options.headers[name] = value;
this._currentRequest.setHeader(name, value);
};
// Clears a header value on the current native request
RedirectableRequest.prototype.removeHeader = function (name) {
delete this._options.headers[name];
this._currentRequest.removeHeader(name);
};
// Global timeout for all underlying requests
RedirectableRequest.prototype.setTimeout = function (msecs, callback) {
var self = this;
// Destroys the socket on timeout
function destroyOnTimeout(socket) {
socket.setTimeout(msecs);
socket.removeListener("timeout", socket.destroy);
socket.addListener("timeout", socket.destroy);
}
// Sets up a timer to trigger a timeout event
function startTimer(socket) {
if (self._timeout) {
clearTimeout(self._timeout);
}
self._timeout = setTimeout(function () {
self.emit("timeout");
clearTimer();
}, msecs);
destroyOnTimeout(socket);
}
// Stops a timeout from triggering
function clearTimer() {
// Clear the timeout
if (self._timeout) {
clearTimeout(self._timeout);
self._timeout = null;
}
// Clean up all attached listeners
self.removeListener("abort", clearTimer);
self.removeListener("error", clearTimer);
self.removeListener("response", clearTimer);
self.removeListener("close", clearTimer);
if (callback) {
self.removeListener("timeout", callback);
}
if (!self.socket) {
self._currentRequest.removeListener("socket", startTimer);
}
}
// Attach callback if passed
if (callback) {
this.on("timeout", callback);
}
// Start the timer if or when the socket is opened
if (this.socket) {
startTimer(this.socket);
}
else {
this._currentRequest.once("socket", startTimer);
}
// Clean up on events
this.on("socket", destroyOnTimeout);
this.on("abort", clearTimer);
this.on("error", clearTimer);
this.on("response", clearTimer);
this.on("close", clearTimer);
return this;
};
// Proxy all other public ClientRequest methods
[
"flushHeaders", "getHeader",
"setNoDelay", "setSocketKeepAlive",
].forEach(function (method) {
RedirectableRequest.prototype[method] = function (a, b) {
return this._currentRequest[method](a, b);
};
});
// Proxy all public ClientRequest properties
["aborted", "connection", "socket"].forEach(function (property) {
Object.defineProperty(RedirectableRequest.prototype, property, {
get: function () { return this._currentRequest[property]; },
});
});
RedirectableRequest.prototype._sanitizeOptions = function (options) {
// Ensure headers are always present
if (!options.headers) {
options.headers = {};
}
// Since http.request treats host as an alias of hostname,
// but the url module interprets host as hostname plus port,
// eliminate the host property to avoid confusion.
if (options.host) {
// Use hostname if set, because it has precedence
if (!options.hostname) {
options.hostname = options.host;
}
delete options.host;
}
// Complete the URL object when necessary
if (!options.pathname && options.path) {
var searchPos = options.path.indexOf("?");
if (searchPos < 0) {
options.pathname = options.path;
}
else {
options.pathname = options.path.substring(0, searchPos);
options.search = options.path.substring(searchPos);
}
}
};
// Executes the next native request (initial or redirect)
RedirectableRequest.prototype._performRequest = function () {
// Load the native protocol
var protocol = this._options.protocol;
var nativeProtocol = this._options.nativeProtocols[protocol];
if (!nativeProtocol) {
throw new TypeError("Unsupported protocol " + protocol);
}
// If specified, use the agent corresponding to the protocol
// (HTTP and HTTPS use different types of agents)
if (this._options.agents) {
var scheme = protocol.slice(0, -1);
this._options.agent = this._options.agents[scheme];
}
// Create the native request and set up its event handlers
var request = this._currentRequest =
nativeProtocol.request(this._options, this._onNativeResponse);
request._redirectable = this;
for (var event of events) {
request.on(event, eventHandlers[event]);
}
// RFC7230§5.3.1: When making a request directly to an origin server, […]
// a client MUST send only the absolute path […] as the request-target.
this._currentUrl = /^\//.test(this._options.path) ?
url.format(this._options) :
// When making a request to a proxy, […]
// a client MUST send the target URI in absolute-form […].
this._options.path;
// End a redirected request
// (The first request must be ended explicitly with RedirectableRequest#end)
if (this._isRedirect) {
// Write the request entity and end
var i = 0;
var self = this;
var buffers = this._requestBodyBuffers;
(function writeNext(error) {
// Only write if this request has not been redirected yet
// istanbul ignore else
if (request === self._currentRequest) {
// Report any write errors
// istanbul ignore if
if (error) {
self.emit("error", error);
}
// Write the next buffer if there are still left
else if (i < buffers.length) {
var buffer = buffers[i++];
// istanbul ignore else
if (!request.finished) {
request.write(buffer.data, buffer.encoding, writeNext);
}
}
// End the request if `end` has been called on us
else if (self._ended) {
request.end();
}
}
}());
}
};
// Processes a response from the current native request
RedirectableRequest.prototype._processResponse = function (response) {
// Store the redirected response
var statusCode = response.statusCode;
if (this._options.trackRedirects) {
this._redirects.push({
url: this._currentUrl,
headers: response.headers,
statusCode: statusCode,
});
}
// RFC7231§6.4: The 3xx (Redirection) class of status code indicates
// that further action needs to be taken by the user agent in order to
// fulfill the request. If a Location header field is provided,
// the user agent MAY automatically redirect its request to the URI
// referenced by the Location field value,
// even if the specific status code is not understood.
// If the response is not a redirect; return it as-is
var location = response.headers.location;
if (!location || this._options.followRedirects === false ||
statusCode < 300 || statusCode >= 400) {
response.responseUrl = this._currentUrl;
response.redirects = this._redirects;
this.emit("response", response);
// Clean up
this._requestBodyBuffers = [];
return;
}
// The response is a redirect, so abort the current request
destroyRequest(this._currentRequest);
// Discard the remainder of the response to avoid waiting for data
response.destroy();
// RFC7231§6.4: A client SHOULD detect and intervene
// in cyclical redirections (i.e., "infinite" redirection loops).
if (++this._redirectCount > this._options.maxRedirects) {
throw new TooManyRedirectsError();
}
// Store the request headers if applicable
var requestHeaders;
var beforeRedirect = this._options.beforeRedirect;
if (beforeRedirect) {
requestHeaders = Object.assign({
// The Host header was set by nativeProtocol.request
Host: response.req.getHeader("host"),
}, this._options.headers);
}
// RFC7231§6.4: Automatic redirection needs to done with
// care for methods not known to be safe, […]
// RFC7231§6.4.2–3: For historical reasons, a user agent MAY change
// the request method from POST to GET for the subsequent request.
var method = this._options.method;
if ((statusCode === 301 || statusCode === 302) && this._options.method === "POST" ||
// RFC7231§6.4.4: The 303 (See Other) status code indicates that
// the server is redirecting the user agent to a different resource […]
// A user agent can perform a retrieval request targeting that URI
// (a GET or HEAD request if using HTTP) […]
(statusCode === 303) && !/^(?:GET|HEAD)$/.test(this._options.method)) {
this._options.method = "GET";
// Drop a possible entity and headers related to it
this._requestBodyBuffers = [];
removeMatchingHeaders(/^content-/i, this._options.headers);
}
// Drop the Host header, as the redirect might lead to a different host
var currentHostHeader = removeMatchingHeaders(/^host$/i, this._options.headers);
// If the redirect is relative, carry over the host of the last request
var currentUrlParts = parseUrl(this._currentUrl);
var currentHost = currentHostHeader || currentUrlParts.host;
var currentUrl = /^\w+:/.test(location) ? this._currentUrl :
url.format(Object.assign(currentUrlParts, { host: currentHost }));
// Create the redirected request
var redirectUrl = resolveUrl(location, currentUrl);
debug("redirecting to", redirectUrl.href);
this._isRedirect = true;
spreadUrlObject(redirectUrl, this._options);
// Drop confidential headers when redirecting to a less secure protocol
// or to a different domain that is not a superdomain
if (redirectUrl.protocol !== currentUrlParts.protocol &&
redirectUrl.protocol !== "https:" ||
redirectUrl.host !== currentHost &&
!isSubdomain(redirectUrl.host, currentHost)) {
removeMatchingHeaders(/^(?:(?:proxy-)?authorization|cookie)$/i, this._options.headers);
}
// Evaluate the beforeRedirect callback
if (isFunction(beforeRedirect)) {
var responseDetails = {
headers: response.headers,
statusCode: statusCode,
};
var requestDetails = {
url: currentUrl,
method: method,
headers: requestHeaders,
};
beforeRedirect(this._options, responseDetails, requestDetails);
this._sanitizeOptions(this._options);
}
// Perform the redirected request
this._performRequest();
};
// Wraps the key/value object of protocols with redirect functionality
function wrap(protocols) {
// Default settings
var exports = {
maxRedirects: 21,
maxBodyLength: 10 * 1024 * 1024,
};
// Wrap each protocol
var nativeProtocols = {};
Object.keys(protocols).forEach(function (scheme) {
var protocol = scheme + ":";
var nativeProtocol = nativeProtocols[protocol] = protocols[scheme];
var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol);
// Executes a request, following redirects
function request(input, options, callback) {
// Parse parameters, ensuring that input is an object
if (isURL(input)) {
input = spreadUrlObject(input);
}
else if (isString(input)) {
input = spreadUrlObject(parseUrl(input));
}
else {
callback = options;
options = validateUrl(input);
input = { protocol: protocol };
}
if (isFunction(options)) {
callback = options;
options = null;
}
// Set defaults
options = Object.assign({
maxRedirects: exports.maxRedirects,
maxBodyLength: exports.maxBodyLength,
}, input, options);
options.nativeProtocols = nativeProtocols;
if (!isString(options.host) && !isString(options.hostname)) {
options.hostname = "::1";
}
assert.equal(options.protocol, protocol, "protocol mismatch");
debug("options", options);
return new RedirectableRequest(options, callback);
}
// Executes a GET request, following redirects
function get(input, options, callback) {
var wrappedRequest = wrappedProtocol.request(input, options, callback);
wrappedRequest.end();
return wrappedRequest;
}
// Expose the properties on the wrapped protocol
Object.defineProperties(wrappedProtocol, {
request: { value: request, configurable: true, enumerable: true, writable: true },
get: { value: get, configurable: true, enumerable: true, writable: true },
});
});
return exports;
}
function noop() { /* empty */ }
function parseUrl(input) {
var parsed;
// istanbul ignore else
if (useNativeURL) {
parsed = new URL(input);
}
else {
// Ensure the URL is valid and absolute
parsed = validateUrl(url.parse(input));
if (!isString(parsed.protocol)) {
throw new InvalidUrlError({ input });
}
}
return parsed;
}
function resolveUrl(relative, base) {
// istanbul ignore next
return useNativeURL ? new URL(relative, base) : parseUrl(url.resolve(base, relative));
}
function validateUrl(input) {
if (/^\[/.test(input.hostname) && !/^\[[:0-9a-f]+\]$/i.test(input.hostname)) {
throw new InvalidUrlError({ input: input.href || input });
}
if (/^\[/.test(input.host) && !/^\[[:0-9a-f]+\](:\d+)?$/i.test(input.host)) {
throw new InvalidUrlError({ input: input.href || input });
}
return input;
}
function spreadUrlObject(urlObject, target) {
var spread = target || {};
for (var key of preservedUrlFields) {
spread[key] = urlObject[key];
}
// Fix IPv6 hostname
if (spread.hostname.startsWith("[")) {
spread.hostname = spread.hostname.slice(1, -1);
}
// Ensure port is a number
if (spread.port !== "") {
spread.port = Number(spread.port);
}
// Concatenate path
spread.path = spread.search ? spread.pathname + spread.search : spread.pathname;
return spread;
}
function removeMatchingHeaders(regex, headers) {
var lastValue;
for (var header in headers) {
if (regex.test(header)) {
lastValue = headers[header];
delete headers[header];
}
}
return (lastValue === null || typeof lastValue === "undefined") ?
undefined : String(lastValue).trim();
}
function createErrorType(code, message, baseClass) {
// Create constructor
function CustomError(properties) {
// istanbul ignore else
if (isFunction(Error.captureStackTrace)) {
Error.captureStackTrace(this, this.constructor);
}
Object.assign(this, properties || {});
this.code = code;
this.message = this.cause ? message + ": " + this.cause.message : message;
}
// Attach constructor and set default properties
CustomError.prototype = new (baseClass || Error)();
Object.defineProperties(CustomError.prototype, {
constructor: {
value: CustomError,
enumerable: false,
},
name: {
value: "Error [" + code + "]",
enumerable: false,
},
});
return CustomError;
}
function destroyRequest(request, error) {
for (var event of events) {
request.removeListener(event, eventHandlers[event]);
}
request.on("error", noop);
request.destroy(error);
}
function isSubdomain(subdomain, domain) {
assert(isString(subdomain) && isString(domain));
var dot = subdomain.length - domain.length - 1;
return dot > 0 && subdomain[dot] === "." && subdomain.endsWith(domain);
}
function isString(value) {
return typeof value === "string" || value instanceof String;
}
function isFunction(value) {
return typeof value === "function";
}
function isBuffer(value) {
return typeof value === "object" && ("length" in value);
}
function isURL(value) {
return URL && value instanceof URL;
}
// Exports
module.exports = wrap({ http: http, https: https });
module.exports.wrap = wrap;

58
node_modules/follow-redirects/package.json generated vendored Normal file
View file

@ -0,0 +1,58 @@
{
"name": "follow-redirects",
"version": "1.15.9",
"description": "HTTP and HTTPS modules that follow redirects.",
"license": "MIT",
"main": "index.js",
"files": [
"*.js"
],
"engines": {
"node": ">=4.0"
},
"scripts": {
"lint": "eslint *.js test",
"test": "nyc mocha"
},
"repository": {
"type": "git",
"url": "git+ssh://git@github.com/follow-redirects/follow-redirects.git"
},
"homepage": "https://github.com/follow-redirects/follow-redirects",
"bugs": {
"url": "https://github.com/follow-redirects/follow-redirects/issues"
},
"keywords": [
"http",
"https",
"url",
"redirect",
"client",
"location",
"utility"
],
"author": "Ruben Verborgh <ruben@verborgh.org> (https://ruben.verborgh.org/)",
"contributors": [
"Olivier Lalonde <olalonde@gmail.com> (http://www.syskall.com)",
"James Talmage <james@talmage.io>"
],
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"peerDependenciesMeta": {
"debug": {
"optional": true
}
},
"devDependencies": {
"concat-stream": "^2.0.0",
"eslint": "^5.16.0",
"express": "^4.16.4",
"lolex": "^3.1.0",
"mocha": "^6.0.2",
"nyc": "^14.1.1"
}
}

32
package-lock.json generated
View file

@ -29,6 +29,7 @@
"del": "^6.1.1",
"fast-deep-equal": "^3.1.3",
"file-url": "^3.0.0",
"follow-redirects": "^1.15.9",
"fs": "0.0.1-security",
"get-folder-size": "^2.0.1",
"js-yaml": "^4.1.0",
@ -48,6 +49,7 @@
"@microsoft/eslint-formatter-sarif": "^3.1.0",
"@types/adm-zip": "^0.5.5",
"@types/console-log-level": "^1.4.5",
"@types/follow-redirects": "^1.14.4",
"@types/get-folder-size": "^2.0.0",
"@types/js-yaml": "^4.0.9",
"@types/node": "20.9.0",
@ -1301,6 +1303,16 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/follow-redirects": {
"version": "1.14.4",
"resolved": "https://registry.npmjs.org/@types/follow-redirects/-/follow-redirects-1.14.4.tgz",
"integrity": "sha512-GWXfsD0Jc1RWiFmMuMFCpXMzi9L7oPDVwxUnZdg89kDNnqsRfUKXEtUYtA98A6lig1WXH/CYY/fvPW9HuN5fTA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/get-folder-size": {
"version": "2.0.0",
"dev": true,
@ -4061,6 +4073,26 @@
"dev": true,
"license": "ISC"
},
"node_modules/follow-redirects": {
"version": "1.15.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz",
"integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"license": "MIT",
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/for-each": {
"version": "0.3.3",
"dev": true,

2
package.json
View file

@ -42,6 +42,7 @@
"del": "^6.1.1",
"fast-deep-equal": "^3.1.3",
"file-url": "^3.0.0",
"follow-redirects": "^1.15.9",
"fs": "0.0.1-security",
"get-folder-size": "^2.0.1",
"js-yaml": "^4.1.0",
@ -64,6 +65,7 @@
"@microsoft/eslint-formatter-sarif": "^3.1.0",
"@types/adm-zip": "^0.5.5",
"@types/console-log-level": "^1.4.5",
"@types/follow-redirects": "^1.14.4",
"@types/get-folder-size": "^2.0.0",
"@types/js-yaml": "^4.0.9",
"@types/node": "20.9.0",

66
pr-checks/checks/zstd-bundle-streaming.yml Normal file
View file

@ -0,0 +1,66 @@
name: "Zstandard bundle (streaming)"
description: "Stream the download and extraction of a Zstandard-compressed CodeQL Bundle"
versions:
- linked
operatingSystems:
- macos
- ubuntu
env:
CODEQL_ACTION_ZSTD_BUNDLE: true
CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION: true
steps:
- name: Remove CodeQL from toolcache
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const codeqlPath = path.join(process.env['RUNNER_TOOL_CACHE'], 'CodeQL');
fs.rmdirSync(codeqlPath, { recursive: true });
- id: init
uses: ./../action/init
with:
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: zstd-bundle.sarif
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check diagnostic with expected tools URL appears in SARIF
uses: actions/github-script@v7
env:
SARIF_PATH: ${{ runner.temp }}/results/javascript.sarif
with:
script: |
const fs = require('fs');
const sarif = JSON.parse(fs.readFileSync(process.env['SARIF_PATH'], 'utf8'));
const run = sarif.runs[0];
const toolExecutionNotifications = run.invocations[0].toolExecutionNotifications;
const downloadTelemetryNotifications = toolExecutionNotifications.filter(n =>
n.descriptor.id === 'codeql-action/bundle-download-telemetry'
);
if (downloadTelemetryNotifications.length !== 1) {
core.setFailed(
'Expected exactly one reporting descriptor in the ' +
`'runs[].invocations[].toolExecutionNotifications[]' SARIF property, but found ` +
`${downloadTelemetryNotifications.length}. All notification reporting descriptors: ` +
`${JSON.stringify(toolExecutionNotifications)}.`
);
}
const toolsUrl = downloadTelemetryNotifications[0].properties.attributes.toolsUrl;
console.log(`Found tools URL: ${toolsUrl}`);
if (!toolsUrl.endsWith('.tar.zst')) {
core.setFailed(
`Expected the tools URL to be a .tar.zst file, but found ${toolsUrl}.`
);
}

6
src/actions-util.ts
View file

@ -483,7 +483,7 @@ export class CommandInvocationError extends Error {
constructor(
public cmd: string,
public args: string[],
public exitCode: number,
public exitCode: number | undefined,
public stderr: string,
public stdout: string,
) {
@ -527,7 +527,9 @@ export async function runTool(
): Promise<string> {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
if (!opts.noStreamStdout) {
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
}
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {

2
src/cli-errors.ts
View file

@ -10,7 +10,7 @@ import { ConfigurationError } from "./util";
* An error from a CodeQL CLI invocation, with associated exit code, stderr, etc.
*/
export class CliError extends Error {
public readonly exitCode: number;
public readonly exitCode: number | undefined;
public readonly stderr: string;
constructor({ cmd, args, exitCode, stderr }: CommandInvocationError) {

49
src/codeql.test.ts
View file

@ -30,6 +30,7 @@ import {
makeVersionInfo,
createTestConfig,
} from "./testing-utils";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import * as util from "./util";
import { initializeEnvironment } from "./util";
@ -138,9 +139,6 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
}
t.is(toolcache.findAllVersions("CodeQL").length, 2);
@ -170,9 +168,9 @@ test("caches semantically versioned bundles using their semantic version number"
t.assert(toolcache.find("CodeQL", `2.14.0`));
t.is(result.toolsVersion, `2.14.0`);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
@ -203,9 +201,9 @@ test("downloads an explicitly requested bundle even if a different version is ca
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
@ -288,7 +286,9 @@ for (const toolcacheVersion of [
);
t.is(result.toolsVersion, SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
t.is(result.toolsSource, ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
});
},
);
@ -320,7 +320,9 @@ test(`uses a cached bundle when no tools input is given on GHES`, async (t) => {
);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
t.is(result.toolsSource, ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
@ -356,9 +358,9 @@ test(`downloads bundle if only an unpinned version is cached on GHES`, async (t)
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
@ -391,9 +393,9 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
@ -430,9 +432,9 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, ToolsSource.Download);
t.true(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
@ -442,6 +444,17 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
});
});
function assertDurationsInteger(
t: ExecutionContext<unknown>,
statusReport: ToolsDownloadStatusReport,
) {
t.assert(Number.isInteger(statusReport?.combinedDurationMs));
if (statusReport.downloadDurationMs !== undefined) {
t.assert(Number.isInteger(statusReport?.downloadDurationMs));
t.assert(Number.isInteger(statusReport?.extractionDurationMs));
}
}
test("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);

7
src/codeql.ts
View file

@ -28,6 +28,7 @@ import { Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
@ -356,7 +357,7 @@ export async function setupCodeQL(
checkVersion: boolean,
): Promise<{
codeql: CodeQL;
toolsDownloadStatusReport?: setupCodeql.ToolsDownloadStatusReport;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: setupCodeql.ToolsSource;
toolsVersion: string;
zstdAvailability: ZstdAvailability;
@ -543,7 +544,9 @@ export async function getCodeQLForCmd(
async getVersion() {
let result = util.getCachedCodeQlVersion();
if (result === undefined) {
const output = await runCli(cmd, ["version", "--format=json"]);
const output = await runCli(cmd, ["version", "--format=json"], {
noStreamStdout: true,
});
try {
result = JSON.parse(output) as VersionInfo;
} catch {

6
src/feature-flags.ts
View file

@ -55,6 +55,7 @@ export enum Feature {
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
ZstdBundle = "zstd_bundle",
ZstdBundleStreamingExtraction = "zstd_bundle_streaming_extraction",
}
export const featureConfig: Record<
@ -146,6 +147,11 @@ export const featureConfig: Record<
// version check separately.
minimumVersion: undefined,
},
[Feature.ZstdBundleStreamingExtraction]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
minimumVersion: undefined,
},
};
/**

5
src/init-action.ts
View file

@ -34,7 +34,7 @@ import {
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ToolsSource } from "./setup-codeql";
import {
ActionName,
StatusReportBase,
@ -43,6 +43,7 @@ import {
sendStatusReport,
} from "./status-report";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { getTotalCacheSize } from "./trap-caching";
import {
@ -153,7 +154,7 @@ async function sendCompletedStatusReport(
const initToolsDownloadFields: InitToolsDownloadFields = {};
if (toolsDownloadStatusReport !== undefined) {
if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadStatusReport.downloadDurationMs;
}

3
src/init.ts
View file

@ -11,8 +11,9 @@ import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";

14
src/logging.ts
View file

@ -40,3 +40,17 @@ export function withGroup<T>(groupName: string, f: () => T): T {
core.endGroup();
}
}
/** Format a duration for use in logs. */
export function formatDuration(durationMs: number) {
if (durationMs < 1000) {
return `${durationMs}ms`;
}
if (durationMs < 60 * 1000) {
return `${(durationMs / 1000).toFixed(1)}s`;
}
const minutes = Math.floor(durationMs / (60 * 1000));
const seconds = Math.floor((durationMs % (60 * 1000)) / 1000);
return `${minutes}m${seconds}s`;
}

6
src/setup-codeql.test.ts
View file

@ -161,9 +161,11 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
streamExtraction: false,
toolsUrl: "toolsUrl",
},
toolsVersion: LINKED_CLI_VERSION.cliVersion,
@ -210,9 +212,11 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
streamExtraction: false,
toolsUrl: bundleUrl,
},
toolsVersion: expectedVersion,
@ -235,7 +239,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
t.is(result.toolsVersion, expectedVersion);
// Ensure message logging CodeQL CLI version was present in user logs.
const expected_message: string = `Using CodeQL CLI version 2.16.0 sourced from ${bundleUrl}.`;
const expected_message: string = `Using CodeQL CLI version 2.16.0 sourced from ${bundleUrl} .`;
t.assert(
loggedMessages.some(
(msg) =>

98
src/setup-codeql.ts
View file

@ -6,13 +6,9 @@ import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { CommandInvocationError, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
import * as defaults from "./defaults.json";
import {
CODEQL_VERSION_ZSTD_BUNDLE,
@ -20,8 +16,12 @@ import {
Feature,
FeatureEnablement,
} from "./feature-flags";
import { Logger } from "./logging";
import { formatDuration, Logger } from "./logging";
import * as tar from "./tar";
import {
downloadAndExtract,
ToolsDownloadStatusReport,
} from "./tools-download";
import * as util from "./util";
import { cleanUpGlob, isGoodVersion } from "./util";
@ -447,9 +447,9 @@ export async function getCodeQLSource(
}
if (cliVersion) {
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url}.`);
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url} .`);
} else {
logger.info(`Using CodeQL CLI sourced from ${url}.`);
logger.info(`Using CodeQL CLI sourced from ${url} .`);
}
return {
bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger),
@ -481,14 +481,6 @@ export async function tryGetFallbackToolcacheVersion(
return fallbackVersion;
}
export interface ToolsDownloadStatusReport {
compressionMethod: tar.CompressionMethod;
downloadDurationMs: number;
extractionDurationMs: number;
toolsUrl: string;
zstdFailureReason?: string;
}
// Exported using `export const` for testing purposes. Specifically, we want to
// be able to stub this function and have other functions in this file use that stub.
export const downloadCodeQL = async function (
@ -498,6 +490,7 @@ export const downloadCodeQL = async function (
apiDetails: api.GitHubApiDetails,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
codeqlFolder: string;
@ -525,49 +518,16 @@ export const downloadCodeQL = async function (
} else {
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
const dest = path.join(tempDir, uuidV4());
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers,
);
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
const { extractedBundlePath, statusReport } = await downloadAndExtract(
codeqlURL,
dest,
authorization,
finalHeaders,
{ "User-Agent": "CodeQL Action", ...headers },
tarVersion,
tempDir,
features,
logger,
);
const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
logger.debug(
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.debug(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
const bundleVersion =
maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
@ -579,12 +539,7 @@ export const downloadCodeQL = async function (
);
return {
codeqlFolder: extractedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? "unknown",
};
}
@ -595,12 +550,19 @@ export const downloadCodeQL = async function (
bundleVersion,
logger,
);
const toolcacheStart = performance.now();
const toolcachedBundlePath = await toolcache.cacheDir(
extractedBundlePath,
"CodeQL",
toolcacheVersion,
);
logger.info(
`Added CodeQL bundle to the tool cache (${formatDuration(
performance.now() - toolcacheStart,
)}).`,
);
// Defensive check: we expect `cacheDir` to copy the bundle to a new location.
if (toolcachedBundlePath !== extractedBundlePath) {
await cleanUpGlob(
@ -612,12 +574,7 @@ export const downloadCodeQL = async function (
return {
codeqlFolder: toolcachedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? toolcacheVersion,
};
};
@ -789,6 +746,7 @@ async function setupCodeQLBundleWithCompressionMethod(
apiDetails,
zstdAvailability.version,
tempDir,
features,
logger,
);
toolsVersion = result.toolsVersion;
@ -809,14 +767,6 @@ async function setupCodeQLBundleWithCompressionMethod(
};
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
)
? url
: "sanitized-value";
}
async function useZstdBundle(
cliVersion: string,
features: FeatureEnablement,

72
src/tar.ts
View file

@ -1,12 +1,14 @@
import { spawn } from "child_process";
import * as fs from "fs";
import path from "path";
import * as stream from "stream";
import { ToolRunner } from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
import { safeWhich } from "@chrisgavin/safe-which";
import { v4 as uuidV4 } from "uuid";
import { getTemporaryDirectory, runTool } from "./actions-util";
import { CommandInvocationError, getTemporaryDirectory } from "./actions-util";
import { Logger } from "./logging";
import { assertNever, cleanUpGlob } from "./util";
@ -123,7 +125,11 @@ export async function extract(
"Could not determine tar version, which is required to extract a Zstandard archive.",
);
}
return await extractTarZst(tarPath, tarVersion, logger);
return await extractTarZst(
fs.createReadStream(tarPath),
tarVersion,
logger,
);
}
}
@ -135,31 +141,15 @@ export async function extract(
* @returns path to the destination directory
*/
export async function extractTarZst(
file: string,
tarStream: stream.Readable,
tarVersion: TarVersion,
logger: Logger,
): Promise<string> {
if (!file) {
throw new Error("parameter 'file' is required");
}
// Create dest
const dest = await createExtractFolder();
try {
// Initialize args
const args = ["-x", "-v"];
let destArg = dest;
let fileArg = file;
if (process.platform === "win32" && tarVersion.type === "gnu") {
args.push("--force-local");
destArg = dest.replace(/\\/g, "/");
// Technically only the dest needs to have `/` but for aesthetic consistency
// convert slashes in the file arg too.
fileArg = file.replace(/\\/g, "/");
}
const args = ["-x", "--zstd"];
if (tarVersion.type === "gnu") {
// Suppress warnings when using GNU tar to extract archives created by BSD tar
@ -167,14 +157,48 @@ export async function extractTarZst(
args.push("--overwrite");
}
args.push("-C", destArg, "-f", fileArg);
await runTool(`tar`, args);
args.push("-f", "-", "-C", dest);
process.stdout.write(`[command]tar ${args.join(" ")}\n`);
const tarProcess = spawn("tar", args, { stdio: "pipe" });
let stdout = "";
tarProcess.stdout?.on("data", (data: Buffer) => {
stdout += data.toString();
process.stdout.write(data);
});
let stderr = "";
tarProcess.stderr?.on("data", (data: Buffer) => {
stderr += data.toString();
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
});
tarStream.pipe(tarProcess.stdin);
await new Promise<void>((resolve, reject) => {
tarProcess.on("exit", (code) => {
if (code !== 0) {
reject(
new CommandInvocationError(
"tar",
args,
code ?? undefined,
stdout,
stderr,
),
);
}
resolve();
});
});
return dest;
} catch (e) {
await cleanUpGlob(dest, "extraction destination directory", logger);
throw e;
}
return dest;
}
async function createExtractFolder(): Promise<string> {

203
src/tools-download.ts Normal file
View file

@ -0,0 +1,203 @@
import { IncomingMessage, OutgoingHttpHeaders } from "http";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { https } from "follow-redirects";
import { v4 as uuidV4 } from "uuid";
import { Feature, FeatureEnablement } from "./feature-flags";
import { formatDuration, Logger } from "./logging";
import * as tar from "./tar";
import { cleanUpGlob } from "./util";
/**
* Timing information for the download and extraction of the CodeQL tools when
* we fully download the bundle before extracting.
*/
type DownloadFirstToolsDownloadDurations = {
combinedDurationMs: number;
downloadDurationMs: number;
extractionDurationMs: number;
streamExtraction: false;
};
function makeDownloadFirstToolsDownloadDurations(
downloadDurationMs: number,
extractionDurationMs: number,
): DownloadFirstToolsDownloadDurations {
return {
combinedDurationMs: downloadDurationMs + extractionDurationMs,
downloadDurationMs,
extractionDurationMs,
streamExtraction: false,
};
}
/**
* Timing information for the download and extraction of the CodeQL tools when
* we stream the download and extraction of the bundle.
*/
type StreamedToolsDownloadDurations = {
combinedDurationMs: number;
downloadDurationMs: undefined;
extractionDurationMs: undefined;
streamExtraction: true;
};
function makeStreamedToolsDownloadDurations(
combinedDurationMs: number,
): StreamedToolsDownloadDurations {
return {
combinedDurationMs,
downloadDurationMs: undefined,
extractionDurationMs: undefined,
streamExtraction: true,
};
}
type ToolsDownloadDurations =
| DownloadFirstToolsDownloadDurations
| StreamedToolsDownloadDurations;
export type ToolsDownloadStatusReport = {
compressionMethod: tar.CompressionMethod;
toolsUrl: string;
zstdFailureReason?: string;
} & ToolsDownloadDurations;
export async function downloadAndExtract(
codeqlURL: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
extractedBundlePath: string;
statusReport: ToolsDownloadStatusReport;
}> {
logger.info(
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
if (
compressionMethod === "zstd" &&
(await features.getValue(Feature.ZstdBundleStreamingExtraction))
) {
logger.info(`Streaming the extraction of the CodeQL bundle.`);
const toolsInstallStart = performance.now();
const extractedBundlePath = await downloadAndExtractZstdWithStreaming(
codeqlURL,
authorization,
headers,
tarVersion!,
logger,
);
const combinedDurationMs = Math.round(
performance.now() - toolsInstallStart,
);
logger.info(
`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${formatDuration(
combinedDurationMs,
)}).`,
);
return {
extractedBundlePath,
statusReport: {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeStreamedToolsDownloadDurations(combinedDurationMs),
},
};
}
const dest = path.join(tempDir, uuidV4());
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
codeqlURL,
dest,
authorization,
headers,
);
const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
logger.info(
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${formatDuration(
downloadDurationMs,
)}).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.info("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.info(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${formatDuration(
extractionDurationMs,
)}).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
return {
extractedBundlePath,
statusReport: {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeDownloadFirstToolsDownloadDurations(
downloadDurationMs,
extractionDurationMs,
),
},
};
}
async function downloadAndExtractZstdWithStreaming(
codeqlURL: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion,
logger: Logger,
): Promise<string> {
headers = Object.assign(
{ "User-Agent": "CodeQL Action", authorization },
headers,
);
const response = await new Promise<IncomingMessage>((resolve) =>
https.get(codeqlURL, { headers }, (r) => resolve(r)),
);
if (response.statusCode !== 200) {
throw new Error(
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
);
}
return await tar.extractTarZst(response, tarVersion, logger);
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
)
? url
: "sanitized-value";
}