robojerk/codeql-action

Stream the download and extraction of zstd bundles

Browse source 
Behind a feature flag
This commit is contained in:
Henry Mercer 2024-10-10 18:23:06 +01:00
parent cd83b08c78
commit ddead5420c
22 changed files with 371 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

32
lib/codeql.test.js generated
View file

@ -101,7 +101,6 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
}
t.is(toolcache.findAllVersions("CodeQL").length, 2);
});
@ -118,7 +117,9 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", `2.14.0`));
t.is(result.toolsVersion, `2.14.0`);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
(0, ava_1.default)("downloads an explicitly requested bundle even if a different version is cached", async (t) => {
@ -136,7 +137,9 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [
@ -184,7 +187,9 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsVersion, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
});
});
}
@ -202,7 +207,9 @@ for (const toolcacheVersion of [
}, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
});
@ -224,7 +231,9 @@ for (const toolcacheVersion of [
}, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
@ -243,7 +252,9 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL("latest", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
@ -265,13 +276,22 @@ for (const toolcacheVersion of [
const result = await codeql.setupCodeQL("https://github.com/codeql-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
t.true(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
t.is(cachedVersions[0], "0.0.0-20230203");
t.false(releasesApiMock.isDone());
});
});
function assertDurationsInteger(t, statusReport) {
t.assert(Number.isInteger(statusReport?.combinedDurationMs));
if (statusReport.downloadDurationMs !== undefined) {
t.assert(Number.isInteger(statusReport?.downloadDurationMs));
t.assert(Number.isInteger(statusReport?.extractionDurationMs));
}
}
(0, ava_1.default)("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);
t.deepEqual(codeql.getExtraOptions({ foo: [42] }, ["foo"], []), ["42"]);

2
lib/codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/feature-flags.js generated
View file

@ -57,6 +57,7 @@ var Feature;
Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
Feature["ZstdBundle"] = "zstd_bundle";
Feature["ZstdBundleStreamingExtraction"] = "zstd_bundle_streaming_extraction";
})(Feature || (exports.Feature = Feature = {}));
exports.featureConfig = {
[Feature.ArtifactV4Upgrade]: {
@ -111,6 +112,11 @@ exports.featureConfig = {
// version check separately.
minimumVersion: undefined,
},
[Feature.ZstdBundleStreamingExtraction]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
minimumVersion: undefined,
},
};
exports.FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
/**

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/init-action.js generated
View file

@ -58,7 +58,7 @@ async function sendCompletedStatusReport(startedAt, config, configFile, toolsDow
workflow_languages: workflowLanguages || "",
};
const initToolsDownloadFields = {};
if (toolsDownloadStatusReport !== undefined) {
if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadStatusReport.downloadDurationMs;
}

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/init.js.map
View file

@ -1 +1 @@
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AA/MD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAI1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AAhND,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAK1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

49
lib/setup-codeql.js generated
View file

@ -36,19 +36,15 @@ exports.getCodeQLURLVersion = getCodeQLURLVersion;
exports.setupCodeQLBundle = setupCodeQLBundle;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const toolcache = __importStar(require("@actions/tool-cache"));
const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
const semver = __importStar(require("semver"));
const uuid_1 = require("uuid");
const actions_util_1 = require("./actions-util");
const api = __importStar(require("./api-client"));
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
const defaults = __importStar(require("./defaults.json"));
const feature_flags_1 = require("./feature-flags");
const tar = __importStar(require("./tar"));
const tools_download_1 = require("./tools-download");
const util = __importStar(require("./util"));
const util_1 = require("./util");
var ToolsSource;
@ -372,7 +368,7 @@ async function tryGetFallbackToolcacheVersion(cliVersion, tagName, logger) {
}
// Exported using `export const` for testing purposes. Specifically, we want to
// be able to stub this function and have other functions in this file use that stub.
const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, logger) {
const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, features, logger) {
const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers = {
@ -395,37 +391,14 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
const dest = path.join(tempDir, (0, uuid_1.v4)());
const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
const toolsDownloadStart = perf_hooks_1.performance.now();
const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
let extractedBundlePath;
let extractionDurationMs;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = perf_hooks_1.performance.now();
extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
}
finally {
await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
}
const { extractedBundlePath, statusReport } = await (0, tools_download_1.downloadAndExtract)(codeqlURL, authorization, { "User-Agent": "CodeQL Action", ...headers }, tarVersion, tempDir, features, logger);
const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`);
return {
codeqlFolder: extractedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? "unknown",
};
}
@ -438,12 +411,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
}
return {
codeqlFolder: toolcachedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? toolcacheVersion,
};
};
@ -529,7 +497,7 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
toolsSource = ToolsSource.Toolcache;
break;
case "download": {
const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, logger);
const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, features, logger);
toolsVersion = result.toolsVersion;
codeqlFolder = result.codeqlFolder;
toolsDownloadStatusReport = result.statusReport;
@ -547,11 +515,6 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
zstdAvailability,
};
}
function sanitizeUrlForStatusReport(url) {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
? url
: "sanitized-value";
}
async function useZstdBundle(cliVersion, features, tarSupportsZstd) {
return (
// In testing, gzip performs better than zstd on Windows.

2
lib/setup-codeql.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/setup-codeql.test.js generated
View file

@ -116,6 +116,7 @@ ava_1.default.beforeEach(() => {
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
@ -145,6 +146,7 @@ ava_1.default.beforeEach(() => {
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,

2
lib/setup-codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

95
lib/tools-download.js generated Normal file
View file

@ -0,0 +1,95 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadAndExtract = downloadAndExtract;
const http = __importStar(require("http"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const toolcache = __importStar(require("@actions/tool-cache"));
const uuid_1 = require("uuid");
const feature_flags_1 = require("./feature-flags");
const tar = __importStar(require("./tar"));
const util_1 = require("./util");
async function downloadAndExtract(codeqlURL, authorization, headers, tarVersion, tempDir, features, logger) {
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
if (compressionMethod === "zstd" &&
(await features.getValue(feature_flags_1.Feature.ZstdBundleStreamingExtraction))) {
const toolsInstallStart = perf_hooks_1.performance.now();
const extractedBundlePath = await downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger);
const combinedDurationMs = Math.round(perf_hooks_1.performance.now() - toolsInstallStart);
logger.info(`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${combinedDurationMs} ms).`);
return {
extractedBundlePath,
statusReport: {
combinedDurationMs,
compressionMethod,
downloadDurationMs: undefined,
extractionDurationMs: undefined,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
};
}
const dest = path.join(tempDir, (0, uuid_1.v4)());
const toolsDownloadStart = perf_hooks_1.performance.now();
const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, headers);
const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.info(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
let extractedBundlePath;
let extractionDurationMs;
try {
logger.info("Extracting CodeQL bundle.");
const extractionStart = perf_hooks_1.performance.now();
extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
logger.info(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
}
finally {
await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
}
return {
extractedBundlePath,
statusReport: {
compressionMethod,
combinedDurationMs: downloadDurationMs + extractionDurationMs,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
};
}
async function downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger) {
headers = Object.assign({ "User-Agent": "CodeQL Action", authorization }, headers);
const response = await new Promise((resolve) => http.get(codeqlURL, { headers }, (r) => resolve(r)));
if (response.statusCode !== 200) {
throw new Error(`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`);
}
return await tar.extractTarZst(response, tarVersion, logger);
}
function sanitizeUrlForStatusReport(url) {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
? url
: "sanitized-value";
}
//# sourceMappingURL=tools-download.js.map

1
lib/tools-download.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA2CA,gDA2FC;AAtID,2CAA6B;AAE7B,2CAA6B;AAC7B,2CAAyC;AAEzC,+DAAiD;AACjD,+BAAoC;AAEpC,mDAA6D;AAE7D,2CAA6B;AAC7B,iCAAqC;AAgC9B,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,OAAe,EACf,QAA2B,EAC3B,MAAc;IAKd,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,IACE,iBAAiB,KAAK,MAAM;QAC5B,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,uBAAO,CAAC,6BAA6B,CAAC,CAAC,EAChE,CAAC;QACD,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,mBAAmB,GAAG,MAAM,mCAAmC,CACnE,SAAS,EACT,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;QACF,MAAM,CAAC,IAAI,CACT,wDAAwD,mBAAmB,KAAK,kBAAkB,OAAO,CAC1G,CAAC;QAEF,OAAO;YACL,mBAAmB;YACnB,YAAY,EAAE;gBACZ,kBAAkB;gBAClB,iBAAiB;gBACjB,kBAAkB,EAAE,SAAS;gBAC7B,oBAAoB,EAAE,SAAS;gBAC/B,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;aAChD;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,kBAAkB,OAAO,CAC1F,CAAC;IAEF,IAAI,mBAA2B,CAAC;IAChC,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,mBAAmB,GAAG,MAAM,GAAG,CAAC,OAAO,CACrC,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,mBAAmB,KAAK,oBAAoB,OAAO,CAC5F,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,YAAY,EAAE;YACZ,iBAAiB;YACjB,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;YAC7D,kBAAkB;YAClB,oBAAoB;YACpB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;SAChD;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,EAChD,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAuB,CAAC,OAAO,EAAE,EAAE,CACnE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CACpD,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}

49
src/codeql.test.ts
View file

@ -30,6 +30,7 @@ import {
makeVersionInfo,
createTestConfig,
} from "./testing-utils";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import * as util from "./util";
import { initializeEnvironment } from "./util";
@ -138,9 +139,6 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
}
t.is(toolcache.findAllVersions("CodeQL").length, 2);
@ -170,9 +168,9 @@ test("caches semantically versioned bundles using their semantic version number"
t.assert(toolcache.find("CodeQL", `2.14.0`));
t.is(result.toolsVersion, `2.14.0`);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
@ -203,9 +201,9 @@ test("downloads an explicitly requested bundle even if a different version is ca
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
@ -288,7 +286,9 @@ for (const toolcacheVersion of [
);
t.is(result.toolsVersion, SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
t.is(result.toolsSource, ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
});
},
);
@ -320,7 +320,9 @@ test(`uses a cached bundle when no tools input is given on GHES`, async (t) => {
);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
t.is(result.toolsSource, ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
@ -356,9 +358,9 @@ test(`downloads bundle if only an unpinned version is cached on GHES`, async (t)
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
@ -391,9 +393,9 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
@ -430,9 +432,9 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, ToolsSource.Download);
t.true(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
@ -442,6 +444,17 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
});
});
function assertDurationsInteger(
t: ExecutionContext<unknown>,
statusReport: ToolsDownloadStatusReport,
) {
t.assert(Number.isInteger(statusReport?.combinedDurationMs));
if (statusReport.downloadDurationMs !== undefined) {
t.assert(Number.isInteger(statusReport?.downloadDurationMs));
t.assert(Number.isInteger(statusReport?.extractionDurationMs));
}
}
test("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);

3
src/codeql.ts
View file

@ -28,6 +28,7 @@ import { Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
@ -356,7 +357,7 @@ export async function setupCodeQL(
checkVersion: boolean,
): Promise<{
codeql: CodeQL;
toolsDownloadStatusReport?: setupCodeql.ToolsDownloadStatusReport;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: setupCodeql.ToolsSource;
toolsVersion: string;
zstdAvailability: ZstdAvailability;

6
src/feature-flags.ts
View file

@ -54,6 +54,7 @@ export enum Feature {
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
ZstdBundle = "zstd_bundle",
ZstdBundleStreamingExtraction = "zstd_bundle_streaming_extraction",
}
export const featureConfig: Record<
@ -139,6 +140,11 @@ export const featureConfig: Record<
// version check separately.
minimumVersion: undefined,
},
[Feature.ZstdBundleStreamingExtraction]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
minimumVersion: undefined,
},
};
/**

5
src/init-action.ts
View file

@ -34,7 +34,7 @@ import {
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ToolsSource } from "./setup-codeql";
import {
ActionName,
StatusReportBase,
@ -43,6 +43,7 @@ import {
sendStatusReport,
} from "./status-report";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { getTotalCacheSize } from "./trap-caching";
import {
@ -153,7 +154,7 @@ async function sendCompletedStatusReport(
const initToolsDownloadFields: InitToolsDownloadFields = {};
if (toolsDownloadStatusReport !== undefined) {
if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadStatusReport.downloadDurationMs;
}

3
src/init.ts
View file

@ -11,8 +11,9 @@ import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";

2
src/setup-codeql.test.ts
View file

@ -161,6 +161,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
@ -210,6 +211,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,

83
src/setup-codeql.ts
View file

@ -1,18 +1,13 @@
import * as fs from "fs";
import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { CommandInvocationError, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
import * as defaults from "./defaults.json";
import {
CODEQL_VERSION_ZSTD_BUNDLE,
@ -22,6 +17,10 @@ import {
} from "./feature-flags";
import { Logger } from "./logging";
import * as tar from "./tar";
import {
downloadAndExtract,
ToolsDownloadStatusReport,
} from "./tools-download";
import * as util from "./util";
import { cleanUpGlob, isGoodVersion } from "./util";
@ -481,14 +480,6 @@ export async function tryGetFallbackToolcacheVersion(
return fallbackVersion;
}
export interface ToolsDownloadStatusReport {
compressionMethod: tar.CompressionMethod;
downloadDurationMs: number;
extractionDurationMs: number;
toolsUrl: string;
zstdFailureReason?: string;
}
// Exported using `export const` for testing purposes. Specifically, we want to
// be able to stub this function and have other functions in this file use that stub.
export const downloadCodeQL = async function (
@ -498,6 +489,7 @@ export const downloadCodeQL = async function (
apiDetails: api.GitHubApiDetails,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
codeqlFolder: string;
@ -529,45 +521,15 @@ export const downloadCodeQL = async function (
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
const dest = path.join(tempDir, uuidV4());
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers,
);
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
const { extractedBundlePath, statusReport } = await downloadAndExtract(
codeqlURL,
dest,
authorization,
finalHeaders,
{ "User-Agent": "CodeQL Action", ...headers },
tarVersion,
tempDir,
features,
logger,
);
const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
logger.debug(
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.debug(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
const bundleVersion =
maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
@ -579,12 +541,7 @@ export const downloadCodeQL = async function (
);
return {
codeqlFolder: extractedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? "unknown",
};
}
@ -612,12 +569,7 @@ export const downloadCodeQL = async function (
return {
codeqlFolder: toolcachedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? toolcacheVersion,
};
};
@ -789,6 +741,7 @@ async function setupCodeQLBundleWithCompressionMethod(
apiDetails,
zstdAvailability.version,
tempDir,
features,
logger,
);
toolsVersion = result.toolsVersion;
@ -809,14 +762,6 @@ async function setupCodeQLBundleWithCompressionMethod(
};
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
)
? url
: "sanitized-value";
}
async function useZstdBundle(
cliVersion: string,
features: FeatureEnablement,

167
src/tools-download.ts Normal file
View file

@ -0,0 +1,167 @@
import * as http from "http";
import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { v4 as uuidV4 } from "uuid";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
import * as tar from "./tar";
import { cleanUpGlob } from "./util";
/**
* Timing information for the download and extraction of the CodeQL tools when
* we fully download the bundle before extracting.
*/
type DownloadFirstToolsDownloadDurations = {
combinedDurationMs: number;
downloadDurationMs: number;
extractionDurationMs: number;
};
/**
* Timing information for the download and extraction of the CodeQL tools when
* we stream the download and extraction of the bundle.
*/
type StreamedToolsDownloadDurations = {
combinedDurationMs: number;
downloadDurationMs: undefined;
extractionDurationMs: undefined;
};
type ToolsDownloadDurations =
| DownloadFirstToolsDownloadDurations
| StreamedToolsDownloadDurations;
export type ToolsDownloadStatusReport = {
compressionMethod: tar.CompressionMethod;
toolsUrl: string;
zstdFailureReason?: string;
} & ToolsDownloadDurations;
export async function downloadAndExtract(
codeqlURL: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
extractedBundlePath: string;
statusReport: ToolsDownloadStatusReport;
}> {
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
if (
compressionMethod === "zstd" &&
(await features.getValue(Feature.ZstdBundleStreamingExtraction))
) {
const toolsInstallStart = performance.now();
const extractedBundlePath = await downloadAndExtractZstdWithStreaming(
codeqlURL,
authorization,
headers,
tarVersion!,
logger,
);
const combinedDurationMs = Math.round(
performance.now() - toolsInstallStart,
);
logger.info(
`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${combinedDurationMs} ms).`,
);
return {
extractedBundlePath,
statusReport: {
combinedDurationMs,
compressionMethod,
downloadDurationMs: undefined,
extractionDurationMs: undefined,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
};
}
const dest = path.join(tempDir, uuidV4());
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
codeqlURL,
dest,
authorization,
headers,
);
const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
logger.info(
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.info("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.info(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
return {
extractedBundlePath,
statusReport: {
compressionMethod,
combinedDurationMs: downloadDurationMs + extractionDurationMs,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
};
}
async function downloadAndExtractZstdWithStreaming(
codeqlURL: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion,
logger: Logger,
): Promise<string> {
headers = Object.assign(
{ "User-Agent": "CodeQL Action", authorization },
headers,
);
const response = await new Promise<http.IncomingMessage>((resolve) =>
http.get(codeqlURL, { headers }, (r) => resolve(r)),
);
if (response.statusCode !== 200) {
throw new Error(
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
);
}
return await tar.extractTarZst(response, tarVersion, logger);
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
)
? url
: "sanitized-value";
}