Merge pull request #2564 from github/henrymercer/only-use-stdin-for-streaming
Bundle installation performance improvements
This commit is contained in:
Henry Mercer
GitHub
commit
b97ec3aa00
12 changed files with 60 additions and 29 deletions
5
lib/setup-codeql.js
generated
5
lib/setup-codeql.js
generated
|
|
@ -219,10 +219,9 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
|
|||
*/
|
||||
const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);
|
||||
if (forceShippedTools) {
|
||||
logger.info(`Overriding the version of the CodeQL tools by ${defaultCliVersion.cliVersion}, the version shipped with the Action since ` +
|
||||
`tools: ${toolsInput} was requested.`);
|
||||
logger.info(`'tools: ${toolsInput}' was requested, so using CodeQL version ${defaultCliVersion.cliVersion}, the version shipped with the Action.`);
|
||||
if (toolsInput === "latest") {
|
||||
logger.warning("`tools: latest` has been renamed to `tools: linked`, but the old name is still supported for now. No action is required.");
|
||||
logger.warning("`tools: latest` has been renamed to `tools: linked`, but the old name is still supported. No action is required.");
|
||||
}
|
||||
}
|
||||
/** CLI version number, for example 2.12.6. */
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
2
lib/setup-codeql.test.js
generated
2
lib/setup-codeql.test.js
generated
|
|
@ -103,7 +103,7 @@ ava_1.default.beforeEach(() => {
|
|||
t.is(source.toolsVersion, testing_utils_1.LINKED_CLI_VERSION.cliVersion);
|
||||
t.is(source.sourceType, "download");
|
||||
// Afterwards, ensure that we see the deprecation message in the log.
|
||||
const expected_message = "`tools: latest` has been renamed to `tools: linked`, but the old name is still supported for now. No action is required.";
|
||||
const expected_message = "`tools: latest` has been renamed to `tools: linked`, but the old name is still supported. No action is required.";
|
||||
t.assert(loggedMessages.some((msg) => typeof msg.message === "string" &&
|
||||
msg.message.includes(expected_message)));
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
16
lib/tar.js
generated
16
lib/tar.js
generated
|
|
@ -33,6 +33,7 @@ exports.inferCompressionMethod = inferCompressionMethod;
|
|||
const child_process_1 = require("child_process");
|
||||
const fs = __importStar(require("fs"));
|
||||
const path_1 = __importDefault(require("path"));
|
||||
const stream = __importStar(require("stream"));
|
||||
const toolrunner_1 = require("@actions/exec/lib/toolrunner");
|
||||
const toolcache = __importStar(require("@actions/tool-cache"));
|
||||
const safe_which_1 = require("@chrisgavin/safe-which");
|
||||
|
|
@ -123,18 +124,21 @@ async function extract(tarPath, compressionMethod, tarVersion, logger) {
|
|||
if (!tarVersion) {
|
||||
throw new Error("Could not determine tar version, which is required to extract a Zstandard archive.");
|
||||
}
|
||||
return await extractTarZst(fs.createReadStream(tarPath), tarVersion, logger);
|
||||
return await extractTarZst(tarPath, tarVersion, logger);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Extract a compressed tar archive
|
||||
*
|
||||
* @param file path to the tar
|
||||
* @param tar tar stream, or path to the tar
|
||||
* @param dest destination directory. Optional.
|
||||
* @returns path to the destination directory
|
||||
*/
|
||||
async function extractTarZst(tarStream, tarVersion, logger) {
|
||||
async function extractTarZst(tar, tarVersion, logger) {
|
||||
const dest = await createExtractFolder();
|
||||
logger.debug(`Extracting to ${dest}.${tar instanceof stream.Readable
|
||||
? ` Input stream has high water mark ${tar.readableHighWaterMark}.`
|
||||
: ""}`);
|
||||
try {
|
||||
// Initialize args
|
||||
const args = ["-x", "--zstd"];
|
||||
|
|
@ -143,7 +147,7 @@ async function extractTarZst(tarStream, tarVersion, logger) {
|
|||
args.push("--warning=no-unknown-keyword");
|
||||
args.push("--overwrite");
|
||||
}
|
||||
args.push("-f", "-", "-C", dest);
|
||||
args.push("-f", tar instanceof stream.Readable ? "-" : tar, "-C", dest);
|
||||
process.stdout.write(`[command]tar ${args.join(" ")}\n`);
|
||||
const tarProcess = (0, child_process_1.spawn)("tar", args, { stdio: "pipe" });
|
||||
let stdout = "";
|
||||
|
|
@ -157,7 +161,9 @@ async function extractTarZst(tarStream, tarVersion, logger) {
|
|||
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
|
||||
process.stdout.write(data);
|
||||
});
|
||||
tarStream.pipe(tarProcess.stdin);
|
||||
if (tar instanceof stream.Readable) {
|
||||
tar.pipe(tarProcess.stdin);
|
||||
}
|
||||
await new Promise((resolve, reject) => {
|
||||
tarProcess.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
11
lib/tools-download.js
generated
11
lib/tools-download.js
generated
|
|
@ -23,6 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.STREAMING_HIGH_WATERMARK_BYTES = void 0;
|
||||
exports.downloadAndExtract = downloadAndExtract;
|
||||
const path = __importStar(require("path"));
|
||||
const perf_hooks_1 = require("perf_hooks");
|
||||
|
|
@ -33,6 +34,10 @@ const feature_flags_1 = require("./feature-flags");
|
|||
const logging_1 = require("./logging");
|
||||
const tar = __importStar(require("./tar"));
|
||||
const util_1 = require("./util");
|
||||
/**
|
||||
* High watermark to use when streaming the download and extraction of the CodeQL tools.
|
||||
*/
|
||||
exports.STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
|
||||
function makeDownloadFirstToolsDownloadDurations(downloadDurationMs, extractionDurationMs) {
|
||||
return {
|
||||
combinedDurationMs: downloadDurationMs + extractionDurationMs,
|
||||
|
|
@ -53,6 +58,7 @@ async function downloadAndExtract(codeqlURL, authorization, headers, tarVersion,
|
|||
logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
|
||||
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
|
||||
if (compressionMethod === "zstd" &&
|
||||
process.platform === "linux" &&
|
||||
(await features.getValue(feature_flags_1.Feature.ZstdBundleStreamingExtraction))) {
|
||||
logger.info(`Streaming the extraction of the CodeQL bundle.`);
|
||||
const toolsInstallStart = perf_hooks_1.performance.now();
|
||||
|
|
@ -96,7 +102,10 @@ async function downloadAndExtract(codeqlURL, authorization, headers, tarVersion,
|
|||
}
|
||||
async function downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger) {
|
||||
headers = Object.assign({ "User-Agent": "CodeQL Action", authorization }, headers);
|
||||
const response = await new Promise((resolve) => follow_redirects_1.https.get(codeqlURL, { headers }, (r) => resolve(r)));
|
||||
const response = await new Promise((resolve) => follow_redirects_1.https.get(codeqlURL, {
|
||||
headers,
|
||||
highWaterMark: exports.STREAMING_HIGH_WATERMARK_BYTES,
|
||||
}, (r) => resolve(r)));
|
||||
if (response.statusCode !== 200) {
|
||||
throw new Error(`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,gDAsGC;AAzKD,2CAA6B;AAC7B,2CAAyC;AAEzC,+DAAiD;AACjD,uDAAyC;AACzC,+BAAoC;AAEpC,mDAA6D;AAC7D,uCAAmD;AACnD,2CAA6B;AAC7B,iCAAqC;AAarC,SAAS,uCAAuC,CAC9C,kBAA0B,EAC1B,oBAA4B;IAE5B,OAAO;QACL,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;QAC7D,kBAAkB;QAClB,oBAAoB;QACpB,gBAAgB,EAAE,KAAK;KACxB,CAAC;AACJ,CAAC;AAaD,SAAS,kCAAkC,CACzC,kBAA0B;IAE1B,OAAO;QACL,kBAAkB;QAClB,kBAAkB,EAAE,SAAS;QAC7B,oBAAoB,EAAE,SAAS;QAC/B,gBAAgB,EAAE,IAAI;KACvB,CAAC;AACJ,CAAC;AAYM,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,OAAe,EACf,QAA2B,EAC3B,MAAc;IAKd,MAAM,CAAC,IAAI,CACT,iCAAiC,SAAS,2BAA2B,CACtE,CAAC;IAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,IACE,iBAAiB,KAAK,MAAM;QAC5B,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,uBAAO,CAAC,6BAA6B,CAAC,CAAC,EAChE,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAE9D,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,mBAAmB,GAAG,MAAM,mCAAmC,CACnE,SAAS,EACT,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;QACF,MAAM,CAAC,IAAI,CACT,wDAAwD,mBAAmB,KAAK,IAAA,wBAAc,EAC5F,kBAAkB,CACnB,IAAI,CACN,CAAC;QAEF,OAAO;YACL,mBAAmB;YACnB,YAAY,EAAE;gBACZ,iBAAiB;gBACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;gBAC/C,GAAG,kCAAkC,CAAC,kBAAkB,CAAC;aAC1D;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,IAAA,wBAAc,EAC5E,kBAAkB,CACnB,IAAI,CACN,CAAC;IAEF,IAAI,mBAA2B,CAAC;IAChC,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,mBAAmB,GAAG,MAAM,GAAG,CAAC,OAAO,CACrC,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,mBAAmB,KAAK,IAAA,wBAAc,EAC5E,oBAAoB,CACrB,IAAI,CACN,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,YAAY,EAAE;YACZ,iBAAiB;YACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;YAC/C,GAAG,uCAAuC,CACxC,kBAAkB,EAClB,oBAAoB,CACrB;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,EAChD,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,EAAE,CAC9D,wBAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CACrD,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}
|
||||
{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,gDAuGC;AA/KD,2CAA6B;AAC7B,2CAAyC;AAEzC,+DAAiD;AACjD,uDAAyC;AACzC,+BAAoC;AAEpC,mDAA6D;AAC7D,uCAAmD;AACnD,2CAA6B;AAC7B,iCAAqC;AAErC;;GAEG;AACU,QAAA,8BAA8B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAavE,SAAS,uCAAuC,CAC9C,kBAA0B,EAC1B,oBAA4B;IAE5B,OAAO;QACL,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;QAC7D,kBAAkB;QAClB,oBAAoB;QACpB,gBAAgB,EAAE,KAAK;KACxB,CAAC;AACJ,CAAC;AAaD,SAAS,kCAAkC,CACzC,kBAA0B;IAE1B,OAAO;QACL,kBAAkB;QAClB,kBAAkB,EAAE,SAAS;QAC7B,oBAAoB,EAAE,SAAS;QAC/B,gBAAgB,EAAE,IAAI;KACvB,CAAC;AACJ,CAAC;AAYM,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,OAAe,EACf,QAA2B,EAC3B,MAAc;IAKd,MAAM,CAAC,IAAI,CACT,iCAAiC,SAAS,2BAA2B,CACtE,CAAC;IAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,IACE,iBAAiB,KAAK,MAAM;QAC5B,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,uBAAO,CAAC,6BAA6B,CAAC,CAAC,EAChE,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAE9D,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,mBAAmB,GAAG,MAAM,mCAAmC,CACnE,SAAS,EACT,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;QACF,MAAM,CAAC,IAAI,CACT,wDAAwD,mBAAmB,KAAK,IAAA,wBAAc,EAC5F,kBAAkB,CACnB,IAAI,CACN,CAAC;QAEF,OAAO;YACL,mBAAmB;YACnB,YAAY,EAAE;gBACZ,iBAAiB;gBACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;gBAC/C,GAAG,kCAAkC,CAAC,kBAAkB,CAAC;aAC1D;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,IAAA,wBAAc,EAC5E,kBAAkB,CACnB,IAAI,CACN,CAAC;IAEF,IAAI,mBAA2B,CAAC;IAChC,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,mBAAmB,GAAG,MAAM,GAAG,CAAC,OAAO,CACrC,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,mBAAmB,KAAK,IAAA,wBAAc,EAC5E,oBAAoB,CACrB,IAAI,CACN,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,YAAY,EAAE;YACZ,iBAAiB;YACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;YAC/C,GAAG,uCAAuC,CACxC,kBAAkB,EAClB,oBAAoB,CACrB;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,EAChD,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,EAAE,CAC9D,wBAAK,CAAC,GAAG,CACP,SAAS,EACT;QACE,OAAO;QACP,aAAa,EAAE,sCAA8B;KACjB,EAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAClB,CACF,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}
|
||||
|
|
@ -141,7 +141,7 @@ test("getCodeQLSource correctly returns bundled CLI version when tools == latest
|
|||
|
||||
// Afterwards, ensure that we see the deprecation message in the log.
|
||||
const expected_message: string =
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported for now. No action is required.";
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported. No action is required.";
|
||||
t.assert(
|
||||
loggedMessages.some(
|
||||
(msg) =>
|
||||
|
|
|
|||
|
|
@ -274,13 +274,12 @@ export async function getCodeQLSource(
|
|||
toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);
|
||||
if (forceShippedTools) {
|
||||
logger.info(
|
||||
`Overriding the version of the CodeQL tools by ${defaultCliVersion.cliVersion}, the version shipped with the Action since ` +
|
||||
`tools: ${toolsInput} was requested.`,
|
||||
`'tools: ${toolsInput}' was requested, so using CodeQL version ${defaultCliVersion.cliVersion}, the version shipped with the Action.`,
|
||||
);
|
||||
|
||||
if (toolsInput === "latest") {
|
||||
logger.warning(
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported for now. No action is required.",
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported. No action is required.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
23
src/tar.ts
23
src/tar.ts
|
|
@ -125,27 +125,30 @@ export async function extract(
|
|||
"Could not determine tar version, which is required to extract a Zstandard archive.",
|
||||
);
|
||||
}
|
||||
return await extractTarZst(
|
||||
fs.createReadStream(tarPath),
|
||||
tarVersion,
|
||||
logger,
|
||||
);
|
||||
return await extractTarZst(tarPath, tarVersion, logger);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a compressed tar archive
|
||||
*
|
||||
* @param file path to the tar
|
||||
* @param tar tar stream, or path to the tar
|
||||
* @param dest destination directory. Optional.
|
||||
* @returns path to the destination directory
|
||||
*/
|
||||
export async function extractTarZst(
|
||||
tarStream: stream.Readable,
|
||||
tar: stream.Readable | string,
|
||||
tarVersion: TarVersion,
|
||||
logger: Logger,
|
||||
): Promise<string> {
|
||||
const dest = await createExtractFolder();
|
||||
logger.debug(
|
||||
`Extracting to ${dest}.${
|
||||
tar instanceof stream.Readable
|
||||
? ` Input stream has high water mark ${tar.readableHighWaterMark}.`
|
||||
: ""
|
||||
}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Initialize args
|
||||
|
|
@ -157,7 +160,7 @@ export async function extractTarZst(
|
|||
args.push("--overwrite");
|
||||
}
|
||||
|
||||
args.push("-f", "-", "-C", dest);
|
||||
args.push("-f", tar instanceof stream.Readable ? "-" : tar, "-C", dest);
|
||||
|
||||
process.stdout.write(`[command]tar ${args.join(" ")}\n`);
|
||||
|
||||
|
|
@ -175,7 +178,9 @@ export async function extractTarZst(
|
|||
process.stdout.write(data);
|
||||
});
|
||||
|
||||
tarStream.pipe(tarProcess.stdin);
|
||||
if (tar instanceof stream.Readable) {
|
||||
tar.pipe(tarProcess.stdin);
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
tarProcess.on("exit", (code) => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { IncomingMessage, OutgoingHttpHeaders } from "http";
|
||||
import { IncomingMessage, OutgoingHttpHeaders, RequestOptions } from "http";
|
||||
import * as path from "path";
|
||||
import { performance } from "perf_hooks";
|
||||
|
||||
|
|
@ -11,6 +11,11 @@ import { formatDuration, Logger } from "./logging";
|
|||
import * as tar from "./tar";
|
||||
import { cleanUpGlob } from "./util";
|
||||
|
||||
/**
|
||||
* High watermark to use when streaming the download and extraction of the CodeQL tools.
|
||||
*/
|
||||
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
|
||||
|
||||
/**
|
||||
* Timing information for the download and extraction of the CodeQL tools when
|
||||
* we fully download the bundle before extracting.
|
||||
|
|
@ -86,6 +91,7 @@ export async function downloadAndExtract(
|
|||
|
||||
if (
|
||||
compressionMethod === "zstd" &&
|
||||
process.platform === "linux" &&
|
||||
(await features.getValue(Feature.ZstdBundleStreamingExtraction))
|
||||
) {
|
||||
logger.info(`Streaming the extraction of the CodeQL bundle.`);
|
||||
|
|
@ -182,7 +188,14 @@ async function downloadAndExtractZstdWithStreaming(
|
|||
headers,
|
||||
);
|
||||
const response = await new Promise<IncomingMessage>((resolve) =>
|
||||
https.get(codeqlURL, { headers }, (r) => resolve(r)),
|
||||
https.get(
|
||||
codeqlURL,
|
||||
{
|
||||
headers,
|
||||
highWaterMark: STREAMING_HIGH_WATERMARK_BYTES,
|
||||
} as unknown as RequestOptions,
|
||||
(r) => resolve(r),
|
||||
),
|
||||
);
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue