robojerk/codeql-action

Stream the download and extraction of zstd bundles

Browse source 
Behind a feature flag
This commit is contained in:
Henry Mercer 2024-10-10 18:23:06 +01:00
parent cd83b08c78
commit ddead5420c
22 changed files with 371 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

49
lib/setup-codeql.js generated
View file

@ -36,19 +36,15 @@ exports.getCodeQLURLVersion = getCodeQLURLVersion;
exports.setupCodeQLBundle = setupCodeQLBundle;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const toolcache = __importStar(require("@actions/tool-cache"));
const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
const semver = __importStar(require("semver"));
const uuid_1 = require("uuid");
const actions_util_1 = require("./actions-util");
const api = __importStar(require("./api-client"));
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
const defaults = __importStar(require("./defaults.json"));
const feature_flags_1 = require("./feature-flags");
const tar = __importStar(require("./tar"));
const tools_download_1 = require("./tools-download");
const util = __importStar(require("./util"));
const util_1 = require("./util");
var ToolsSource;
@ -372,7 +368,7 @@ async function tryGetFallbackToolcacheVersion(cliVersion, tagName, logger) {
}
// Exported using `export const` for testing purposes. Specifically, we want to
// be able to stub this function and have other functions in this file use that stub.
const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, logger) {
const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, features, logger) {
const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers = {
@ -395,37 +391,14 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
const dest = path.join(tempDir, (0, uuid_1.v4)());
const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
const toolsDownloadStart = perf_hooks_1.performance.now();
const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
let extractedBundlePath;
let extractionDurationMs;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = perf_hooks_1.performance.now();
extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
}
finally {
await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
}
const { extractedBundlePath, statusReport } = await (0, tools_download_1.downloadAndExtract)(codeqlURL, authorization, { "User-Agent": "CodeQL Action", ...headers }, tarVersion, tempDir, features, logger);
const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`);
return {
codeqlFolder: extractedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? "unknown",
};
}
@ -438,12 +411,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
}
return {
codeqlFolder: toolcachedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? toolcacheVersion,
};
};
@ -529,7 +497,7 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
toolsSource = ToolsSource.Toolcache;
break;
case "download": {
const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, logger);
const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, features, logger);
toolsVersion = result.toolsVersion;
codeqlFolder = result.codeqlFolder;
toolsDownloadStatusReport = result.statusReport;
@ -547,11 +515,6 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
zstdAvailability,
};
}
function sanitizeUrlForStatusReport(url) {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
? url
: "sanitized-value";
}
async function useZstdBundle(cliVersion, features, tarSupportsZstd) {
return (
// In testing, gzip performs better than zstd on Windows.