Extract directly to toolcache when feature enabled

This commit is contained in:
Henry Mercer 2024-12-02 18:23:41 +00:00
parent 6695503422
commit 198fb4b4b2
15 changed files with 175 additions and 153 deletions

View file

@ -1,21 +1,27 @@
import { IncomingMessage, OutgoingHttpHeaders, RequestOptions } from "http";
import * as os from "os";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { https } from "follow-redirects";
import { v4 as uuidV4 } from "uuid";
import * as semver from "semver";
import { Feature, FeatureEnablement } from "./feature-flags";
import { formatDuration, Logger } from "./logging";
import * as tar from "./tar";
import { cleanUpGlob } from "./util";
import { cleanUpGlob, getRequiredEnvParam } from "./util";
/**
* High watermark to use when streaming the download and extraction of the CodeQL tools.
*/
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* The name of the tool cache directory for the CodeQL tools.
*/
const TOOLCACHE_TOOL_NAME = "CodeQL";
/**
* Timing information for the download and extraction of the CodeQL tools when
* we fully download the bundle before extracting.
@ -73,16 +79,13 @@ export type ToolsDownloadStatusReport = {
export async function downloadAndExtract(
codeqlURL: string,
dest: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
extractedBundlePath: string;
statusReport: ToolsDownloadStatusReport;
}> {
): Promise<ToolsDownloadStatusReport> {
logger.info(
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
);
@ -99,10 +102,9 @@ export async function downloadAndExtract(
logger.info(`Streaming the extraction of the CodeQL bundle.`);
const toolsInstallStart = performance.now();
const extractedBundlePath = await tar.createExtractFolder();
await downloadAndExtractZstdWithStreaming(
codeqlURL,
extractedBundlePath,
dest,
authorization,
headers,
tarVersion!,
@ -113,27 +115,22 @@ export async function downloadAndExtract(
performance.now() - toolsInstallStart,
);
logger.info(
`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${formatDuration(
`Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration(
combinedDurationMs,
)}).`,
);
return {
extractedBundlePath,
statusReport: {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeStreamedToolsDownloadDurations(combinedDurationMs),
},
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeStreamedToolsDownloadDurations(combinedDurationMs),
};
}
const dest = path.join(tempDir, uuidV4());
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
codeqlURL,
dest,
undefined,
authorization,
headers,
);
@ -145,21 +142,21 @@ export async function downloadAndExtract(
)}).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.info("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
await tar.extract(
archivedBundlePath,
dest,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.info(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${formatDuration(
`Finished extracting CodeQL bundle to ${dest} (${formatDuration(
extractionDurationMs,
)}).`,
);
@ -168,15 +165,12 @@ export async function downloadAndExtract(
}
return {
extractedBundlePath,
statusReport: {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeDownloadFirstToolsDownloadDurations(
downloadDurationMs,
extractionDurationMs,
),
},
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeDownloadFirstToolsDownloadDurations(
downloadDurationMs,
extractionDurationMs,
),
};
}
@ -213,6 +207,16 @@ async function downloadAndExtractZstdWithStreaming(
await tar.extractTarZst(response, dest, tarVersion, logger);
}
/** Gets the path to the toolcache directory for the specified version of the CodeQL tools. */
export function getToolcacheDirectory(version: string): string {
return path.join(
getRequiredEnvParam("RUNNER_TOOL_CACHE"),
TOOLCACHE_TOOL_NAME,
semver.clean(version) || version,
os.arch() || "",
);
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),