Merge branch 'main' into use-cache-cleanup-flag

src/codeql.ts

@@ -373,6 +373,13 @@ export async function setupCodeQL(
       defaultCliVersion,
       logger,
     );
+
+    logger.debug(
+      `Bundle download status report: ${JSON.stringify(
+        toolsDownloadStatusReport,
+      )}`,
+    );
+
     let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
     if (process.platform === "win32") {
       codeqlCmd += ".exe";

src/init-action.ts

@@ -42,6 +42,7 @@ import {
   getActionsStatus,
   sendStatusReport,
 } from "./status-report";
+import { isZstdAvailable } from "./tar";
 import { ToolsFeature } from "./tools-features";
 import { getTotalCacheSize } from "./trap-caching";
 import {
@@ -375,6 +376,8 @@ async function run() {
   try {
     cleanupDatabaseClusterDirectory(config, logger);
 
+    await logZstdAvailability(config, logger);
+
     // Log CodeQL download telemetry, if appropriate
     if (toolsDownloadStatusReport) {
       addDiagnostic(
@@ -670,6 +673,29 @@ function getTrapCachingEnabled(): boolean {
   return true;
 }
 
+async function logZstdAvailability(config: configUtils.Config, logger: Logger) {
+  // Log zstd availability
+  const zstdAvailableResult = await isZstdAvailable(logger);
+  addDiagnostic(
+    config,
+    // Arbitrarily choose the first language. We could also choose all languages, but that
+    // increases the risk of misinterpreting the data.
+    config.languages[0],
+    makeDiagnostic(
+      "codeql-action/zstd-availability",
+      "Zstandard availability",
+      {
+        attributes: zstdAvailableResult,
+        visibility: {
+          cliSummaryTable: false,
+          statusPage: false,
+          telemetry: true,
+        },
+      },
+    ),
+  );
+}
+
 async function runWrapper() {
   try {
     await run();

src/setup-codeql.test.ts

@@ -154,6 +154,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
   sinon.stub(setupCodeql, "downloadCodeQL").resolves({
     codeqlFolder: "codeql",
     statusReport: {
+      compressionMethod: "gzip",
       downloadDurationMs: 200,
       extractionDurationMs: 300,
     },
@@ -200,6 +201,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
   sinon.stub(setupCodeql, "downloadCodeQL").resolves({
     codeqlFolder: "codeql",
     statusReport: {
+      compressionMethod: "gzip",
       downloadDurationMs: 200,
       extractionDurationMs: 300,
     },

src/setup-codeql.ts

@@ -17,6 +17,7 @@ import * as api from "./api-client";
 import * as defaults from "./defaults.json";
 import { CodeQLDefaultVersionInfo } from "./feature-flags";
 import { Logger } from "./logging";
+import * as tar from "./tar";
 import * as util from "./util";
 import { isGoodVersion } from "./util";
 
@@ -462,6 +463,7 @@ export async function tryGetFallbackToolcacheVersion(
 }
 
 export interface ToolsDownloadStatusReport {
+  compressionMethod: tar.CompressionMethod;
   downloadDurationMs: number;
   extractionDurationMs: number;
 }
@@ -505,6 +507,7 @@ export const downloadCodeQL = async function (
     `Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
   );
 
+  const compressionMethod = tar.inferCompressionMethod(codeqlURL);
   const dest = path.join(tempDir, uuidV4());
   const finalHeaders = Object.assign(
     { "User-Agent": "CodeQL Action" },
@@ -526,7 +529,10 @@ export const downloadCodeQL = async function (
 
   logger.debug("Extracting CodeQL bundle.");
   const extractionStart = performance.now();
-  const extractedBundlePath = await toolcache.extractTar(archivedBundlePath);
+  const extractedBundlePath = await tar.extract(
+    archivedBundlePath,
+    compressionMethod,
+  );
   const extractionDurationMs = Math.round(performance.now() - extractionStart);
   logger.debug(
     `Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
@@ -544,6 +550,7 @@ export const downloadCodeQL = async function (
     return {
       codeqlFolder: extractedBundlePath,
       statusReport: {
+        compressionMethod,
         downloadDurationMs,
         extractionDurationMs,
       },
@@ -575,6 +582,7 @@ export const downloadCodeQL = async function (
   return {
     codeqlFolder: toolcachedBundlePath,
     statusReport: {
+      compressionMethod,
       downloadDurationMs,
       extractionDurationMs,
     },
@@ -619,17 +627,16 @@ function getCanonicalToolcacheVersion(
   return cliVersion;
 }
 
+export interface SetupCodeQLResult {
+  codeqlFolder: string;
+  toolsDownloadStatusReport?: ToolsDownloadStatusReport;
+  toolsSource: ToolsSource;
+  toolsVersion: string;
+}
+
 /**
  * Obtains the CodeQL bundle, installs it in the toolcache if appropriate, and extracts it.
  *
- * @param toolsInput
- * @param apiDetails
- * @param tempDir
- * @param variant
- * @param defaultCliVersion
- * @param logger
- * @param checkVersion Whether to check that CodeQL CLI meets the minimum
- *        version requirement. Must be set to true outside tests.
  * @returns the path to the extracted bundle, and the version of the tools
  */
 export async function setupCodeQLBundle(
@@ -639,12 +646,7 @@ export async function setupCodeQLBundle(
   variant: util.GitHubVariant,
   defaultCliVersion: CodeQLDefaultVersionInfo,
   logger: Logger,
-): Promise<{
-  codeqlFolder: string;
-  toolsDownloadStatusReport?: ToolsDownloadStatusReport;
-  toolsSource: ToolsSource;
-  toolsVersion: string;
-}> {
+): Promise<SetupCodeQLResult> {
   const source = await getCodeQLSource(
     toolsInput,
     defaultCliVersion,
@@ -658,10 +660,14 @@ export async function setupCodeQLBundle(
   let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined;
   let toolsSource: ToolsSource;
   switch (source.sourceType) {
-    case "local":
-      codeqlFolder = await toolcache.extractTar(source.codeqlTarPath);
+    case "local": {
+      const compressionMethod = tar.inferCompressionMethod(
+        source.codeqlTarPath,
+      );
+      codeqlFolder = await tar.extract(source.codeqlTarPath, compressionMethod);
       toolsSource = ToolsSource.Local;
       break;
+    }
     case "toolcache":
       codeqlFolder = source.codeqlFolder;
       logger.debug(`CodeQL found in cache ${codeqlFolder}`);

src/tar.ts

@@ -0,0 +1,103 @@
+import { ToolRunner } from "@actions/exec/lib/toolrunner";
+import * as toolcache from "@actions/tool-cache";
+import { safeWhich } from "@chrisgavin/safe-which";
+
+import { Logger } from "./logging";
+import { assertNever } from "./util";
+
+const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
+const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
+
+export type TarVersion = {
+  type: "gnu" | "bsd";
+  version: string;
+};
+
+async function getTarVersion(): Promise<TarVersion> {
+  const tar = await safeWhich("tar");
+  let stdout = "";
+  const exitCode = await new ToolRunner(tar, ["--version"], {
+    listeners: {
+      stdout: (data: Buffer) => {
+        stdout += data.toString();
+      },
+    },
+  }).exec();
+  if (exitCode !== 0) {
+    throw new Error("Failed to call tar --version");
+  }
+  // Return whether this is GNU tar or BSD tar, and the version number
+  if (stdout.includes("GNU tar")) {
+    const match = stdout.match(/tar \(GNU tar\) ([0-9.]+)/);
+    if (!match || !match[1]) {
+      throw new Error("Failed to parse output of tar --version.");
+    }
+
+    return { type: "gnu", version: match[1] };
+  } else if (stdout.includes("bsdtar")) {
+    const match = stdout.match(/bsdtar ([0-9.]+)/);
+    if (!match || !match[1]) {
+      throw new Error("Failed to parse output of tar --version.");
+    }
+
+    return { type: "bsd", version: match[1] };
+  } else {
+    throw new Error("Unknown tar version");
+  }
+}
+
+export async function isZstdAvailable(
+  logger: Logger,
+): Promise<{ available: boolean; version?: TarVersion }> {
+  try {
+    const tarVersion = await getTarVersion();
+    const { type, version } = tarVersion;
+    logger.info(`Found ${type} tar version ${version}.`);
+    switch (type) {
+      case "gnu":
+        return {
+          available: version >= MIN_REQUIRED_GNU_TAR_VERSION,
+          version: tarVersion,
+        };
+      case "bsd":
+        return {
+          available: version >= MIN_REQUIRED_BSD_TAR_VERSION,
+          version: tarVersion,
+        };
+      default:
+        assertNever(type);
+    }
+  } catch (e) {
+    logger.error(
+      "Failed to determine tar version, therefore will assume zstd may not be available. " +
+        `The underlying error was: ${e}`,
+    );
+    return { available: false };
+  }
+}
+
+export type CompressionMethod = "gzip" | "zstd";
+
+export async function extract(
+  path: string,
+  compressionMethod: CompressionMethod,
+): Promise<string> {
+  switch (compressionMethod) {
+    case "gzip":
+      // While we could also ask tar to autodetect the compression method,
+      // we defensively keep the gzip call identical as requesting a gzipped
+      // bundle will soon be a fallback option.
+      return await toolcache.extractTar(path);
+    case "zstd":
+      // By specifying only the "x" flag, we ask tar to autodetect the
+      // compression method.
+      return await toolcache.extractTar(path, undefined, "x");
+  }
+}
+
+export function inferCompressionMethod(path: string): CompressionMethod {
+  if (path.endsWith(".tar.gz")) {
+    return "gzip";
+  }
+  return "zstd";
+}