Merge pull request #2442 from github/henrymercer/zstd-bundles

Add support for using zstd-compressed nightly bundles

.github/actions/prepare-test/action.yml

@@ -32,14 +32,20 @@ runs:
       run: |
         set -e # Fail this Action if `gh release list` fails.
 
+        if [[ ${{ inputs.version }} == "nightly-latest" ]]; then
+          extension="tar.zst"
+        else
+          extension="tar.gz"
+        fi
+
         if [[ ${{ inputs.use-all-platform-bundle }} == "true" ]]; then
-          artifact_name="codeql-bundle.tar.gz"
+          artifact_name="codeql-bundle.$extension"
         elif [[ "$RUNNER_OS" == "Linux" ]]; then
-          artifact_name="codeql-bundle-linux64.tar.gz"
+          artifact_name="codeql-bundle-linux64.$extension"
         elif [[ "$RUNNER_OS" == "macOS" ]]; then
-          artifact_name="codeql-bundle-osx64.tar.gz"
+          artifact_name="codeql-bundle-osx64.$extension"
         elif [[ "$RUNNER_OS" == "Windows" ]]; then
-          artifact_name="codeql-bundle-win64.tar.gz"
+          artifact_name="codeql-bundle-win64.$extension"
         else
           echo "::error::Unrecognized OS $RUNNER_OS"
           exit 1

.github/workflows/__test-local-codeql.yml

@@ -66,7 +66,7 @@ jobs:
         with:
       # Swift is not supported on Ubuntu so we manually exclude it from the list here
           languages: cpp,csharp,go,java,javascript,python,ruby
-          tools: ./codeql-bundle-linux64.tar.gz
+          tools: ./codeql-bundle-linux64.tar.zst
       - name: Build code
         shell: bash
         run: ./build.sh

lib/codeql.js

@@ -121,6 +121,7 @@ const CODEQL_VERSION_INCLUDE_QUERY_HELP = "2.15.2";
 async function setupCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger, checkVersion) {
     try {
         const { codeqlFolder, toolsDownloadStatusReport, toolsSource, toolsVersion, } = await setupCodeql.setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger);
+        logger.debug(`Bundle download status report: ${JSON.stringify(toolsDownloadStatusReport)}`);
         let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
         if (process.platform === "win32") {
             codeqlCmd += ".exe";

lib/init-action.js

@@ -40,6 +40,7 @@ const logging_1 = require("./logging");
 const repository_1 = require("./repository");
 const setup_codeql_1 = require("./setup-codeql");
 const status_report_1 = require("./status-report");
+const tar_1 = require("./tar");
 const tools_features_1 = require("./tools-features");
 const trap_caching_1 = require("./trap-caching");
 const util_1 = require("./util");
@@ -208,6 +209,7 @@ async function run() {
     }
     try {
         (0, init_1.cleanupDatabaseClusterDirectory)(config, logger);
+        await logZstdAvailability(config, logger);
         // Log CodeQL download telemetry, if appropriate
         if (toolsDownloadStatusReport) {
             (0, diagnostics_1.addDiagnostic)(config, 
@@ -390,6 +392,21 @@ function getTrapCachingEnabled() {
     // On hosted runners, enable TRAP caching by default
     return true;
 }
+async function logZstdAvailability(config, logger) {
+    // Log zstd availability
+    const zstdAvailableResult = await (0, tar_1.isZstdAvailable)(logger);
+    (0, diagnostics_1.addDiagnostic)(config, 
+    // Arbitrarily choose the first language. We could also choose all languages, but that
+    // increases the risk of misinterpreting the data.
+    config.languages[0], (0, diagnostics_1.makeDiagnostic)("codeql-action/zstd-availability", "Zstandard availability", {
+        attributes: zstdAvailableResult,
+        visibility: {
+            cliSummaryTable: false,
+            statusPage: false,
+            telemetry: true,
+        },
+    }));
+}
 async function runWrapper() {
     try {
         await run();

lib/setup-codeql.js

@@ -48,6 +48,7 @@ const api = __importStar(require("./api-client"));
 // creation scripts. Ensure that any changes to the format of this file are compatible with both of
 // these dependents.
 const defaults = __importStar(require("./defaults.json"));
+const tar = __importStar(require("./tar"));
 const util = __importStar(require("./util"));
 const util_1 = require("./util");
 var ToolsSource;
@@ -389,6 +390,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
         logger.debug("Downloading CodeQL tools without an authorization token.");
     }
     logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
+    const compressionMethod = tar.inferCompressionMethod(codeqlURL);
     const dest = path.join(tempDir, (0, uuid_1.v4)());
     const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
     const toolsDownloadStart = perf_hooks_1.performance.now();
@@ -397,7 +399,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
     logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
     logger.debug("Extracting CodeQL bundle.");
     const extractionStart = perf_hooks_1.performance.now();
-    const extractedBundlePath = await toolcache.extractTar(archivedBundlePath);
+    const extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod);
     const extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
     logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
     await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
@@ -408,6 +410,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
         return {
             codeqlFolder: extractedBundlePath,
             statusReport: {
+                compressionMethod,
                 downloadDurationMs,
                 extractionDurationMs,
             },
@@ -424,6 +427,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
     return {
         codeqlFolder: toolcachedBundlePath,
         statusReport: {
+            compressionMethod,
             downloadDurationMs,
             extractionDurationMs,
         },
@@ -463,14 +467,6 @@ function getCanonicalToolcacheVersion(cliVersion, bundleVersion, logger) {
 /**
  * Obtains the CodeQL bundle, installs it in the toolcache if appropriate, and extracts it.
  *
- * @param toolsInput
- * @param apiDetails
- * @param tempDir
- * @param variant
- * @param defaultCliVersion
- * @param logger
- * @param checkVersion Whether to check that CodeQL CLI meets the minimum
- *        version requirement. Must be set to true outside tests.
  * @returns the path to the extracted bundle, and the version of the tools
  */
 async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger) {
@@ -480,10 +476,12 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau
     let toolsDownloadStatusReport;
     let toolsSource;
     switch (source.sourceType) {
-        case "local":
-            codeqlFolder = await toolcache.extractTar(source.codeqlTarPath);
+        case "local": {
+            const compressionMethod = tar.inferCompressionMethod(source.codeqlTarPath);
+            codeqlFolder = await tar.extract(source.codeqlTarPath, compressionMethod);
             toolsSource = ToolsSource.Local;
             break;
+        }
         case "toolcache":
             codeqlFolder = source.codeqlFolder;
             logger.debug(`CodeQL found in cache ${codeqlFolder}`);

lib/setup-codeql.test.js

@@ -116,6 +116,7 @@ ava_1.default.beforeEach(() => {
     sinon.stub(setupCodeql, "downloadCodeQL").resolves({
         codeqlFolder: "codeql",
         statusReport: {
+            compressionMethod: "gzip",
             downloadDurationMs: 200,
             extractionDurationMs: 300,
         },
@@ -143,6 +144,7 @@ ava_1.default.beforeEach(() => {
     sinon.stub(setupCodeql, "downloadCodeQL").resolves({
         codeqlFolder: "codeql",
         statusReport: {
+            compressionMethod: "gzip",
             downloadDurationMs: 200,
             extractionDurationMs: 300,
         },

lib/tar.js

@@ -0,0 +1,112 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isZstdAvailable = isZstdAvailable;
+exports.extract = extract;
+exports.inferCompressionMethod = inferCompressionMethod;
+const toolrunner_1 = require("@actions/exec/lib/toolrunner");
+const toolcache = __importStar(require("@actions/tool-cache"));
+const safe_which_1 = require("@chrisgavin/safe-which");
+const util_1 = require("./util");
+const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
+const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
+async function getTarVersion() {
+    const tar = await (0, safe_which_1.safeWhich)("tar");
+    let stdout = "";
+    const exitCode = await new toolrunner_1.ToolRunner(tar, ["--version"], {
+        listeners: {
+            stdout: (data) => {
+                stdout += data.toString();
+            },
+        },
+    }).exec();
+    if (exitCode !== 0) {
+        throw new Error("Failed to call tar --version");
+    }
+    // Return whether this is GNU tar or BSD tar, and the version number
+    if (stdout.includes("GNU tar")) {
+        const match = stdout.match(/tar \(GNU tar\) ([0-9.]+)/);
+        if (!match || !match[1]) {
+            throw new Error("Failed to parse output of tar --version.");
+        }
+        return { type: "gnu", version: match[1] };
+    }
+    else if (stdout.includes("bsdtar")) {
+        const match = stdout.match(/bsdtar ([0-9.]+)/);
+        if (!match || !match[1]) {
+            throw new Error("Failed to parse output of tar --version.");
+        }
+        return { type: "bsd", version: match[1] };
+    }
+    else {
+        throw new Error("Unknown tar version");
+    }
+}
+async function isZstdAvailable(logger) {
+    try {
+        const tarVersion = await getTarVersion();
+        const { type, version } = tarVersion;
+        logger.info(`Found ${type} tar version ${version}.`);
+        switch (type) {
+            case "gnu":
+                return {
+                    available: version >= MIN_REQUIRED_GNU_TAR_VERSION,
+                    version: tarVersion,
+                };
+            case "bsd":
+                return {
+                    available: version >= MIN_REQUIRED_BSD_TAR_VERSION,
+                    version: tarVersion,
+                };
+            default:
+                (0, util_1.assertNever)(type);
+        }
+    }
+    catch (e) {
+        logger.error("Failed to determine tar version, therefore will assume zstd may not be available. " +
+            `The underlying error was: ${e}`);
+        return { available: false };
+    }
+}
+async function extract(path, compressionMethod) {
+    switch (compressionMethod) {
+        case "gzip":
+            // While we could also ask tar to autodetect the compression method,
+            // we defensively keep the gzip call identical as requesting a gzipped
+            // bundle will soon be a fallback option.
+            return await toolcache.extractTar(path);
+        case "zstd":
+            // By specifying only the "x" flag, we ask tar to autodetect the
+            // compression method.
+            return await toolcache.extractTar(path, undefined, "x");
+    }
+}
+function inferCompressionMethod(path) {
+    if (path.endsWith(".tar.gz")) {
+        return "gzip";
+    }
+    return "zstd";
+}
+//# sourceMappingURL=tar.js.map

lib/tar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tar.js","sourceRoot":"","sources":["../src/tar.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,0CA4BC;AAID,0BAeC;AAED,wDAKC;AAtGD,6DAA0D;AAC1D,+DAAiD;AACjD,uDAAmD;AAGnD,iCAAqC;AAErC,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAC7C,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAO5C,KAAK,UAAU,aAAa;IAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAS,EAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,uBAAU,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE;QACxD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IACV,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,oEAAoE;IACpE,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,gBAAgB,OAAO,GAAG,CAAC,CAAC;QACrD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,OAAO,IAAI,4BAA4B;oBAClD,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS,EAAE,OAAO,IAAI,4BAA4B;oBAClD,OAAO,EAAE,UAAU;iBACpB,CAAC;YACJ;gBACE,IAAA,kBAAW,EAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,oFAAoF;YAClF,6BAA6B,CAAC,EAAE,CACnC,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;AACH,CAAC;AAIM,KAAK,UAAU,OAAO,CAC3B,IAAY,EACZ,iBAAoC;IAEpC,QAAQ,iBAAiB,EAAE,CAAC;QAC1B,KAAK,MAAM;YACT,oEAAoE;YACpE,sEAAsE;YACtE,yCAAyC;YACzC,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1C,KAAK,MAAM;YACT,gEAAgE;YAChE,sBAAsB;YACtB,OAAO,MAAM,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,SAAgB,sBAAsB,CAAC,IAAY;IACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

pr-checks/checks/test-local-codeql.yml

@@ -14,7 +14,7 @@ steps:
     with:
       # Swift is not supported on Ubuntu so we manually exclude it from the list here
       languages: cpp,csharp,go,java,javascript,python,ruby
-      tools: ./codeql-bundle-linux64.tar.gz
+      tools: ./codeql-bundle-linux64.tar.zst
   - name: Build code
     shell: bash
     run: ./build.sh

src/codeql.ts

@@ -368,6 +368,13 @@ export async function setupCodeQL(
       defaultCliVersion,
       logger,
     );
+
+    logger.debug(
+      `Bundle download status report: ${JSON.stringify(
+        toolsDownloadStatusReport,
+      )}`,
+    );
+
     let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
     if (process.platform === "win32") {
       codeqlCmd += ".exe";

src/init-action.ts

@@ -42,6 +42,7 @@ import {
   getActionsStatus,
   sendStatusReport,
 } from "./status-report";
+import { isZstdAvailable } from "./tar";
 import { ToolsFeature } from "./tools-features";
 import { getTotalCacheSize } from "./trap-caching";
 import {
@@ -375,6 +376,8 @@ async function run() {
   try {
     cleanupDatabaseClusterDirectory(config, logger);
 
+    await logZstdAvailability(config, logger);
+
     // Log CodeQL download telemetry, if appropriate
     if (toolsDownloadStatusReport) {
       addDiagnostic(
@@ -670,6 +673,29 @@ function getTrapCachingEnabled(): boolean {
   return true;
 }
 
+async function logZstdAvailability(config: configUtils.Config, logger: Logger) {
+  // Log zstd availability
+  const zstdAvailableResult = await isZstdAvailable(logger);
+  addDiagnostic(
+    config,
+    // Arbitrarily choose the first language. We could also choose all languages, but that
+    // increases the risk of misinterpreting the data.
+    config.languages[0],
+    makeDiagnostic(
+      "codeql-action/zstd-availability",
+      "Zstandard availability",
+      {
+        attributes: zstdAvailableResult,
+        visibility: {
+          cliSummaryTable: false,
+          statusPage: false,
+          telemetry: true,
+        },
+      },
+    ),
+  );
+}
+
 async function runWrapper() {
   try {
     await run();

src/setup-codeql.test.ts

@@ -154,6 +154,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
   sinon.stub(setupCodeql, "downloadCodeQL").resolves({
     codeqlFolder: "codeql",
     statusReport: {
+      compressionMethod: "gzip",
       downloadDurationMs: 200,
       extractionDurationMs: 300,
     },
@@ -200,6 +201,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
   sinon.stub(setupCodeql, "downloadCodeQL").resolves({
     codeqlFolder: "codeql",
     statusReport: {
+      compressionMethod: "gzip",
       downloadDurationMs: 200,
       extractionDurationMs: 300,
     },

src/setup-codeql.ts

@@ -17,6 +17,7 @@ import * as api from "./api-client";
 import * as defaults from "./defaults.json";
 import { CodeQLDefaultVersionInfo } from "./feature-flags";
 import { Logger } from "./logging";
+import * as tar from "./tar";
 import * as util from "./util";
 import { isGoodVersion } from "./util";
 
@@ -462,6 +463,7 @@ export async function tryGetFallbackToolcacheVersion(
 }
 
 export interface ToolsDownloadStatusReport {
+  compressionMethod: tar.CompressionMethod;
   downloadDurationMs: number;
   extractionDurationMs: number;
 }
@@ -505,6 +507,7 @@ export const downloadCodeQL = async function (
     `Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
   );
 
+  const compressionMethod = tar.inferCompressionMethod(codeqlURL);
   const dest = path.join(tempDir, uuidV4());
   const finalHeaders = Object.assign(
     { "User-Agent": "CodeQL Action" },
@@ -526,7 +529,10 @@ export const downloadCodeQL = async function (
 
   logger.debug("Extracting CodeQL bundle.");
   const extractionStart = performance.now();
-  const extractedBundlePath = await toolcache.extractTar(archivedBundlePath);
+  const extractedBundlePath = await tar.extract(
+    archivedBundlePath,
+    compressionMethod,
+  );
   const extractionDurationMs = Math.round(performance.now() - extractionStart);
   logger.debug(
     `Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
@@ -544,6 +550,7 @@ export const downloadCodeQL = async function (
     return {
       codeqlFolder: extractedBundlePath,
       statusReport: {
+        compressionMethod,
         downloadDurationMs,
         extractionDurationMs,
       },
@@ -575,6 +582,7 @@ export const downloadCodeQL = async function (
   return {
     codeqlFolder: toolcachedBundlePath,
     statusReport: {
+      compressionMethod,
       downloadDurationMs,
       extractionDurationMs,
     },
@@ -619,17 +627,16 @@ function getCanonicalToolcacheVersion(
   return cliVersion;
 }
 
+export interface SetupCodeQLResult {
+  codeqlFolder: string;
+  toolsDownloadStatusReport?: ToolsDownloadStatusReport;
+  toolsSource: ToolsSource;
+  toolsVersion: string;
+}
+
 /**
  * Obtains the CodeQL bundle, installs it in the toolcache if appropriate, and extracts it.
  *
- * @param toolsInput
- * @param apiDetails
- * @param tempDir
- * @param variant
- * @param defaultCliVersion
- * @param logger
- * @param checkVersion Whether to check that CodeQL CLI meets the minimum
- *        version requirement. Must be set to true outside tests.
  * @returns the path to the extracted bundle, and the version of the tools
  */
 export async function setupCodeQLBundle(
@@ -639,12 +646,7 @@ export async function setupCodeQLBundle(
   variant: util.GitHubVariant,
   defaultCliVersion: CodeQLDefaultVersionInfo,
   logger: Logger,
-): Promise<{
-  codeqlFolder: string;
-  toolsDownloadStatusReport?: ToolsDownloadStatusReport;
-  toolsSource: ToolsSource;
-  toolsVersion: string;
-}> {
+): Promise<SetupCodeQLResult> {
   const source = await getCodeQLSource(
     toolsInput,
     defaultCliVersion,
@@ -658,10 +660,14 @@ export async function setupCodeQLBundle(
   let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined;
   let toolsSource: ToolsSource;
   switch (source.sourceType) {
-    case "local":
-      codeqlFolder = await toolcache.extractTar(source.codeqlTarPath);
+    case "local": {
+      const compressionMethod = tar.inferCompressionMethod(
+        source.codeqlTarPath,
+      );
+      codeqlFolder = await tar.extract(source.codeqlTarPath, compressionMethod);
       toolsSource = ToolsSource.Local;
       break;
+    }
     case "toolcache":
       codeqlFolder = source.codeqlFolder;
       logger.debug(`CodeQL found in cache ${codeqlFolder}`);

src/tar.ts

@@ -0,0 +1,103 @@
+import { ToolRunner } from "@actions/exec/lib/toolrunner";
+import * as toolcache from "@actions/tool-cache";
+import { safeWhich } from "@chrisgavin/safe-which";
+
+import { Logger } from "./logging";
+import { assertNever } from "./util";
+
+const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
+const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
+
+export type TarVersion = {
+  type: "gnu" | "bsd";
+  version: string;
+};
+
+async function getTarVersion(): Promise<TarVersion> {
+  const tar = await safeWhich("tar");
+  let stdout = "";
+  const exitCode = await new ToolRunner(tar, ["--version"], {
+    listeners: {
+      stdout: (data: Buffer) => {
+        stdout += data.toString();
+      },
+    },
+  }).exec();
+  if (exitCode !== 0) {
+    throw new Error("Failed to call tar --version");
+  }
+  // Return whether this is GNU tar or BSD tar, and the version number
+  if (stdout.includes("GNU tar")) {
+    const match = stdout.match(/tar \(GNU tar\) ([0-9.]+)/);
+    if (!match || !match[1]) {
+      throw new Error("Failed to parse output of tar --version.");
+    }
+
+    return { type: "gnu", version: match[1] };
+  } else if (stdout.includes("bsdtar")) {
+    const match = stdout.match(/bsdtar ([0-9.]+)/);
+    if (!match || !match[1]) {
+      throw new Error("Failed to parse output of tar --version.");
+    }
+
+    return { type: "bsd", version: match[1] };
+  } else {
+    throw new Error("Unknown tar version");
+  }
+}
+
+export async function isZstdAvailable(
+  logger: Logger,
+): Promise<{ available: boolean; version?: TarVersion }> {
+  try {
+    const tarVersion = await getTarVersion();
+    const { type, version } = tarVersion;
+    logger.info(`Found ${type} tar version ${version}.`);
+    switch (type) {
+      case "gnu":
+        return {
+          available: version >= MIN_REQUIRED_GNU_TAR_VERSION,
+          version: tarVersion,
+        };
+      case "bsd":
+        return {
+          available: version >= MIN_REQUIRED_BSD_TAR_VERSION,
+          version: tarVersion,
+        };
+      default:
+        assertNever(type);
+    }
+  } catch (e) {
+    logger.error(
+      "Failed to determine tar version, therefore will assume zstd may not be available. " +
+        `The underlying error was: ${e}`,
+    );
+    return { available: false };
+  }
+}
+
+export type CompressionMethod = "gzip" | "zstd";
+
+export async function extract(
+  path: string,
+  compressionMethod: CompressionMethod,
+): Promise<string> {
+  switch (compressionMethod) {
+    case "gzip":
+      // While we could also ask tar to autodetect the compression method,
+      // we defensively keep the gzip call identical as requesting a gzipped
+      // bundle will soon be a fallback option.
+      return await toolcache.extractTar(path);
+    case "zstd":
+      // By specifying only the "x" flag, we ask tar to autodetect the
+      // compression method.
+      return await toolcache.extractTar(path, undefined, "x");
+  }
+}
+
+export function inferCompressionMethod(path: string): CompressionMethod {
+  if (path.endsWith(".tar.gz")) {
+    return "gzip";
+  }
+  return "zstd";
+}