Stream the download and extraction of zstd bundles

Behind a feature flag

lib/codeql.test.js

@@ -101,7 +101,6 @@ function mockApiDetails(apiDetails) {
             t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
             t.is(result.toolsVersion, `0.0.0-${version}`);
             t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
-            t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
         }
         t.is(toolcache.findAllVersions("CodeQL").length, 2);
     });
@@ -118,7 +117,9 @@ function mockApiDetails(apiDetails) {
         t.assert(toolcache.find("CodeQL", `2.14.0`));
         t.is(result.toolsVersion, `2.14.0`);
         t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
-        t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
+        if (result.toolsDownloadStatusReport) {
+            assertDurationsInteger(t, result.toolsDownloadStatusReport);
+        }
     });
 });
 (0, ava_1.default)("downloads an explicitly requested bundle even if a different version is cached", async (t) => {
@@ -136,7 +137,9 @@ function mockApiDetails(apiDetails) {
         t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
         t.deepEqual(result.toolsVersion, "0.0.0-20200610");
         t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
-        t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
+        if (result.toolsDownloadStatusReport) {
+            assertDurationsInteger(t, result.toolsDownloadStatusReport);
+        }
     });
 });
 const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [
@@ -184,7 +187,9 @@ for (const toolcacheVersion of [
             const result = await codeql.setupCodeQL(undefined, testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
             t.is(result.toolsVersion, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
             t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
+            t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
             t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
+            t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
         });
     });
 }
@@ -202,7 +207,9 @@ for (const toolcacheVersion of [
         }, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
         t.deepEqual(result.toolsVersion, "0.0.0-20200601");
         t.is(result.toolsSource, setup_codeql_1.ToolsSource.Toolcache);
+        t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
         t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
+        t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
         const cachedVersions = toolcache.findAllVersions("CodeQL");
         t.is(cachedVersions.length, 1);
     });
@@ -224,7 +231,9 @@ for (const toolcacheVersion of [
         }, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
         t.deepEqual(result.toolsVersion, defaults.cliVersion);
         t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
-        t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
+        if (result.toolsDownloadStatusReport) {
+            assertDurationsInteger(t, result.toolsDownloadStatusReport);
+        }
         const cachedVersions = toolcache.findAllVersions("CodeQL");
         t.is(cachedVersions.length, 2);
     });
@@ -243,7 +252,9 @@ for (const toolcacheVersion of [
         const result = await codeql.setupCodeQL("latest", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
         t.deepEqual(result.toolsVersion, defaults.cliVersion);
         t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
-        t.assert(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
+        if (result.toolsDownloadStatusReport) {
+            assertDurationsInteger(t, result.toolsDownloadStatusReport);
+        }
         const cachedVersions = toolcache.findAllVersions("CodeQL");
         t.is(cachedVersions.length, 2);
     });
@@ -265,13 +276,22 @@ for (const toolcacheVersion of [
         const result = await codeql.setupCodeQL("https://github.com/codeql-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz", testing_utils_1.SAMPLE_DOTCOM_API_DETAILS, tmpDir, util.GitHubVariant.DOTCOM, testing_utils_1.SAMPLE_DEFAULT_CLI_VERSION, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true), false);
         t.is(result.toolsVersion, "0.0.0-20230203");
         t.is(result.toolsSource, setup_codeql_1.ToolsSource.Download);
-        t.true(Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs));
+        if (result.toolsDownloadStatusReport) {
+            assertDurationsInteger(t, result.toolsDownloadStatusReport);
+        }
         const cachedVersions = toolcache.findAllVersions("CodeQL");
         t.is(cachedVersions.length, 1);
         t.is(cachedVersions[0], "0.0.0-20230203");
         t.false(releasesApiMock.isDone());
     });
 });
+function assertDurationsInteger(t, statusReport) {
+    t.assert(Number.isInteger(statusReport?.combinedDurationMs));
+    if (statusReport.downloadDurationMs !== undefined) {
+        t.assert(Number.isInteger(statusReport?.downloadDurationMs));
+        t.assert(Number.isInteger(statusReport?.extractionDurationMs));
+    }
+}
 (0, ava_1.default)("getExtraOptions works for explicit paths", (t) => {
     t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);
     t.deepEqual(codeql.getExtraOptions({ foo: [42] }, ["foo"], []), ["42"]);

lib/feature-flags.js

@@ -57,6 +57,7 @@ var Feature;
     Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
     Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
     Feature["ZstdBundle"] = "zstd_bundle";
+    Feature["ZstdBundleStreamingExtraction"] = "zstd_bundle_streaming_extraction";
 })(Feature || (exports.Feature = Feature = {}));
 exports.featureConfig = {
     [Feature.ArtifactV4Upgrade]: {
@@ -111,6 +112,11 @@ exports.featureConfig = {
         // version check separately.
         minimumVersion: undefined,
     },
+    [Feature.ZstdBundleStreamingExtraction]: {
+        defaultValue: false,
+        envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
+        minimumVersion: undefined,
+    },
 };
 exports.FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
 /**

lib/init-action.js

@@ -58,7 +58,7 @@ async function sendCompletedStatusReport(startedAt, config, configFile, toolsDow
         workflow_languages: workflowLanguages || "",
     };
     const initToolsDownloadFields = {};
-    if (toolsDownloadStatusReport !== undefined) {
+    if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
         initToolsDownloadFields.tools_download_duration_ms =
             toolsDownloadStatusReport.downloadDurationMs;
     }

lib/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AA/MD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAI1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,gCAyCC;AAED,gCAgBC;AAED,0BAkCC;AAED,0DAeC;AAMD,sDAkBC;AAED,0EAkDC;AAhND,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAK1D,qDAAgD;AAChD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,QAA2B,EAC3B,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

lib/setup-codeql.js

@@ -36,19 +36,15 @@ exports.getCodeQLURLVersion = getCodeQLURLVersion;
 exports.setupCodeQLBundle = setupCodeQLBundle;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
-const perf_hooks_1 = require("perf_hooks");
 const toolcache = __importStar(require("@actions/tool-cache"));
 const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
 const semver = __importStar(require("semver"));
-const uuid_1 = require("uuid");
 const actions_util_1 = require("./actions-util");
 const api = __importStar(require("./api-client"));
-// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
-// creation scripts. Ensure that any changes to the format of this file are compatible with both of
-// these dependents.
 const defaults = __importStar(require("./defaults.json"));
 const feature_flags_1 = require("./feature-flags");
 const tar = __importStar(require("./tar"));
+const tools_download_1 = require("./tools-download");
 const util = __importStar(require("./util"));
 const util_1 = require("./util");
 var ToolsSource;
@@ -372,7 +368,7 @@ async function tryGetFallbackToolcacheVersion(cliVersion, tagName, logger) {
 }
 // Exported using `export const` for testing purposes. Specifically, we want to
 // be able to stub this function and have other functions in this file use that stub.
-const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, logger) {
+const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, features, logger) {
     const parsedCodeQLURL = new URL(codeqlURL);
     const searchParams = new URLSearchParams(parsedCodeQLURL.search);
     const headers = {
@@ -395,37 +391,14 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
         logger.debug("Downloading CodeQL tools without an authorization token.");
     }
     logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
-    const compressionMethod = tar.inferCompressionMethod(codeqlURL);
+    const { extractedBundlePath, statusReport } = await (0, tools_download_1.downloadAndExtract)(codeqlURL, authorization, { "User-Agent": "CodeQL Action", ...headers }, tarVersion, tempDir, features, logger);
-    const dest = path.join(tempDir, (0, uuid_1.v4)());
-    const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
-    const toolsDownloadStart = perf_hooks_1.performance.now();
-    const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
-    const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
-    logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
-    let extractedBundlePath;
-    let extractionDurationMs;
-    try {
-        logger.debug("Extracting CodeQL bundle.");
-        const extractionStart = perf_hooks_1.performance.now();
-        extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
-        extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
-        logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
-    }
-    finally {
-        await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
-    }
     const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
     if (bundleVersion === undefined) {
         logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
             `URL ${codeqlURL}.`);
         return {
             codeqlFolder: extractedBundlePath,
-            statusReport: {
+            statusReport,
-                compressionMethod,
-                downloadDurationMs,
-                extractionDurationMs,
-                toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
-            },
             toolsVersion: maybeCliVersion ?? "unknown",
         };
     }
@@ -438,12 +411,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
     }
     return {
         codeqlFolder: toolcachedBundlePath,
-        statusReport: {
+        statusReport,
-            compressionMethod,
-            downloadDurationMs,
-            extractionDurationMs,
-            toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
-        },
         toolsVersion: maybeCliVersion ?? toolcacheVersion,
     };
 };
@@ -529,7 +497,7 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
             toolsSource = ToolsSource.Toolcache;
             break;
         case "download": {
-            const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, logger);
+            const result = await (0, exports.downloadCodeQL)(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, zstdAvailability.version, tempDir, features, logger);
             toolsVersion = result.toolsVersion;
             codeqlFolder = result.codeqlFolder;
             toolsDownloadStatusReport = result.statusReport;
@@ -547,11 +515,6 @@ async function setupCodeQLBundleWithCompressionMethod(toolsInput, apiDetails, te
         zstdAvailability,
     };
 }
-function sanitizeUrlForStatusReport(url) {
-    return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
-        ? url
-        : "sanitized-value";
-}
 async function useZstdBundle(cliVersion, features, tarSupportsZstd) {
     return (
     // In testing, gzip performs better than zstd on Windows.

lib/setup-codeql.test.js

@@ -116,6 +116,7 @@ ava_1.default.beforeEach(() => {
     sinon.stub(setupCodeql, "downloadCodeQL").resolves({
         codeqlFolder: "codeql",
         statusReport: {
+            combinedDurationMs: 500,
             compressionMethod: "gzip",
             downloadDurationMs: 200,
             extractionDurationMs: 300,
@@ -145,6 +146,7 @@ ava_1.default.beforeEach(() => {
     sinon.stub(setupCodeql, "downloadCodeQL").resolves({
         codeqlFolder: "codeql",
         statusReport: {
+            combinedDurationMs: 500,
             compressionMethod: "gzip",
             downloadDurationMs: 200,
             extractionDurationMs: 300,

lib/tools-download.js

@@ -0,0 +1,95 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadAndExtract = downloadAndExtract;
+const http = __importStar(require("http"));
+const path = __importStar(require("path"));
+const perf_hooks_1 = require("perf_hooks");
+const toolcache = __importStar(require("@actions/tool-cache"));
+const uuid_1 = require("uuid");
+const feature_flags_1 = require("./feature-flags");
+const tar = __importStar(require("./tar"));
+const util_1 = require("./util");
+async function downloadAndExtract(codeqlURL, authorization, headers, tarVersion, tempDir, features, logger) {
+    const compressionMethod = tar.inferCompressionMethod(codeqlURL);
+    if (compressionMethod === "zstd" &&
+        (await features.getValue(feature_flags_1.Feature.ZstdBundleStreamingExtraction))) {
+        const toolsInstallStart = perf_hooks_1.performance.now();
+        const extractedBundlePath = await downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger);
+        const combinedDurationMs = Math.round(perf_hooks_1.performance.now() - toolsInstallStart);
+        logger.info(`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${combinedDurationMs} ms).`);
+        return {
+            extractedBundlePath,
+            statusReport: {
+                combinedDurationMs,
+                compressionMethod,
+                downloadDurationMs: undefined,
+                extractionDurationMs: undefined,
+                toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
+            },
+        };
+    }
+    const dest = path.join(tempDir, (0, uuid_1.v4)());
+    const toolsDownloadStart = perf_hooks_1.performance.now();
+    const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, headers);
+    const downloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
+    logger.info(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`);
+    let extractedBundlePath;
+    let extractionDurationMs;
+    try {
+        logger.info("Extracting CodeQL bundle.");
+        const extractionStart = perf_hooks_1.performance.now();
+        extractedBundlePath = await tar.extract(archivedBundlePath, compressionMethod, tarVersion, logger);
+        extractionDurationMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
+        logger.info(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`);
+    }
+    finally {
+        await (0, util_1.cleanUpGlob)(archivedBundlePath, "CodeQL bundle archive", logger);
+    }
+    return {
+        extractedBundlePath,
+        statusReport: {
+            compressionMethod,
+            combinedDurationMs: downloadDurationMs + extractionDurationMs,
+            downloadDurationMs,
+            extractionDurationMs,
+            toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
+        },
+    };
+}
+async function downloadAndExtractZstdWithStreaming(codeqlURL, authorization, headers, tarVersion, logger) {
+    headers = Object.assign({ "User-Agent": "CodeQL Action", authorization }, headers);
+    const response = await new Promise((resolve) => http.get(codeqlURL, { headers }, (r) => resolve(r)));
+    if (response.statusCode !== 200) {
+        throw new Error(`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`);
+    }
+    return await tar.extractTarZst(response, tarVersion, logger);
+}
+function sanitizeUrlForStatusReport(url) {
+    return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some((repo) => url.startsWith(`https://github.com/${repo}/releases/download/`))
+        ? url
+        : "sanitized-value";
+}
+//# sourceMappingURL=tools-download.js.map

lib/tools-download.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA2CA,gDA2FC;AAtID,2CAA6B;AAE7B,2CAA6B;AAC7B,2CAAyC;AAEzC,+DAAiD;AACjD,+BAAoC;AAEpC,mDAA6D;AAE7D,2CAA6B;AAC7B,iCAAqC;AAgC9B,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,OAAe,EACf,QAA2B,EAC3B,MAAc;IAKd,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,IACE,iBAAiB,KAAK,MAAM;QAC5B,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,uBAAO,CAAC,6BAA6B,CAAC,CAAC,EAChE,CAAC;QACD,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,mBAAmB,GAAG,MAAM,mCAAmC,CACnE,SAAS,EACT,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;QACF,MAAM,CAAC,IAAI,CACT,wDAAwD,mBAAmB,KAAK,kBAAkB,OAAO,CAC1G,CAAC;QAEF,OAAO;YACL,mBAAmB;YACnB,YAAY,EAAE;gBACZ,kBAAkB;gBAClB,iBAAiB;gBACjB,kBAAkB,EAAE,SAAS;gBAC7B,oBAAoB,EAAE,SAAS;gBAC/B,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;aAChD;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAA,SAAM,GAAE,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,kBAAkB,OAAO,CAC1F,CAAC;IAEF,IAAI,mBAA2B,CAAC;IAChC,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,mBAAmB,GAAG,MAAM,GAAG,CAAC,OAAO,CACrC,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,mBAAmB,KAAK,oBAAoB,OAAO,CAC5F,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,YAAY,EAAE;YACZ,iBAAiB;YACjB,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;YAC7D,kBAAkB;YAClB,oBAAoB;YACpB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;SAChD;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,EAChD,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAuB,CAAC,OAAO,EAAE,EAAE,CACnE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CACpD,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}

src/codeql.test.ts

@@ -30,6 +30,7 @@ import {
   makeVersionInfo,
   createTestConfig,
 } from "./testing-utils";
+import { ToolsDownloadStatusReport } from "./tools-download";
 import { ToolsFeature } from "./tools-features";
 import * as util from "./util";
 import { initializeEnvironment } from "./util";
@@ -138,9 +139,6 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
       t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
       t.is(result.toolsVersion, `0.0.0-${version}`);
       t.is(result.toolsSource, ToolsSource.Download);
-      t.assert(
-        Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
-      );
     }
 
     t.is(toolcache.findAllVersions("CodeQL").length, 2);
@@ -170,9 +168,9 @@ test("caches semantically versioned bundles using their semantic version number"
     t.assert(toolcache.find("CodeQL", `2.14.0`));
     t.is(result.toolsVersion, `2.14.0`);
     t.is(result.toolsSource, ToolsSource.Download);
-    t.assert(
+    if (result.toolsDownloadStatusReport) {
-      Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
+      assertDurationsInteger(t, result.toolsDownloadStatusReport);
-    );
+    }
   });
 });
 
@@ -203,9 +201,9 @@ test("downloads an explicitly requested bundle even if a different version is ca
     t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
     t.deepEqual(result.toolsVersion, "0.0.0-20200610");
     t.is(result.toolsSource, ToolsSource.Download);
-    t.assert(
+    if (result.toolsDownloadStatusReport) {
-      Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
+      assertDurationsInteger(t, result.toolsDownloadStatusReport);
-    );
+    }
   });
 });
 
@@ -288,7 +286,9 @@ for (const toolcacheVersion of [
         );
         t.is(result.toolsVersion, SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
         t.is(result.toolsSource, ToolsSource.Toolcache);
+        t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
         t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
+        t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
       });
     },
   );
@@ -320,7 +320,9 @@ test(`uses a cached bundle when no tools input is given on GHES`, async (t) => {
     );
     t.deepEqual(result.toolsVersion, "0.0.0-20200601");
     t.is(result.toolsSource, ToolsSource.Toolcache);
+    t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
     t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
+    t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
     t.is(cachedVersions.length, 1);
@@ -356,9 +358,9 @@ test(`downloads bundle if only an unpinned version is cached on GHES`, async (t)
     );
     t.deepEqual(result.toolsVersion, defaults.cliVersion);
     t.is(result.toolsSource, ToolsSource.Download);
-    t.assert(
+    if (result.toolsDownloadStatusReport) {
-      Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
+      assertDurationsInteger(t, result.toolsDownloadStatusReport);
-    );
+    }
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
     t.is(cachedVersions.length, 2);
@@ -391,9 +393,9 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
     );
     t.deepEqual(result.toolsVersion, defaults.cliVersion);
     t.is(result.toolsSource, ToolsSource.Download);
-    t.assert(
+    if (result.toolsDownloadStatusReport) {
-      Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
+      assertDurationsInteger(t, result.toolsDownloadStatusReport);
-    );
+    }
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
     t.is(cachedVersions.length, 2);
@@ -430,9 +432,9 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
 
     t.is(result.toolsVersion, "0.0.0-20230203");
     t.is(result.toolsSource, ToolsSource.Download);
-    t.true(
+    if (result.toolsDownloadStatusReport) {
-      Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
+      assertDurationsInteger(t, result.toolsDownloadStatusReport);
-    );
+    }
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
     t.is(cachedVersions.length, 1);
@@ -442,6 +444,17 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
   });
 });
 
+function assertDurationsInteger(
+  t: ExecutionContext<unknown>,
+  statusReport: ToolsDownloadStatusReport,
+) {
+  t.assert(Number.isInteger(statusReport?.combinedDurationMs));
+  if (statusReport.downloadDurationMs !== undefined) {
+    t.assert(Number.isInteger(statusReport?.downloadDurationMs));
+    t.assert(Number.isInteger(statusReport?.extractionDurationMs));
+  }
+}
+
 test("getExtraOptions works for explicit paths", (t) => {
   t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);
 

src/codeql.ts

@@ -28,6 +28,7 @@ import { Language } from "./languages";
 import { Logger } from "./logging";
 import * as setupCodeql from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
+import { ToolsDownloadStatusReport } from "./tools-download";
 import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
 import { shouldEnableIndirectTracing } from "./tracer-config";
 import * as util from "./util";
@@ -356,7 +357,7 @@ export async function setupCodeQL(
   checkVersion: boolean,
 ): Promise<{
   codeql: CodeQL;
-  toolsDownloadStatusReport?: setupCodeql.ToolsDownloadStatusReport;
+  toolsDownloadStatusReport?: ToolsDownloadStatusReport;
   toolsSource: setupCodeql.ToolsSource;
   toolsVersion: string;
   zstdAvailability: ZstdAvailability;

src/feature-flags.ts

@@ -54,6 +54,7 @@ export enum Feature {
   ExportDiagnosticsEnabled = "export_diagnostics_enabled",
   QaTelemetryEnabled = "qa_telemetry_enabled",
   ZstdBundle = "zstd_bundle",
+  ZstdBundleStreamingExtraction = "zstd_bundle_streaming_extraction",
 }
 
 export const featureConfig: Record<
@@ -139,6 +140,11 @@ export const featureConfig: Record<
     // version check separately.
     minimumVersion: undefined,
   },
+  [Feature.ZstdBundleStreamingExtraction]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
+    minimumVersion: undefined,
+  },
 };
 
 /**

src/init-action.ts

@@ -34,7 +34,7 @@ import {
 import { Language } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
-import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
+import { ToolsSource } from "./setup-codeql";
 import {
   ActionName,
   StatusReportBase,
@@ -43,6 +43,7 @@ import {
   sendStatusReport,
 } from "./status-report";
 import { ZstdAvailability } from "./tar";
+import { ToolsDownloadStatusReport } from "./tools-download";
 import { ToolsFeature } from "./tools-features";
 import { getTotalCacheSize } from "./trap-caching";
 import {
@@ -153,7 +154,7 @@ async function sendCompletedStatusReport(
 
   const initToolsDownloadFields: InitToolsDownloadFields = {};
 
-  if (toolsDownloadStatusReport !== undefined) {
+  if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
     initToolsDownloadFields.tools_download_duration_ms =
       toolsDownloadStatusReport.downloadDurationMs;
   }

src/init.ts

@@ -11,8 +11,9 @@ import * as configUtils from "./config-utils";
 import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
 import { Language, isScannedLanguage } from "./languages";
 import { Logger } from "./logging";
-import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
+import { ToolsSource } from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
+import { ToolsDownloadStatusReport } from "./tools-download";
 import { ToolsFeature } from "./tools-features";
 import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
 import * as util from "./util";

src/setup-codeql.test.ts

@@ -161,6 +161,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
   sinon.stub(setupCodeql, "downloadCodeQL").resolves({
     codeqlFolder: "codeql",
     statusReport: {
+      combinedDurationMs: 500,
       compressionMethod: "gzip",
       downloadDurationMs: 200,
       extractionDurationMs: 300,
@@ -210,6 +211,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
   sinon.stub(setupCodeql, "downloadCodeQL").resolves({
     codeqlFolder: "codeql",
     statusReport: {
+      combinedDurationMs: 500,
       compressionMethod: "gzip",
       downloadDurationMs: 200,
       extractionDurationMs: 300,

src/setup-codeql.ts

@@ -1,18 +1,13 @@
 import * as fs from "fs";
 import { OutgoingHttpHeaders } from "http";
 import * as path from "path";
-import { performance } from "perf_hooks";
 
 import * as toolcache from "@actions/tool-cache";
 import { default as deepEqual } from "fast-deep-equal";
 import * as semver from "semver";
-import { v4 as uuidV4 } from "uuid";
 
 import { CommandInvocationError, isRunningLocalAction } from "./actions-util";
 import * as api from "./api-client";
-// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
-// creation scripts. Ensure that any changes to the format of this file are compatible with both of
-// these dependents.
 import * as defaults from "./defaults.json";
 import {
   CODEQL_VERSION_ZSTD_BUNDLE,
@@ -22,6 +17,10 @@ import {
 } from "./feature-flags";
 import { Logger } from "./logging";
 import * as tar from "./tar";
+import {
+  downloadAndExtract,
+  ToolsDownloadStatusReport,
+} from "./tools-download";
 import * as util from "./util";
 import { cleanUpGlob, isGoodVersion } from "./util";
 
@@ -481,14 +480,6 @@ export async function tryGetFallbackToolcacheVersion(
   return fallbackVersion;
 }
 
-export interface ToolsDownloadStatusReport {
-  compressionMethod: tar.CompressionMethod;
-  downloadDurationMs: number;
-  extractionDurationMs: number;
-  toolsUrl: string;
-  zstdFailureReason?: string;
-}
-
 // Exported using `export const` for testing purposes. Specifically, we want to
 // be able to stub this function and have other functions in this file use that stub.
 export const downloadCodeQL = async function (
@@ -498,6 +489,7 @@ export const downloadCodeQL = async function (
   apiDetails: api.GitHubApiDetails,
   tarVersion: tar.TarVersion | undefined,
   tempDir: string,
+  features: FeatureEnablement,
   logger: Logger,
 ): Promise<{
   codeqlFolder: string;
@@ -529,45 +521,15 @@ export const downloadCodeQL = async function (
     `Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
   );
 
-  const compressionMethod = tar.inferCompressionMethod(codeqlURL);
+  const { extractedBundlePath, statusReport } = await downloadAndExtract(
-  const dest = path.join(tempDir, uuidV4());
-  const finalHeaders = Object.assign(
-    { "User-Agent": "CodeQL Action" },
-    headers,
-  );
-
-  const toolsDownloadStart = performance.now();
-  const archivedBundlePath = await toolcache.downloadTool(
     codeqlURL,
-    dest,
     authorization,
-    finalHeaders,
+    { "User-Agent": "CodeQL Action", ...headers },
+    tarVersion,
+    tempDir,
+    features,
+    logger,
   );
-  const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
-
-  logger.debug(
-    `Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
-  );
-
-  let extractedBundlePath: string;
-  let extractionDurationMs: number;
-
-  try {
-    logger.debug("Extracting CodeQL bundle.");
-    const extractionStart = performance.now();
-    extractedBundlePath = await tar.extract(
-      archivedBundlePath,
-      compressionMethod,
-      tarVersion,
-      logger,
-    );
-    extractionDurationMs = Math.round(performance.now() - extractionStart);
-    logger.debug(
-      `Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
-    );
-  } finally {
-    await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
-  }
 
   const bundleVersion =
     maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
@@ -579,12 +541,7 @@ export const downloadCodeQL = async function (
     );
     return {
       codeqlFolder: extractedBundlePath,
-      statusReport: {
+      statusReport,
-        compressionMethod,
-        downloadDurationMs,
-        extractionDurationMs,
-        toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
-      },
       toolsVersion: maybeCliVersion ?? "unknown",
     };
   }
@@ -612,12 +569,7 @@ export const downloadCodeQL = async function (
 
   return {
     codeqlFolder: toolcachedBundlePath,
-    statusReport: {
+    statusReport,
-      compressionMethod,
-      downloadDurationMs,
-      extractionDurationMs,
-      toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
-    },
     toolsVersion: maybeCliVersion ?? toolcacheVersion,
   };
 };
@@ -789,6 +741,7 @@ async function setupCodeQLBundleWithCompressionMethod(
         apiDetails,
         zstdAvailability.version,
         tempDir,
+        features,
         logger,
       );
       toolsVersion = result.toolsVersion;
@@ -809,14 +762,6 @@ async function setupCodeQLBundleWithCompressionMethod(
   };
 }
 
-function sanitizeUrlForStatusReport(url: string): string {
-  return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
-    (repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
-  )
-    ? url
-    : "sanitized-value";
-}
-
 async function useZstdBundle(
   cliVersion: string,
   features: FeatureEnablement,

src/tools-download.ts

@@ -0,0 +1,167 @@
+import * as http from "http";
+import { OutgoingHttpHeaders } from "http";
+import * as path from "path";
+import { performance } from "perf_hooks";
+
+import * as toolcache from "@actions/tool-cache";
+import { v4 as uuidV4 } from "uuid";
+
+import { Feature, FeatureEnablement } from "./feature-flags";
+import { Logger } from "./logging";
+import * as tar from "./tar";
+import { cleanUpGlob } from "./util";
+
+/**
+ * Timing information for the download and extraction of the CodeQL tools when
+ * we fully download the bundle before extracting.
+ */
+type DownloadFirstToolsDownloadDurations = {
+  combinedDurationMs: number;
+  downloadDurationMs: number;
+  extractionDurationMs: number;
+};
+
+/**
+ * Timing information for the download and extraction of the CodeQL tools when
+ * we stream the download and extraction of the bundle.
+ */
+type StreamedToolsDownloadDurations = {
+  combinedDurationMs: number;
+  downloadDurationMs: undefined;
+  extractionDurationMs: undefined;
+};
+
+type ToolsDownloadDurations =
+  | DownloadFirstToolsDownloadDurations
+  | StreamedToolsDownloadDurations;
+
+export type ToolsDownloadStatusReport = {
+  compressionMethod: tar.CompressionMethod;
+  toolsUrl: string;
+  zstdFailureReason?: string;
+} & ToolsDownloadDurations;
+
+export async function downloadAndExtract(
+  codeqlURL: string,
+  authorization: string | undefined,
+  headers: OutgoingHttpHeaders,
+  tarVersion: tar.TarVersion | undefined,
+  tempDir: string,
+  features: FeatureEnablement,
+  logger: Logger,
+): Promise<{
+  extractedBundlePath: string;
+  statusReport: ToolsDownloadStatusReport;
+}> {
+  const compressionMethod = tar.inferCompressionMethod(codeqlURL);
+
+  if (
+    compressionMethod === "zstd" &&
+    (await features.getValue(Feature.ZstdBundleStreamingExtraction))
+  ) {
+    const toolsInstallStart = performance.now();
+    const extractedBundlePath = await downloadAndExtractZstdWithStreaming(
+      codeqlURL,
+      authorization,
+      headers,
+      tarVersion!,
+      logger,
+    );
+
+    const combinedDurationMs = Math.round(
+      performance.now() - toolsInstallStart,
+    );
+    logger.info(
+      `Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${combinedDurationMs} ms).`,
+    );
+
+    return {
+      extractedBundlePath,
+      statusReport: {
+        combinedDurationMs,
+        compressionMethod,
+        downloadDurationMs: undefined,
+        extractionDurationMs: undefined,
+        toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
+      },
+    };
+  }
+
+  const dest = path.join(tempDir, uuidV4());
+
+  const toolsDownloadStart = performance.now();
+  const archivedBundlePath = await toolcache.downloadTool(
+    codeqlURL,
+    dest,
+    authorization,
+    headers,
+  );
+  const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
+
+  logger.info(
+    `Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
+  );
+
+  let extractedBundlePath: string;
+  let extractionDurationMs: number;
+
+  try {
+    logger.info("Extracting CodeQL bundle.");
+    const extractionStart = performance.now();
+    extractedBundlePath = await tar.extract(
+      archivedBundlePath,
+      compressionMethod,
+      tarVersion,
+      logger,
+    );
+    extractionDurationMs = Math.round(performance.now() - extractionStart);
+    logger.info(
+      `Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
+    );
+  } finally {
+    await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
+  }
+
+  return {
+    extractedBundlePath,
+    statusReport: {
+      compressionMethod,
+      combinedDurationMs: downloadDurationMs + extractionDurationMs,
+      downloadDurationMs,
+      extractionDurationMs,
+      toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
+    },
+  };
+}
+
+async function downloadAndExtractZstdWithStreaming(
+  codeqlURL: string,
+  authorization: string | undefined,
+  headers: OutgoingHttpHeaders,
+  tarVersion: tar.TarVersion,
+  logger: Logger,
+): Promise<string> {
+  headers = Object.assign(
+    { "User-Agent": "CodeQL Action", authorization },
+    headers,
+  );
+  const response = await new Promise<http.IncomingMessage>((resolve) =>
+    http.get(codeqlURL, { headers }, (r) => resolve(r)),
+  );
+
+  if (response.statusCode !== 200) {
+    throw new Error(
+      `Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
+    );
+  }
+
+  return await tar.extractTarZst(response, tarVersion, logger);
+}
+
+function sanitizeUrlForStatusReport(url: string): string {
+  return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
+    (repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
+  )
+    ? url
+    : "sanitized-value";
+}