Merge pull request #1820 from github/henrymercer/delete-bundle-archive

Clean up the bundle archive after extracting it

CHANGELOG.md

@@ -5,6 +5,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
 ## [UNRELEASED]
 
 - We are rolling out a feature in August 2023 that will improve multi-threaded performance on larger runners. [#1817](https://github.com/github/codeql-action/pull/1817)
+- Reduce disk space usage when downloading the CodeQL bundle. [#1820](https://github.com/github/codeql-action/pull/1820)
 
 ## 2.21.2 - 28 Jul 2023
 

lib/setup-codeql.js

@@ -31,6 +31,7 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const perf_hooks_1 = require("perf_hooks");
 const toolcache = __importStar(require("@actions/tool-cache"));
+const del_1 = __importDefault(require("del"));
 const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
 const semver = __importStar(require("semver"));
 const uuid_1 = require("uuid");
@@ -420,17 +421,22 @@ async function downloadCodeQL(codeqlURL, maybeBundleVersion, maybeCliVersion, ap
     const dest = path.join(tempDir, (0, uuid_1.v4)());
     const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
     const toolsDownloadStart = perf_hooks_1.performance.now();
-    const codeqlPath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
+    const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
     const toolsDownloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
-    logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-    const codeqlExtracted = await toolcache.extractTar(codeqlPath);
+    logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${toolsDownloadDurationMs} ms).`);
+    logger.debug("Extracting CodeQL bundle.");
+    const extractionStart = perf_hooks_1.performance.now();
+    const extractedBundlePath = await toolcache.extractTar(archivedBundlePath);
+    const extractionMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
+    logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionMs} ms).`);
+    await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
     const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
     if (bundleVersion === undefined) {
         logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
             `URL ${codeqlURL}.`);
         return {
             toolsVersion: maybeCliVersion ?? "unknown",
-            codeqlFolder: codeqlExtracted,
+            codeqlFolder: extractedBundlePath,
             toolsDownloadDurationMs,
         };
     }
@@ -452,9 +458,15 @@ async function downloadCodeQL(codeqlURL, maybeBundleVersion, maybeCliVersion, ap
     const toolcacheVersion = maybeCliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)
         ? `${maybeCliVersion}-${bundleVersion}`
         : convertToSemVer(bundleVersion, logger);
+    logger.debug("Caching CodeQL bundle.");
+    const toolcachedBundlePath = await toolcache.cacheDir(extractedBundlePath, "CodeQL", toolcacheVersion);
+    // Defensive check: we expect `cacheDir` to copy the bundle to a new location.
+    if (toolcachedBundlePath !== extractedBundlePath) {
+        await cleanUpGlob(extractedBundlePath, "CodeQL bundle from temporary directory", logger);
+    }
     return {
         toolsVersion: maybeCliVersion ?? toolcacheVersion,
-        codeqlFolder: await toolcache.cacheDir(codeqlExtracted, "CodeQL", toolcacheVersion),
+        codeqlFolder: toolcachedBundlePath,
         toolsDownloadDurationMs,
     };
 }
@@ -510,4 +522,22 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau
     return { codeqlFolder, toolsDownloadDurationMs, toolsSource, toolsVersion };
 }
 exports.setupCodeQLBundle = setupCodeQLBundle;
+async function cleanUpGlob(glob, name, logger) {
+    logger.debug(`Cleaning up ${name}.`);
+    try {
+        const deletedPaths = await (0, del_1.default)(glob, { force: true });
+        if (deletedPaths.length === 0) {
+            logger.warning(`Failed to clean up ${name}: no files found matching ${glob}.`);
+        }
+        else if (deletedPaths.length === 1) {
+            logger.debug(`Cleaned up ${name}.`);
+        }
+        else {
+            logger.debug(`Cleaned up ${name} (${deletedPaths.length} files).`);
+        }
+    }
+    catch (e) {
+        logger.warning(`Failed to clean up ${name}: ${e}.`);
+    }
+}
 //# sourceMappingURL=setup-codeql.js.map

src/setup-codeql.ts

@@ -4,6 +4,7 @@ import * as path from "path";
 import { performance } from "perf_hooks";
 
 import * as toolcache from "@actions/tool-cache";
+import del from "del";
 import { default as deepEqual } from "fast-deep-equal";
 import * as semver from "semver";
 import { v4 as uuidV4 } from "uuid";
@@ -559,7 +560,7 @@ export async function downloadCodeQL(
   );
 
   const toolsDownloadStart = performance.now();
-  const codeqlPath = await toolcache.downloadTool(
+  const archivedBundlePath = await toolcache.downloadTool(
     codeqlURL,
     dest,
     authorization,
@@ -569,9 +570,18 @@ export async function downloadCodeQL(
     performance.now() - toolsDownloadStart,
   );
 
-  logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+  logger.debug(
+    `Finished downloading CodeQL bundle to ${archivedBundlePath} (${toolsDownloadDurationMs} ms).`,
+  );
 
-  const codeqlExtracted = await toolcache.extractTar(codeqlPath);
+  logger.debug("Extracting CodeQL bundle.");
+  const extractionStart = performance.now();
+  const extractedBundlePath = await toolcache.extractTar(archivedBundlePath);
+  const extractionMs = Math.round(performance.now() - extractionStart);
+  logger.debug(
+    `Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionMs} ms).`,
+  );
+  await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
 
   const bundleVersion =
     maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
@@ -583,7 +593,7 @@ export async function downloadCodeQL(
     );
     return {
       toolsVersion: maybeCliVersion ?? "unknown",
-      codeqlFolder: codeqlExtracted,
+      codeqlFolder: extractedBundlePath,
       toolsDownloadDurationMs,
     };
   }
@@ -612,13 +622,26 @@ export async function downloadCodeQL(
   const toolcacheVersion = maybeCliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)
     ? `${maybeCliVersion}-${bundleVersion}`
     : convertToSemVer(bundleVersion, logger);
+
+  logger.debug("Caching CodeQL bundle.");
+  const toolcachedBundlePath = await toolcache.cacheDir(
+    extractedBundlePath,
+    "CodeQL",
+    toolcacheVersion,
+  );
+
+  // Defensive check: we expect `cacheDir` to copy the bundle to a new location.
+  if (toolcachedBundlePath !== extractedBundlePath) {
+    await cleanUpGlob(
+      extractedBundlePath,
+      "CodeQL bundle from temporary directory",
+      logger,
+    );
+  }
+
   return {
     toolsVersion: maybeCliVersion ?? toolcacheVersion,
-    codeqlFolder: await toolcache.cacheDir(
-      codeqlExtracted,
-      "CodeQL",
-      toolcacheVersion,
-    ),
+    codeqlFolder: toolcachedBundlePath,
     toolsDownloadDurationMs,
   };
 }
@@ -702,3 +725,21 @@ export async function setupCodeQLBundle(
   }
   return { codeqlFolder, toolsDownloadDurationMs, toolsSource, toolsVersion };
 }
+
+async function cleanUpGlob(glob: string, name: string, logger: Logger) {
+  logger.debug(`Cleaning up ${name}.`);
+  try {
+    const deletedPaths = await del(glob, { force: true });
+    if (deletedPaths.length === 0) {
+      logger.warning(
+        `Failed to clean up ${name}: no files found matching ${glob}.`,
+      );
+    } else if (deletedPaths.length === 1) {
+      logger.debug(`Cleaned up ${name}.`);
+    } else {
+      logger.debug(`Cleaned up ${name} (${deletedPaths.length} files).`);
+    }
+  } catch (e) {
+    logger.warning(`Failed to clean up ${name}: ${e}.`);
+  }
+}