TRAP Caching: Skip uploading of small caches

lib/trap-caching.js

@@ -18,19 +18,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getTotalCacheSize = exports.getLanguagesSupportingCaching = exports.uploadTrapCaches = exports.downloadTrapCaches = exports.getTrapCachingExtractorConfigArgsForLang = exports.getTrapCachingExtractorConfigArgs = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
-const util_1 = require("util");
 const cache = __importStar(require("@actions/cache"));
-const get_folder_size_1 = __importDefault(require("get-folder-size"));
 const actionsUtil = __importStar(require("./actions-util"));
 const codeql_1 = require("./codeql");
-const util_2 = require("./util");
+const util_1 = require("./util");
 // This constant should be bumped if we make a breaking change
 // to how the CodeQL Action stores or retrieves the TRAP cache,
 // and will invalidate previous caches. We don't need to bump
@@ -39,6 +34,9 @@ const util_2 = require("./util");
 const CACHE_VERSION = 1;
 // This constant sets the size of each TRAP cache in megabytes.
 const CACHE_SIZE_MB = 1024;
+// This constant sets the minimum size in megabytes of a TRAP
+// cache for us to consider it worth uploading.
+const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
 async function getTrapCachingExtractorConfigArgs(config) {
     const result = [];
     for (const language of config.languages)
@@ -126,6 +124,15 @@ async function uploadTrapCaches(codeql, config, logger) {
         const cacheDir = config.trapCaches[language];
         if (cacheDir === undefined)
             continue;
+        const trapFolderSize = await (0, util_1.tryGetFolderBytes)(cacheDir, logger);
+        if (trapFolderSize === undefined) {
+            logger.info(`Skipping upload of TRAP cache for ${language} as we couldn't determine its size`);
+            continue;
+        }
+        if (trapFolderSize < MINIMUM_CACHE_MB_TO_UPLOAD * 1048576) {
+            logger.info(`Skipping upload of TRAP cache for ${language} as it is too small`);
+            continue;
+        }
         const key = await cacheKey(codeql, language, process.env.GITHUB_SHA || "unknown");
         logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
         toAwait.push(cache.saveCache([cacheDir], key));
@@ -137,7 +144,7 @@ exports.uploadTrapCaches = uploadTrapCaches;
 async function getLanguagesSupportingCaching(codeql, languages, logger) {
     var _a, _b, _c, _d;
     const result = [];
-    if (!(await (0, util_2.codeQlVersionAbove)(codeql, codeql_1.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES)))
+    if (!(await (0, util_1.codeQlVersionAbove)(codeql, codeql_1.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES)))
         return result;
     const resolveResult = await codeql.betterResolveLanguages();
     outer: for (const lang of languages) {
@@ -168,16 +175,8 @@ async function getLanguagesSupportingCaching(codeql, languages, logger) {
 }
 exports.getLanguagesSupportingCaching = getLanguagesSupportingCaching;
 async function getTotalCacheSize(trapCaches, logger) {
-    try {
+    const sizes = await Promise.all(Object.values(trapCaches).map((cacheDir) => (0, util_1.tryGetFolderBytes)(cacheDir, logger)));
-        const sizes = await Promise.all(Object.values(trapCaches).map(async (cacheDir) => {
+    return sizes.map((a) => a || 0).reduce((a, b) => a + b, 0);
-            return (0, util_1.promisify)(get_folder_size_1.default)(cacheDir);
-        }));
-        return sizes.reduce((a, b) => a + b, 0);
-    }
-    catch (e) {
-        logger.warning(`Encountered an error while getting TRAP cache size: ${e}`);
-        return 0;
-    }
 }
 exports.getTotalCacheSize = getTotalCacheSize;
 async function cacheKey(codeql, language, baseSha) {

lib/trap-caching.test.js

@@ -164,6 +164,7 @@ function getTestConfigWithTempDir(tmpDir) {
     const loggedMessages = [];
     const logger = (0, testing_utils_1.getRecordingLogger)(loggedMessages);
     sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+    sinon.stub(util, "tryGetFolderBytes").resolves(999999999);
     const stubSave = sinon.stub(cache, "saveCache");
     process.env.GITHUB_SHA = "somesha";
     await (0, trap_caching_1.uploadTrapCaches)(stubCodeql, testConfigWithoutTmpDir, logger);

lib/util.js

@@ -22,12 +22,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isGoExtractionReconciliationEnabled = exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
+exports.tryGetFolderBytes = exports.isGoExtractionReconciliationEnabled = exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
+const util_1 = require("util");
 const core = __importStar(require("@actions/core"));
 const del_1 = __importDefault(require("del"));
+const get_folder_size_1 = __importDefault(require("get-folder-size"));
 const semver = __importStar(require("semver"));
 const api = __importStar(require("./api-client"));
 const api_client_1 = require("./api-client");
@@ -700,4 +702,22 @@ async function isGoExtractionReconciliationEnabled(featureFlags) {
         (await featureFlags.getValue(feature_flags_1.FeatureFlag.GolangExtractionReconciliationEnabled)));
 }
 exports.isGoExtractionReconciliationEnabled = isGoExtractionReconciliationEnabled;
+/**
+ * Get the size a folder in bytes. This will log any filesystem errors
+ * as a warning and then return undefined.
+ *
+ * @param cacheDir A directory to get the size of.
+ * @param logger A logger to log any errors to.
+ * @returns The size in bytes of the folder, or undefined if errors occurred.
+ */
+async function tryGetFolderBytes(cacheDir, logger) {
+    try {
+        return await (0, util_1.promisify)(get_folder_size_1.default)(cacheDir);
+    }
+    catch (e) {
+        logger.warning(`Encountered an error while getting size of folder: ${e}`);
+        return undefined;
+    }
+}
+exports.tryGetFolderBytes = tryGetFolderBytes;
 //# sourceMappingURL=util.js.map

src/trap-caching.test.ts

@@ -165,6 +165,7 @@ test("upload cache key contains right fields", async (t) => {
   const loggedMessages = [];
   const logger = getRecordingLogger(loggedMessages);
   sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+  sinon.stub(util, "tryGetFolderBytes").resolves(999_999_999);
   const stubSave = sinon.stub(cache, "saveCache");
   process.env.GITHUB_SHA = "somesha";
   await uploadTrapCaches(stubCodeql, testConfigWithoutTmpDir, logger);

src/trap-caching.ts

@@ -1,16 +1,14 @@
 import * as fs from "fs";
 import * as path from "path";
-import { promisify } from "util";
 
 import * as cache from "@actions/cache";
-import getFolderSize from "get-folder-size";
 
 import * as actionsUtil from "./actions-util";
 import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql";
 import { Config } from "./config-utils";
 import { Language } from "./languages";
 import { Logger } from "./logging";
-import { codeQlVersionAbove } from "./util";
+import { codeQlVersionAbove, tryGetFolderBytes } from "./util";
 
 // This constant should be bumped if we make a breaking change
 // to how the CodeQL Action stores or retrieves the TRAP cache,
@@ -22,6 +20,10 @@ const CACHE_VERSION = 1;
 // This constant sets the size of each TRAP cache in megabytes.
 const CACHE_SIZE_MB = 1024;
 
+// This constant sets the minimum size in megabytes of a TRAP
+// cache for us to consider it worth uploading.
+const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
+
 export async function getTrapCachingExtractorConfigArgs(
   config: Config
 ): Promise<string[]> {
@@ -138,6 +140,19 @@ export async function uploadTrapCaches(
   for (const language of config.languages) {
     const cacheDir = config.trapCaches[language];
     if (cacheDir === undefined) continue;
+    const trapFolderSize = await tryGetFolderBytes(cacheDir, logger);
+    if (trapFolderSize === undefined) {
+      logger.info(
+        `Skipping upload of TRAP cache for ${language} as we couldn't determine its size`
+      );
+      continue;
+    }
+    if (trapFolderSize < MINIMUM_CACHE_MB_TO_UPLOAD * 1_048_576) {
+      logger.info(
+        `Skipping upload of TRAP cache for ${language} as it is too small`
+      );
+      continue;
+    }
     const key = await cacheKey(
       codeql,
       language,
@@ -201,17 +216,12 @@ export async function getTotalCacheSize(
   trapCaches: Partial<Record<Language, string>>,
   logger: Logger
 ): Promise<number> {
-  try {
+  const sizes = await Promise.all(
-    const sizes = await Promise.all(
+    Object.values(trapCaches).map((cacheDir) =>
-      Object.values(trapCaches).map(async (cacheDir) => {
+      tryGetFolderBytes(cacheDir, logger)
-        return promisify<string, number>(getFolderSize)(cacheDir);
+    )
-      })
+  );
-    );
+  return sizes.map((a) => a || 0).reduce((a, b) => a + b, 0);
-    return sizes.reduce((a, b) => a + b, 0);
-  } catch (e) {
-    logger.warning(`Encountered an error while getting TRAP cache size: ${e}`);
-    return 0;
-  }
 }
 
 async function cacheKey(

src/util.ts

@@ -2,9 +2,11 @@ import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
 import { Readable } from "stream";
+import { promisify } from "util";
 
 import * as core from "@actions/core";
 import del from "del";
+import getFolderSize from "get-folder-size";
 import * as semver from "semver";
 
 import * as api from "./api-client";
@@ -836,3 +838,23 @@ export async function isGoExtractionReconciliationEnabled(
     ))
   );
 }
+
+/**
+ * Get the size a folder in bytes. This will log any filesystem errors
+ * as a warning and then return undefined.
+ *
+ * @param cacheDir A directory to get the size of.
+ * @param logger A logger to log any errors to.
+ * @returns The size in bytes of the folder, or undefined if errors occurred.
+ */
+export async function tryGetFolderBytes(
+  cacheDir: string,
+  logger: Logger
+): Promise<number | undefined> {
+  try {
+    return await promisify<string, number>(getFolderSize)(cacheDir);
+  } catch (e) {
+    logger.warning(`Encountered an error while getting size of folder: ${e}`);
+    return undefined;
+  }
+}