robojerk/codeql-action

TRAP Caching: Add timeouts to upload/download operations

Browse source
This commit is contained in:
Edoardo Pirovano 2022-09-30 10:44:36 +01:00
parent 61b87c69a6
commit c0641ea1d3
No known key found for this signature in database
GPG key ID: 047556B5D93FFE28
9 changed files with 145 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

17
lib/trap-caching.js generated
View file

@ -37,6 +37,11 @@ const CACHE_SIZE_MB = 1024;
// This constant sets the minimum size in megabytes of a TRAP
// cache for us to consider it worth uploading.
const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
// The maximum number of milliseconds to wait for TRAP cache
// uploads or downloads to complete before continuing. Note
// this timeout is per operation, so will be run as many
// times as there are languages with TRAP caching enabled.
const MAX_CACHE_OPERATION_MS = 120000; // Two minutes
async function getTrapCachingExtractorConfigArgs(config) {
const result = [];
for (const language of config.languages)
@ -95,9 +100,11 @@ async function downloadTrapCaches(codeql, languages, logger) {
// The SHA from the base of the PR is the most similar commit we might have a cache for
const preferredKey = await cacheKey(codeql, language, baseSha);
logger.info(`Looking in Actions cache for TRAP cache with key ${preferredKey}`);
const found = await cache.restoreCache([cacheDir], preferredKey, [
const found = await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, cache.restoreCache([cacheDir], preferredKey, [
await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
]);
]), () => {
logger.info(`Timed out waiting for TRAP cache download for ${language}, will continue without it`);
});
if (found === undefined) {
// We didn't find a TRAP cache in the Actions cache, so the directory on disk is
// still just an empty directory. There's no reason to tell the extractor to use it,
@ -119,7 +126,6 @@ exports.downloadTrapCaches = downloadTrapCaches;
async function uploadTrapCaches(codeql, config, logger) {
if (!(await actionsUtil.isAnalyzingDefaultBranch()))
return false; // Only upload caches from the default branch
const toAwait = [];
for (const language of config.languages) {
const cacheDir = config.trapCaches[language];
if (cacheDir === undefined)
@ -135,9 +141,10 @@ async function uploadTrapCaches(codeql, config, logger) {
}
const key = await cacheKey(codeql, language, process.env.GITHUB_SHA || "unknown");
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
toAwait.push(cache.saveCache([cacheDir], key));
await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, cache.saveCache([cacheDir], key), () => {
logger.info(`Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading`);
});
}
await Promise.all(toAwait);
return true;
}
exports.uploadTrapCaches = uploadTrapCaches;

2
lib/trap-caching.js.map
View file

File diff suppressed because one or more lines are too long

21
lib/util.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.tryGetFolderBytes = exports.isGoExtractionReconciliationEnabled = exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.withTimeout = exports.tryGetFolderBytes = exports.isGoExtractionReconciliationEnabled = exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@ -735,4 +735,23 @@ async function tryGetFolderBytes(cacheDir, logger) {
}
}
exports.tryGetFolderBytes = tryGetFolderBytes;
/**
* Run a promise for a given amount of time, and if it doesn't resolve within
* that time, call the provided callback and then return undefined.
*
* @param timeoutMs The timeout in milliseconds.
* @param promise The promise to run.
* @param onTimeout A callback to call if the promise times out.
* @returns The result of the promise, or undefined if the promise times out.
*/
async function withTimeout(timeoutMs, promise, onTimeout) {
const timeout = new Promise((resolve) => {
setTimeout(() => {
onTimeout();
resolve(undefined);
}, timeoutMs);
});
return await Promise.race([promise, timeout]);
}
exports.withTimeout = withTimeout;
//# sourceMappingURL=util.js.map

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

28
lib/util.test.js generated
View file

@ -396,4 +396,32 @@ function mockVersion(version) {
},
};
}
const longTime = 999999;
const shortTime = 10;
(0, ava_1.default)("withTimeout on long task", async (t) => {
let longTaskTimedOut = false;
const longTask = new Promise((resolve) => {
setTimeout(() => {
resolve(42);
}, longTime);
});
const result = await util.withTimeout(shortTime, longTask, () => {
longTaskTimedOut = true;
});
t.deepEqual(longTaskTimedOut, true);
t.deepEqual(result, undefined);
});
(0, ava_1.default)("withTimeout on short task", async (t) => {
let shortTaskTimedOut = false;
const shortTask = new Promise((resolve) => {
setTimeout(() => {
resolve(99);
}, shortTime);
});
const result = await util.withTimeout(longTime, shortTask, () => {
shortTaskTimedOut = true;
});
t.deepEqual(shortTaskTimedOut, false);
t.deepEqual(result, 99);
});
//# sourceMappingURL=util.test.js.map

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long