robojerk/codeql-action

Merge pull request #2306 from github/henrymercer/trap-cache-space-efficiency

Browse source 
Automatically clean up old TRAP caches
This commit is contained in:
Henry Mercer 2024-05-24 15:51:20 +01:00 • committed by GitHub
commit 584871bb84
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 459 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -6,7 +6,7 @@ Note that the only difference between `v2` and `v3` of the CodeQL Action is the
## [UNRELEASED]
No user facing changes.
- We are rolling out a feature in May/June 2024 that will reduce the Actions cache usage of the Action by keeping only the newest TRAP cache for each language. [#2306](https://github.com/github/codeql-action/pull/2306)
## 3.25.6 - 20 May 2024

3
lib/actions-util.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getFileType = exports.FileCmdNotFoundError = exports.getWorkflowRunAttempt = exports.getWorkflowRunID = exports.getUploadValue = exports.printDebugLogs = exports.isAnalyzingDefaultBranch = exports.getRelativeScriptPath = exports.isRunningLocalAction = exports.getWorkflowEventName = exports.getActionVersion = exports.getRef = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getTemporaryDirectory = exports.getOptionalInput = exports.getRequiredInput = void 0;
exports.getFileType = exports.FileCmdNotFoundError = exports.getWorkflowRunAttempt = exports.getWorkflowRunID = exports.getUploadValue = exports.printDebugLogs = exports.isAnalyzingDefaultBranch = exports.getWorkflowEvent = exports.getRelativeScriptPath = exports.isRunningLocalAction = exports.getWorkflowEventName = exports.getActionVersion = exports.getRef = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getTemporaryDirectory = exports.getOptionalInput = exports.getRequiredInput = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const core = __importStar(require("@actions/core"));
@ -275,6 +275,7 @@ function getWorkflowEvent() {
throw new Error(`Unable to read workflow event JSON from ${eventJsonFile}: ${e}`);
}
}
exports.getWorkflowEvent = getWorkflowEvent;
function removeRefsHeadsPrefix(ref) {
return ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) : ref;
}

2
lib/actions-util.js.map
View file

File diff suppressed because one or more lines are too long

16
lib/analyze-action.js generated
View file

@ -48,7 +48,7 @@ const status_report_1 = require("./status-report");
const trap_caching_1 = require("./trap-caching");
const uploadLib = __importStar(require("./upload-lib"));
const util = __importStar(require("./util"));
async function sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger) {
async function sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, logger) {
const status = (0, status_report_1.getActionsStatus)(error, stats?.analyze_failure_language);
const statusReportBase = await (0, status_report_1.createStatusReportBase)(status_report_1.ActionName.Analyze, status, startedAt, config, await util.checkDiskUsage(), logger, error?.message, error?.stack);
if (statusReportBase !== undefined) {
@ -56,6 +56,7 @@ async function sendStatusReport(startedAt, config, stats, error, trapCacheUpload
...statusReportBase,
...(stats || {}),
...(dbCreationTimings || {}),
...(trapCacheCleanup || {}),
};
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport = {
@ -141,6 +142,7 @@ async function run() {
let uploadResult = undefined;
let runStats = undefined;
let config = undefined;
let trapCacheCleanupTelemetry = undefined;
let trapCacheUploadTime = undefined;
let dbCreationTimings = undefined;
let didUploadTrapCaches = false;
@ -196,6 +198,8 @@ async function run() {
const trapCacheUploadStartTime = perf_hooks_1.performance.now();
didUploadTrapCaches = await (0, trap_caching_1.uploadTrapCaches)(codeql, config, logger);
trapCacheUploadTime = perf_hooks_1.performance.now() - trapCacheUploadStartTime;
// Clean up TRAP caches
trapCacheCleanupTelemetry = await (0, trap_caching_1.cleanupTrapCaches)(config, features, logger);
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
logger.debug("In test mode. Waiting for processing is disabled.");
@ -218,10 +222,10 @@ async function run() {
}
if (error instanceof analyze_1.CodeQLAnalysisError) {
const stats = { ...error.queriesStatusReport };
await sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
else {
await sendStatusReport(startedAt, config, undefined, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, undefined, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
return;
}
@ -229,13 +233,13 @@ async function run() {
await sendStatusReport(startedAt, config, {
...runStats,
...uploadResult.statusReport,
}, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
}, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
else if (runStats) {
await sendStatusReport(startedAt, config, { ...runStats }, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, { ...runStats }, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
else {
await sendStatusReport(startedAt, config, undefined, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, undefined, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
}
exports.runPromise = run();

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

24
lib/api-client.js generated
View file

@ -26,12 +26,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.wrapApiConfigurationError = exports.computeAutomationID = exports.getAutomationID = exports.getAnalysisKey = exports.getWorkflowRelativePath = exports.getGitHubVersion = exports.getGitHubVersionFromApi = exports.getApiClientWithExternalAuth = exports.getApiClient = exports.getApiDetails = exports.DisallowedAPIVersionReason = void 0;
exports.wrapApiConfigurationError = exports.deleteActionsCache = exports.listActionsCaches = exports.computeAutomationID = exports.getAutomationID = exports.getAnalysisKey = exports.getWorkflowRelativePath = exports.getGitHubVersion = exports.getGitHubVersionFromApi = exports.getApiClientWithExternalAuth = exports.getApiClient = exports.getApiDetails = exports.DisallowedAPIVersionReason = void 0;
const core = __importStar(require("@actions/core"));
const githubUtils = __importStar(require("@actions/github/lib/utils"));
const retry = __importStar(require("@octokit/plugin-retry"));
const console_log_level_1 = __importDefault(require("console-log-level"));
const actions_util_1 = require("./actions-util");
const repository_1 = require("./repository");
const util_1 = require("./util");
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
var DisallowedAPIVersionReason;
@ -163,6 +164,27 @@ function computeAutomationID(analysis_key, environment) {
return automationID;
}
exports.computeAutomationID = computeAutomationID;
/** List all Actions cache entries matching the provided key and ref. */
async function listActionsCaches(key, ref) {
const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
return await getApiClient().paginate("GET /repos/{owner}/{repo}/actions/caches", {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
key,
ref,
});
}
exports.listActionsCaches = listActionsCaches;
/** Delete an Actions cache item by its ID. */
async function deleteActionsCache(id) {
const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
await getApiClient().rest.actions.deleteActionsCacheById({
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
cache_id: id,
});
}
exports.deleteActionsCache = deleteActionsCache;
function wrapApiConfigurationError(e) {
if ((0, util_1.isHTTPError)(e)) {
if (e.message.includes("API rate limit exceeded for site ID installation") ||

2
lib/api-client.js.map
View file

@ -1 +1 @@
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AACtC,uEAAyD;AACzD,6DAA+C;AAC/C,0EAAgD;AAEhD,iDAAoE;AACpE,iCASgB;AAEhB,MAAM,gCAAgC,GAAG,6BAA6B,CAAC;AAEvE,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,0CAA1B,0BAA0B,QAGrC;AAiBD,SAAS,0BAA0B,CACjC,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,IAAI,eAAe,CACxB,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,UAAU,CAAC,MAAM;QAC1B,SAAS,EAAE,iBAAiB,IAAA,+BAAgB,GAAE,EAAE;QAChD,GAAG,EAAE,IAAA,2BAAe,EAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa;IAC3B,OAAO;QACL,IAAI,EAAE,IAAA,+BAAgB,EAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;QAC7C,MAAM,EAAE,IAAA,0BAAmB,EAAC,gBAAgB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAND,sCAMC;AAED,SAAgB,YAAY;IAC1B,OAAO,0BAA0B,CAAC,aAAa,EAAE,CAAC,CAAC;AACrD,CAAC;AAFD,oCAEC;AAED,SAAgB,4BAA4B,CAC1C,UAAoC;IAEpC,OAAO,0BAA0B,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;AACzE,CAAC;AAJD,oEAIC;AAED,IAAI,mBAAmB,GAA8B,SAAS,CAAC;AAExD,KAAK,UAAU,uBAAuB,CAC3C,SAAc,EACd,UAA4B;IAE5B,iEAAiE;IACjE,IAAI,IAAA,qBAAc,EAAC,UAAU,CAAC,GAAG,CAAC,KAAK,wBAAiB,EAAE,CAAC;QACzD,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,8DAA8D;IAC9D,mEAAmE;IACnE,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjD,8EAA8E;IAC9E,wEAAwE;IACxE,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAW,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAzBD,0DAyBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,gBAAgB;IACpC,IAAI,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACtC,mBAAmB,GAAG,MAAM,uBAAuB,CACjD,YAAY,EAAE,EACd,aAAa,EAAE,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AARD,4CAQC;AAED;;GAEG;AACI,KAAK,UAAU,uBAAuB;IAC3C,MAAM,QAAQ,GAAG,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAA,0BAAmB,EAAC,eAAe,CAAC,CAAC,CAAC;IAE5D,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,OAAO,CAC1C,yEAAyE,EACzE;QACE,KAAK;QACL,IAAI;QACJ,MAAM;KACP,CACF,CAAC;IACF,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC;IAEnD,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC;IAEvE,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;AACpC,CAAC;AApBD,0DAoBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc;IAClC,MAAM,iBAAiB,GAAG,4BAA4B,CAAC;IAEvD,IAAI,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,uBAAuB,EAAE,CAAC;IACrD,MAAM,OAAO,GAAG,IAAA,0BAAmB,EAAC,YAAY,CAAC,CAAC;IAElD,WAAW,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;IACpD,OAAO,WAAW,CAAC;AACrB,CAAC;AAdD,wCAcC;AAEM,KAAK,UAAU,eAAe;IACnC,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAA,+BAAgB,EAAC,QAAQ,CAAC,CAAC;IAE/C,OAAO,mBAAmB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;AACxD,CAAC;AALD,0CAKC;AAED,SAAgB,mBAAmB,CACjC,YAAoB,EACpB,WAA+B;IAE/B,IAAI,YAAY,GAAG,GAAG,YAAY,GAAG,CAAC;IAEtC,MAAM,MAAM,GAAG,IAAA,uBAAgB,EAAC,WAAW,CAAC,CAAC;IAC7C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,uDAAuD;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAClD,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACjC,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,qDAAqD;gBACrD,6CAA6C;gBAC7C,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AArBD,kDAqBC;AAED,SAAgB,yBAAyB,CAAC,CAAU;IAClD,IAAI,IAAA,kBAAW,EAAC,CAAC,CAAC,EAAE,CAAC;QACnB,IACE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kDAAkD,CAAC;YACtE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACtC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EACvD,CAAC;YACD,OAAO,IAAI,yBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAXD,8DAWC"}
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AACtC,uEAAyD;AACzD,6DAA+C;AAC/C,0EAAgD;AAEhD,iDAAoE;AACpE,6CAAkD;AAClD,iCASgB;AAEhB,MAAM,gCAAgC,GAAG,6BAA6B,CAAC;AAEvE,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,0CAA1B,0BAA0B,QAGrC;AAiBD,SAAS,0BAA0B,CACjC,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,IAAI,eAAe,CACxB,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,UAAU,CAAC,MAAM;QAC1B,SAAS,EAAE,iBAAiB,IAAA,+BAAgB,GAAE,EAAE;QAChD,GAAG,EAAE,IAAA,2BAAe,EAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa;IAC3B,OAAO;QACL,IAAI,EAAE,IAAA,+BAAgB,EAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;QAC7C,MAAM,EAAE,IAAA,0BAAmB,EAAC,gBAAgB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAND,sCAMC;AAED,SAAgB,YAAY;IAC1B,OAAO,0BAA0B,CAAC,aAAa,EAAE,CAAC,CAAC;AACrD,CAAC;AAFD,oCAEC;AAED,SAAgB,4BAA4B,CAC1C,UAAoC;IAEpC,OAAO,0BAA0B,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;AACzE,CAAC;AAJD,oEAIC;AAED,IAAI,mBAAmB,GAA8B,SAAS,CAAC;AAExD,KAAK,UAAU,uBAAuB,CAC3C,SAAc,EACd,UAA4B;IAE5B,iEAAiE;IACjE,IAAI,IAAA,qBAAc,EAAC,UAAU,CAAC,GAAG,CAAC,KAAK,wBAAiB,EAAE,CAAC;QACzD,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,8DAA8D;IAC9D,mEAAmE;IACnE,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjD,8EAA8E;IAC9E,wEAAwE;IACxE,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAW,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAzBD,0DAyBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,gBAAgB;IACpC,IAAI,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACtC,mBAAmB,GAAG,MAAM,uBAAuB,CACjD,YAAY,EAAE,EACd,aAAa,EAAE,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AARD,4CAQC;AAED;;GAEG;AACI,KAAK,UAAU,uBAAuB;IAC3C,MAAM,QAAQ,GAAG,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAA,0BAAmB,EAAC,eAAe,CAAC,CAAC,CAAC;IAE5D,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,OAAO,CAC1C,yEAAyE,EACzE;QACE,KAAK;QACL,IAAI;QACJ,MAAM;KACP,CACF,CAAC;IACF,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC;IAEnD,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC;IAEvE,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;AACpC,CAAC;AApBD,0DAoBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc;IAClC,MAAM,iBAAiB,GAAG,4BAA4B,CAAC;IAEvD,IAAI,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,uBAAuB,EAAE,CAAC;IACrD,MAAM,OAAO,GAAG,IAAA,0BAAmB,EAAC,YAAY,CAAC,CAAC;IAElD,WAAW,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;IACpD,OAAO,WAAW,CAAC;AACrB,CAAC;AAdD,wCAcC;AAEM,KAAK,UAAU,eAAe;IACnC,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAA,+BAAgB,EAAC,QAAQ,CAAC,CAAC;IAE/C,OAAO,mBAAmB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;AACxD,CAAC;AALD,0CAKC;AAED,SAAgB,mBAAmB,CACjC,YAAoB,EACpB,WAA+B;IAE/B,IAAI,YAAY,GAAG,GAAG,YAAY,GAAG,CAAC;IAEtC,MAAM,MAAM,GAAG,IAAA,uBAAgB,EAAC,WAAW,CAAC,CAAC;IAC7C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,uDAAuD;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAClD,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACjC,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,qDAAqD;gBACrD,6CAA6C;gBAC7C,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AArBD,kDAqBC;AASD,wEAAwE;AACjE,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,GAAW;IAEX,MAAM,aAAa,GAAG,IAAA,+BAAkB,EACtC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CACzC,CAAC;IAEF,OAAO,MAAM,YAAY,EAAE,CAAC,QAAQ,CAClC,0CAA0C,EAC1C;QACE,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,GAAG;QACH,GAAG;KACJ,CACF,CAAC;AACJ,CAAC;AAjBD,8CAiBC;AAED,8CAA8C;AACvC,KAAK,UAAU,kBAAkB,CAAC,EAAU;IACjD,MAAM,aAAa,GAAG,IAAA,+BAAkB,EACtC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CACzC,CAAC;IAEF,MAAM,YAAY,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC;QACvD,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;AACL,CAAC;AAVD,gDAUC;AAED,SAAgB,yBAAyB,CAAC,CAAU;IAClD,IAAI,IAAA,kBAAW,EAAC,CAAC,CAAC,EAAE,CAAC;QACnB,IACE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kDAAkD,CAAC;YACtE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACtC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EACvD,CAAC;YACD,OAAO,IAAI,yBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAXD,8DAWC"}

6
lib/feature-flags.js generated
View file

@ -50,6 +50,7 @@ exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = "2.15.1";
var Feature;
(function (Feature) {
Feature["AutobuildDirectTracing"] = "autobuild_direct_tracing";
Feature["CleanupTrapCaches"] = "cleanup_trap_caches";
Feature["CppDependencyInstallation"] = "cpp_dependency_installation_enabled";
Feature["CppTrapCachingEnabled"] = "cpp_trap_caching_enabled";
Feature["DisableJavaBuildlessEnabled"] = "disable_java_buildless_enabled";
@ -64,6 +65,11 @@ exports.featureConfig = {
minimumVersion: undefined,
toolsFeature: tools_features_1.ToolsFeature.TraceCommandUseBuildMode,
},
[Feature.CleanupTrapCaches]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
minimumVersion: undefined,
},
[Feature.CppDependencyInstallation]: {
defaultValue: false,
envVar: "CODEQL_EXTRACTOR_CPP_AUTOINSTALL_DEPENDENCIES",

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

76
lib/trap-caching.js generated
View file

@ -23,11 +23,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getTotalCacheSize = exports.getLanguagesSupportingCaching = exports.uploadTrapCaches = exports.downloadTrapCaches = void 0;
exports.getTotalCacheSize = exports.getLanguagesSupportingCaching = exports.cleanupTrapCaches = exports.uploadTrapCaches = exports.downloadTrapCaches = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const cache = __importStar(require("@actions/cache"));
const actionsCache = __importStar(require("@actions/cache"));
const actionsUtil = __importStar(require("./actions-util"));
const apiClient = __importStar(require("./api-client"));
const feature_flags_1 = require("./feature-flags");
const util_1 = require("./util");
// This constant should be bumped if we make a breaking change
// to how the CodeQL Action stores or retrieves the TRAP cache,
@ -35,6 +37,7 @@ const util_1 = require("./util");
// this for CLI/extractor changes, since the CLI version also
// goes into the cache key.
const CACHE_VERSION = 1;
const CODEQL_TRAP_CACHE_PREFIX = "codeql-trap";
// This constant sets the minimum size in megabytes of a TRAP
// cache for us to consider it worth uploading.
const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
@ -81,7 +84,7 @@ async function downloadTrapCaches(codeql, languages, logger) {
// The SHA from the base of the PR is the most similar commit we might have a cache for
const preferredKey = await cacheKey(codeql, language, baseSha);
logger.info(`Looking in Actions cache for TRAP cache with key ${preferredKey}`);
const found = await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, cache.restoreCache([cacheDir], preferredKey, [
const found = await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, actionsCache.restoreCache([cacheDir], preferredKey, [
// Fall back to any cache with the right key prefix
await cachePrefix(codeql, language),
]), () => {
@ -123,13 +126,76 @@ async function uploadTrapCaches(codeql, config, logger) {
}
const key = await cacheKey(codeql, language, process.env.GITHUB_SHA || "unknown");
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, cache.saveCache([cacheDir], key), () => {
await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, actionsCache.saveCache([cacheDir], key), () => {
logger.info(`Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading`);
});
}
return true;
}
exports.uploadTrapCaches = uploadTrapCaches;
async function cleanupTrapCaches(config, features, logger) {
if (!(await features.getValue(feature_flags_1.Feature.CleanupTrapCaches))) {
return {
trap_cache_cleanup_skipped_because: "feature disabled",
};
}
if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
return {
trap_cache_cleanup_skipped_because: "not analyzing default branch",
};
}
try {
let totalBytesCleanedUp = 0;
const allCaches = await apiClient.listActionsCaches(CODEQL_TRAP_CACHE_PREFIX, await actionsUtil.getRef());
for (const language of config.languages) {
if (config.trapCaches[language]) {
const cachesToRemove = await getTrapCachesForLanguage(allCaches, language, logger);
// Dates returned by the API are in ISO 8601 format, so we can sort them lexicographically
cachesToRemove.sort((a, b) => a.created_at.localeCompare(b.created_at));
// Keep the most recent cache
const mostRecentCache = cachesToRemove.pop();
logger.debug(`Keeping most recent TRAP cache (${JSON.stringify(mostRecentCache)})`);
if (cachesToRemove.length === 0) {
logger.info(`No TRAP caches to clean up for ${language}.`);
continue;
}
for (const cache of cachesToRemove) {
logger.debug(`Cleaning up TRAP cache (${JSON.stringify(cache)})`);
await apiClient.deleteActionsCache(cache.id);
}
const bytesCleanedUp = cachesToRemove.reduce((acc, item) => acc + item.size_in_bytes, 0);
totalBytesCleanedUp += bytesCleanedUp;
const megabytesCleanedUp = (bytesCleanedUp / (1024 * 1024)).toFixed(2);
logger.info(`Cleaned up ${megabytesCleanedUp} MiB of old TRAP caches for ${language}.`);
}
}
return { trap_cache_cleanup_size_bytes: totalBytesCleanedUp };
}
catch (e) {
if ((0, util_1.isHTTPError)(e) && e.status === 403) {
logger.warning("Could not cleanup TRAP caches as the token did not have the required permissions. " +
'To clean up TRAP caches, ensure the token has the "actions:write" permission. ' +
"For more information, see https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs");
}
else {
logger.info(`Failed to cleanup TRAP caches, continuing. Details: ${e}`);
}
return { trap_cache_cleanup_error: (0, util_1.wrapError)(e).message };
}
}
exports.cleanupTrapCaches = cleanupTrapCaches;
async function getTrapCachesForLanguage(allCaches, language, logger) {
logger.debug(`Listing TRAP caches for ${language}`);
for (const cache of allCaches) {
if (!cache.created_at || !cache.id || !cache.key || !cache.size_in_bytes) {
throw new Error("An unexpected cache item was returned from the API that was missing one or " +
`more required fields: ${JSON.stringify(cache)}`);
}
}
return allCaches.filter((cache) => {
return cache.key?.includes(`-${language}-`);
});
}
async function getLanguagesSupportingCaching(codeql, languages, logger) {
const result = [];
const resolveResult = await codeql.betterResolveLanguages();
@ -169,6 +235,6 @@ async function cacheKey(codeql, language, baseSha) {
return `${await cachePrefix(codeql, language)}${baseSha}`;
}
async function cachePrefix(codeql, language) {
return `codeql-trap-${CACHE_VERSION}-${(await codeql.getVersion()).version}-${language}-`;
return `${CODEQL_TRAP_CACHE_PREFIX}-${CACHE_VERSION}-${(await codeql.getVersion()).version}-${language}-`;
}
//# sourceMappingURL=trap-caching.js.map

2
lib/trap-caching.js.map
View file

File diff suppressed because one or more lines are too long

73
lib/trap-caching.test.js generated
View file

@ -32,8 +32,11 @@ const cache = __importStar(require("@actions/cache"));
const ava_1 = __importDefault(require("ava"));
const sinon = __importStar(require("sinon"));
const actionsUtil = __importStar(require("./actions-util"));
const apiClient = __importStar(require("./api-client"));
const codeql_1 = require("./codeql");
const feature_flags_1 = require("./feature-flags");
const languages_1 = require("./languages");
const logging_1 = require("./logging");
const testing_utils_1 = require("./testing-utils");
const trap_caching_1 = require("./trap-caching");
const util = __importStar(require("./util"));
@ -168,4 +171,74 @@ function getTestConfigWithTempDir(tempDir) {
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
});
});
(0, ava_1.default)("cleanup removes only old CodeQL TRAP caches", async (t) => {
await util.withTmpDir(async (tmpDir) => {
// This config specifies that we are analyzing JavaScript and Ruby, but not Swift.
const config = getTestConfigWithTempDir(tmpDir);
sinon.stub(actionsUtil, "getRef").resolves("refs/heads/main");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const listStub = sinon.stub(apiClient, "listActionsCaches").resolves([
// Should be kept, since it's not relevant to CodeQL. In reality, the API shouldn't return
// this in the first place, but this is a defensive check.
{
id: 1,
key: "some-other-key",
created_at: "2024-05-23T14:25:00Z",
size_in_bytes: 100 * 1024 * 1024,
},
// Should be kept, since it's the newest TRAP cache for JavaScript
{
id: 2,
key: "codeql-trap-1-2.0.0-javascript-newest",
created_at: "2024-04-23T14:25:00Z",
size_in_bytes: 50 * 1024 * 1024,
},
// Should be cleaned up
{
id: 3,
key: "codeql-trap-1-2.0.0-javascript-older",
created_at: "2024-03-22T14:25:00Z",
size_in_bytes: 200 * 1024 * 1024,
},
// Should be cleaned up
{
id: 4,
key: "codeql-trap-1-2.0.0-javascript-oldest",
created_at: "2024-02-21T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
// Should be kept, since it's the newest TRAP cache for Ruby
{
id: 5,
key: "codeql-trap-1-2.0.0-ruby-newest",
created_at: "2024-02-20T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
// Should be kept, since we aren't analyzing Swift
{
id: 6,
key: "codeql-trap-1-2.0.0-swift-newest",
created_at: "2024-02-22T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
// Should be kept, since we aren't analyzing Swift
{
id: 7,
key: "codeql-trap-1-2.0.0-swift-older",
created_at: "2024-02-21T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
]);
const deleteStub = sinon.stub(apiClient, "deleteActionsCache").resolves();
const statusReport = await (0, trap_caching_1.cleanupTrapCaches)(config, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CleanupTrapCaches]), (0, logging_1.getRunnerLogger)(true));
t.is(listStub.callCount, 1);
t.assert(listStub.calledWithExactly("codeql-trap", "refs/heads/main"));
t.deepEqual(statusReport, {
trap_cache_cleanup_size_bytes: 500 * 1024 * 1024,
});
t.is(deleteStub.callCount, 2);
t.assert(deleteStub.calledWithExactly(3));
t.assert(deleteStub.calledWithExactly(4));
});
});
//# sourceMappingURL=trap-caching.test.js.map

2
lib/trap-caching.test.js.map
View file

File diff suppressed because one or more lines are too long

2
src/actions-util.ts
View file

@ -291,7 +291,7 @@ export function getRelativeScriptPath(): string {
}
/** Returns the contents of `GITHUB_EVENT_PATH` as a JSON object. */
function getWorkflowEvent(): any {
export function getWorkflowEvent(): any {
const eventJsonFile = getRequiredEnvParam("GITHUB_EVENT_PATH");
try {
return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));

23
src/analyze-action.ts
View file

@ -32,7 +32,12 @@ import {
getActionsStatus,
StatusReportBase,
} from "./status-report";
import { getTotalCacheSize, uploadTrapCaches } from "./trap-caching";
import {
cleanupTrapCaches,
getTotalCacheSize,
TrapCacheCleanupStatusReport,
uploadTrapCaches,
} from "./trap-caching";
import * as uploadLib from "./upload-lib";
import { UploadResult } from "./upload-lib";
import * as util from "./util";
@ -61,6 +66,7 @@ async function sendStatusReport(
trapCacheUploadTime: number | undefined,
dbCreationTimings: DatabaseCreationTimings | undefined,
didUploadTrapCaches: boolean,
trapCacheCleanup: TrapCacheCleanupStatusReport | undefined,
logger: Logger,
) {
const status = getActionsStatus(error, stats?.analyze_failure_language);
@ -79,6 +85,7 @@ async function sendStatusReport(
...statusReportBase,
...(stats || {}),
...(dbCreationTimings || {}),
...(trapCacheCleanup || {}),
};
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = {
@ -189,6 +196,8 @@ async function run() {
let uploadResult: UploadResult | undefined = undefined;
let runStats: QueriesStatusReport | undefined = undefined;
let config: Config | undefined = undefined;
let trapCacheCleanupTelemetry: TrapCacheCleanupStatusReport | undefined =
undefined;
let trapCacheUploadTime: number | undefined = undefined;
let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
let didUploadTrapCaches = false;
@ -311,6 +320,13 @@ async function run() {
didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger);
trapCacheUploadTime = performance.now() - trapCacheUploadStartTime;
// Clean up TRAP caches
trapCacheCleanupTelemetry = await cleanupTrapCaches(
config,
features,
logger,
);
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
logger.debug("In test mode. Waiting for processing is disabled.");
@ -350,6 +366,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
} else {
@ -361,6 +378,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
}
@ -380,6 +398,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
} else if (runStats) {
@ -391,6 +410,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
} else {
@ -402,6 +422,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
}

41
src/api-client.ts
View file

@ -4,6 +4,7 @@ import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getActionVersion, getRequiredInput } from "./actions-util";
import { parseRepositoryNwo } from "./repository";
import {
ConfigurationError,
getRequiredEnvParam,
@ -195,6 +196,46 @@ export function computeAutomationID(
return automationID;
}
export interface ActionsCacheItem {
created_at?: string;
id?: number;
key?: string;
size_in_bytes?: number;
}
/** List all Actions cache entries matching the provided key and ref. */
export async function listActionsCaches(
key: string,
ref: string,
): Promise<ActionsCacheItem[]> {
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
return await getApiClient().paginate(
"GET /repos/{owner}/{repo}/actions/caches",
{
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
key,
ref,
},
);
}
/** Delete an Actions cache item by its ID. */
export async function deleteActionsCache(id: number) {
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
await getApiClient().rest.actions.deleteActionsCacheById({
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
cache_id: id,
});
}
export function wrapApiConfigurationError(e: unknown) {
if (isHTTPError(e)) {
if (

2
src/diagnostics.ts
View file

@ -1,7 +1,7 @@
import { existsSync, mkdirSync, writeFileSync } from "fs";
import path from "path";
import { Config } from "./config-utils";
import type { Config } from "./config-utils";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { getCodeQLDatabasePath } from "./util";

6
src/feature-flags.ts
View file

@ -46,6 +46,7 @@ export interface FeatureEnablement {
*/
export enum Feature {
AutobuildDirectTracing = "autobuild_direct_tracing",
CleanupTrapCaches = "cleanup_trap_caches",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
CppTrapCachingEnabled = "cpp_trap_caching_enabled",
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
@ -91,6 +92,11 @@ export const featureConfig: Record<
minimumVersion: undefined,
toolsFeature: ToolsFeature.TraceCommandUseBuildMode,
},
[Feature.CleanupTrapCaches]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
minimumVersion: undefined,
},
[Feature.CppDependencyInstallation]: {
defaultValue: false,
envVar: "CODEQL_EXTRACTOR_CPP_AUTOINSTALL_DEPENDENCIES",

86
src/trap-caching.test.ts
View file

@ -6,20 +6,25 @@ import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as apiClient from "./api-client";
import {
setCodeQL,
getTrapCachingExtractorConfigArgs,
getTrapCachingExtractorConfigArgsForLang,
} from "./codeql";
import * as configUtils from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import {
createFeatures,
createTestConfig,
getRecordingLogger,
makeVersionInfo,
setupTests,
} from "./testing-utils";
import {
cleanupTrapCaches,
downloadTrapCaches,
getLanguagesSupportingCaching,
uploadTrapCaches,
@ -189,3 +194,84 @@ test("download cache looks for the right key and creates dir", async (t) => {
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
});
});
test("cleanup removes only old CodeQL TRAP caches", async (t) => {
await util.withTmpDir(async (tmpDir) => {
// This config specifies that we are analyzing JavaScript and Ruby, but not Swift.
const config = getTestConfigWithTempDir(tmpDir);
sinon.stub(actionsUtil, "getRef").resolves("refs/heads/main");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const listStub = sinon.stub(apiClient, "listActionsCaches").resolves([
// Should be kept, since it's not relevant to CodeQL. In reality, the API shouldn't return
// this in the first place, but this is a defensive check.
{
id: 1,
key: "some-other-key",
created_at: "2024-05-23T14:25:00Z",
size_in_bytes: 100 * 1024 * 1024,
},
// Should be kept, since it's the newest TRAP cache for JavaScript
{
id: 2,
key: "codeql-trap-1-2.0.0-javascript-newest",
created_at: "2024-04-23T14:25:00Z",
size_in_bytes: 50 * 1024 * 1024,
},
// Should be cleaned up
{
id: 3,
key: "codeql-trap-1-2.0.0-javascript-older",
created_at: "2024-03-22T14:25:00Z",
size_in_bytes: 200 * 1024 * 1024,
},
// Should be cleaned up
{
id: 4,
key: "codeql-trap-1-2.0.0-javascript-oldest",
created_at: "2024-02-21T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
// Should be kept, since it's the newest TRAP cache for Ruby
{
id: 5,
key: "codeql-trap-1-2.0.0-ruby-newest",
created_at: "2024-02-20T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
// Should be kept, since we aren't analyzing Swift
{
id: 6,
key: "codeql-trap-1-2.0.0-swift-newest",
created_at: "2024-02-22T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
// Should be kept, since we aren't analyzing Swift
{
id: 7,
key: "codeql-trap-1-2.0.0-swift-older",
created_at: "2024-02-21T14:25:00Z",
size_in_bytes: 300 * 1024 * 1024,
},
]);
const deleteStub = sinon.stub(apiClient, "deleteActionsCache").resolves();
const statusReport = await cleanupTrapCaches(
config,
createFeatures([Feature.CleanupTrapCaches]),
getRunnerLogger(true),
);
t.is(listStub.callCount, 1);
t.assert(listStub.calledWithExactly("codeql-trap", "refs/heads/main"));
t.deepEqual(statusReport, {
trap_cache_cleanup_size_bytes: 500 * 1024 * 1024,
});
t.is(deleteStub.callCount, 2);
t.assert(deleteStub.calledWithExactly(3));
t.assert(deleteStub.calledWithExactly(4));
});
});

115
src/trap-caching.ts
View file

@ -1,14 +1,16 @@
import * as fs from "fs";
import * as path from "path";
import * as cache from "@actions/cache";
import * as actionsCache from "@actions/cache";
import * as actionsUtil from "./actions-util";
import * as apiClient from "./api-client";
import { CodeQL } from "./codeql";
import type { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { Logger } from "./logging";
import { tryGetFolderBytes, withTimeout } from "./util";
import { isHTTPError, tryGetFolderBytes, withTimeout, wrapError } from "./util";
// This constant should be bumped if we make a breaking change
// to how the CodeQL Action stores or retrieves the TRAP cache,
@ -17,6 +19,8 @@ import { tryGetFolderBytes, withTimeout } from "./util";
// goes into the cache key.
const CACHE_VERSION = 1;
const CODEQL_TRAP_CACHE_PREFIX = "codeql-trap";
// This constant sets the minimum size in megabytes of a TRAP
// cache for us to consider it worth uploading.
const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
@ -87,7 +91,7 @@ export async function downloadTrapCaches(
);
const found = await withTimeout(
MAX_CACHE_OPERATION_MS,
cache.restoreCache([cacheDir], preferredKey, [
actionsCache.restoreCache([cacheDir], preferredKey, [
// Fall back to any cache with the right key prefix
await cachePrefix(codeql, language),
]),
@ -147,7 +151,7 @@ export async function uploadTrapCaches(
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
await withTimeout(
MAX_CACHE_OPERATION_MS,
cache.saveCache([cacheDir], key),
actionsCache.saveCache([cacheDir], key),
() => {
logger.info(
`Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading`,
@ -158,6 +162,107 @@ export async function uploadTrapCaches(
return true;
}
export interface TrapCacheCleanupStatusReport {
trap_cache_cleanup_error?: string;
trap_cache_cleanup_size_bytes?: number;
trap_cache_cleanup_skipped_because?: string;
}
export async function cleanupTrapCaches(
config: Config,
features: FeatureEnablement,
logger: Logger,
): Promise<TrapCacheCleanupStatusReport> {
if (!(await features.getValue(Feature.CleanupTrapCaches))) {
return {
trap_cache_cleanup_skipped_because: "feature disabled",
};
}
if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
return {
trap_cache_cleanup_skipped_because: "not analyzing default branch",
};
}
try {
let totalBytesCleanedUp = 0;
const allCaches = await apiClient.listActionsCaches(
CODEQL_TRAP_CACHE_PREFIX,
await actionsUtil.getRef(),
);
for (const language of config.languages) {
if (config.trapCaches[language]) {
const cachesToRemove = await getTrapCachesForLanguage(
allCaches,
language,
logger,
);
// Dates returned by the API are in ISO 8601 format, so we can sort them lexicographically
cachesToRemove.sort((a, b) => a.created_at.localeCompare(b.created_at));
// Keep the most recent cache
const mostRecentCache = cachesToRemove.pop();
logger.debug(
`Keeping most recent TRAP cache (${JSON.stringify(mostRecentCache)})`,
);
if (cachesToRemove.length === 0) {
logger.info(`No TRAP caches to clean up for ${language}.`);
continue;
}
for (const cache of cachesToRemove) {
logger.debug(`Cleaning up TRAP cache (${JSON.stringify(cache)})`);
await apiClient.deleteActionsCache(cache.id);
}
const bytesCleanedUp = cachesToRemove.reduce(
(acc, item) => acc + item.size_in_bytes,
0,
);
totalBytesCleanedUp += bytesCleanedUp;
const megabytesCleanedUp = (bytesCleanedUp / (1024 * 1024)).toFixed(2);
logger.info(
`Cleaned up ${megabytesCleanedUp} MiB of old TRAP caches for ${language}.`,
);
}
}
return { trap_cache_cleanup_size_bytes: totalBytesCleanedUp };
} catch (e) {
if (isHTTPError(e) && e.status === 403) {
logger.warning(
"Could not cleanup TRAP caches as the token did not have the required permissions. " +
'To clean up TRAP caches, ensure the token has the "actions:write" permission. ' +
"For more information, see https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs",
);
} else {
logger.info(`Failed to cleanup TRAP caches, continuing. Details: ${e}`);
}
return { trap_cache_cleanup_error: wrapError(e).message };
}
}
async function getTrapCachesForLanguage(
allCaches: apiClient.ActionsCacheItem[],
language: Language,
logger: Logger,
): Promise<Array<Required<apiClient.ActionsCacheItem>>> {
logger.debug(`Listing TRAP caches for ${language}`);
for (const cache of allCaches) {
if (!cache.created_at || !cache.id || !cache.key || !cache.size_in_bytes) {
throw new Error(
"An unexpected cache item was returned from the API that was missing one or " +
`more required fields: ${JSON.stringify(cache)}`,
);
}
}
return allCaches.filter((cache) => {
return cache.key?.includes(`-${language}-`);
}) as Array<Required<apiClient.ActionsCacheItem>>;
}
export async function getLanguagesSupportingCaching(
codeql: CodeQL,
languages: Language[],
@ -225,7 +330,7 @@ async function cachePrefix(
codeql: CodeQL,
language: Language,
): Promise<string> {
return `codeql-trap-${CACHE_VERSION}-${
return `${CODEQL_TRAP_CACHE_PREFIX}-${CACHE_VERSION}-${
(await codeql.getVersion()).version
}-${language}-`;
}