Merge pull request #2306 from github/henrymercer/trap-cache-space-efficiency
Automatically clean up old TRAP caches
This commit is contained in:
Henry Mercer
GitHub
commit
584871bb84
20 changed files with 459 additions and 28 deletions
3
lib/actions-util.js
generated
3
lib/actions-util.js
generated
|
|
@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getFileType = exports.FileCmdNotFoundError = exports.getWorkflowRunAttempt = exports.getWorkflowRunID = exports.getUploadValue = exports.printDebugLogs = exports.isAnalyzingDefaultBranch = exports.getRelativeScriptPath = exports.isRunningLocalAction = exports.getWorkflowEventName = exports.getActionVersion = exports.getRef = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getTemporaryDirectory = exports.getOptionalInput = exports.getRequiredInput = void 0;
|
||||
exports.getFileType = exports.FileCmdNotFoundError = exports.getWorkflowRunAttempt = exports.getWorkflowRunID = exports.getUploadValue = exports.printDebugLogs = exports.isAnalyzingDefaultBranch = exports.getWorkflowEvent = exports.getRelativeScriptPath = exports.isRunningLocalAction = exports.getWorkflowEventName = exports.getActionVersion = exports.getRef = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getTemporaryDirectory = exports.getOptionalInput = exports.getRequiredInput = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const core = __importStar(require("@actions/core"));
|
||||
|
|
@ -275,6 +275,7 @@ function getWorkflowEvent() {
|
|||
throw new Error(`Unable to read workflow event JSON from ${eventJsonFile}: ${e}`);
|
||||
}
|
||||
}
|
||||
exports.getWorkflowEvent = getWorkflowEvent;
|
||||
function removeRefsHeadsPrefix(ref) {
|
||||
return ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) : ref;
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
16
lib/analyze-action.js
generated
16
lib/analyze-action.js
generated
|
|
@ -48,7 +48,7 @@ const status_report_1 = require("./status-report");
|
|||
const trap_caching_1 = require("./trap-caching");
|
||||
const uploadLib = __importStar(require("./upload-lib"));
|
||||
const util = __importStar(require("./util"));
|
||||
async function sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger) {
|
||||
async function sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, logger) {
|
||||
const status = (0, status_report_1.getActionsStatus)(error, stats?.analyze_failure_language);
|
||||
const statusReportBase = await (0, status_report_1.createStatusReportBase)(status_report_1.ActionName.Analyze, status, startedAt, config, await util.checkDiskUsage(), logger, error?.message, error?.stack);
|
||||
if (statusReportBase !== undefined) {
|
||||
|
|
@ -56,6 +56,7 @@ async function sendStatusReport(startedAt, config, stats, error, trapCacheUpload
|
|||
...statusReportBase,
|
||||
...(stats || {}),
|
||||
...(dbCreationTimings || {}),
|
||||
...(trapCacheCleanup || {}),
|
||||
};
|
||||
if (config && didUploadTrapCaches) {
|
||||
const trapCacheUploadStatusReport = {
|
||||
|
|
@ -141,6 +142,7 @@ async function run() {
|
|||
let uploadResult = undefined;
|
||||
let runStats = undefined;
|
||||
let config = undefined;
|
||||
let trapCacheCleanupTelemetry = undefined;
|
||||
let trapCacheUploadTime = undefined;
|
||||
let dbCreationTimings = undefined;
|
||||
let didUploadTrapCaches = false;
|
||||
|
|
@ -196,6 +198,8 @@ async function run() {
|
|||
const trapCacheUploadStartTime = perf_hooks_1.performance.now();
|
||||
didUploadTrapCaches = await (0, trap_caching_1.uploadTrapCaches)(codeql, config, logger);
|
||||
trapCacheUploadTime = perf_hooks_1.performance.now() - trapCacheUploadStartTime;
|
||||
// Clean up TRAP caches
|
||||
trapCacheCleanupTelemetry = await (0, trap_caching_1.cleanupTrapCaches)(config, features, logger);
|
||||
// We don't upload results in test mode, so don't wait for processing
|
||||
if (util.isInTestMode()) {
|
||||
logger.debug("In test mode. Waiting for processing is disabled.");
|
||||
|
|
@ -218,10 +222,10 @@ async function run() {
|
|||
}
|
||||
if (error instanceof analyze_1.CodeQLAnalysisError) {
|
||||
const stats = { ...error.queriesStatusReport };
|
||||
await sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
|
||||
await sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
|
||||
}
|
||||
else {
|
||||
await sendStatusReport(startedAt, config, undefined, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
|
||||
await sendStatusReport(startedAt, config, undefined, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
|
@ -229,13 +233,13 @@ async function run() {
|
|||
await sendStatusReport(startedAt, config, {
|
||||
...runStats,
|
||||
...uploadResult.statusReport,
|
||||
}, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
|
||||
}, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
|
||||
}
|
||||
else if (runStats) {
|
||||
await sendStatusReport(startedAt, config, { ...runStats }, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
|
||||
await sendStatusReport(startedAt, config, { ...runStats }, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
|
||||
}
|
||||
else {
|
||||
await sendStatusReport(startedAt, config, undefined, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
|
||||
await sendStatusReport(startedAt, config, undefined, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
|
||||
}
|
||||
}
|
||||
exports.runPromise = run();
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
24
lib/api-client.js
generated
24
lib/api-client.js
generated
|
|
@ -26,12 +26,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.wrapApiConfigurationError = exports.computeAutomationID = exports.getAutomationID = exports.getAnalysisKey = exports.getWorkflowRelativePath = exports.getGitHubVersion = exports.getGitHubVersionFromApi = exports.getApiClientWithExternalAuth = exports.getApiClient = exports.getApiDetails = exports.DisallowedAPIVersionReason = void 0;
|
||||
exports.wrapApiConfigurationError = exports.deleteActionsCache = exports.listActionsCaches = exports.computeAutomationID = exports.getAutomationID = exports.getAnalysisKey = exports.getWorkflowRelativePath = exports.getGitHubVersion = exports.getGitHubVersionFromApi = exports.getApiClientWithExternalAuth = exports.getApiClient = exports.getApiDetails = exports.DisallowedAPIVersionReason = void 0;
|
||||
const core = __importStar(require("@actions/core"));
|
||||
const githubUtils = __importStar(require("@actions/github/lib/utils"));
|
||||
const retry = __importStar(require("@octokit/plugin-retry"));
|
||||
const console_log_level_1 = __importDefault(require("console-log-level"));
|
||||
const actions_util_1 = require("./actions-util");
|
||||
const repository_1 = require("./repository");
|
||||
const util_1 = require("./util");
|
||||
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
|
||||
var DisallowedAPIVersionReason;
|
||||
|
|
@ -163,6 +164,27 @@ function computeAutomationID(analysis_key, environment) {
|
|||
return automationID;
|
||||
}
|
||||
exports.computeAutomationID = computeAutomationID;
|
||||
/** List all Actions cache entries matching the provided key and ref. */
|
||||
async function listActionsCaches(key, ref) {
|
||||
const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
|
||||
return await getApiClient().paginate("GET /repos/{owner}/{repo}/actions/caches", {
|
||||
owner: repositoryNwo.owner,
|
||||
repo: repositoryNwo.repo,
|
||||
key,
|
||||
ref,
|
||||
});
|
||||
}
|
||||
exports.listActionsCaches = listActionsCaches;
|
||||
/** Delete an Actions cache item by its ID. */
|
||||
async function deleteActionsCache(id) {
|
||||
const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
|
||||
await getApiClient().rest.actions.deleteActionsCacheById({
|
||||
owner: repositoryNwo.owner,
|
||||
repo: repositoryNwo.repo,
|
||||
cache_id: id,
|
||||
});
|
||||
}
|
||||
exports.deleteActionsCache = deleteActionsCache;
|
||||
function wrapApiConfigurationError(e) {
|
||||
if ((0, util_1.isHTTPError)(e)) {
|
||||
if (e.message.includes("API rate limit exceeded for site ID installation") ||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AACtC,uEAAyD;AACzD,6DAA+C;AAC/C,0EAAgD;AAEhD,iDAAoE;AACpE,iCASgB;AAEhB,MAAM,gCAAgC,GAAG,6BAA6B,CAAC;AAEvE,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,0CAA1B,0BAA0B,QAGrC;AAiBD,SAAS,0BAA0B,CACjC,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,IAAI,eAAe,CACxB,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,UAAU,CAAC,MAAM;QAC1B,SAAS,EAAE,iBAAiB,IAAA,+BAAgB,GAAE,EAAE;QAChD,GAAG,EAAE,IAAA,2BAAe,EAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa;IAC3B,OAAO;QACL,IAAI,EAAE,IAAA,+BAAgB,EAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;QAC7C,MAAM,EAAE,IAAA,0BAAmB,EAAC,gBAAgB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAND,sCAMC;AAED,SAAgB,YAAY;IAC1B,OAAO,0BAA0B,CAAC,aAAa,EAAE,CAAC,CAAC;AACrD,CAAC;AAFD,oCAEC;AAED,SAAgB,4BAA4B,CAC1C,UAAoC;IAEpC,OAAO,0BAA0B,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;AACzE,CAAC;AAJD,oEAIC;AAED,IAAI,mBAAmB,GAA8B,SAAS,CAAC;AAExD,KAAK,UAAU,uBAAuB,CAC3C,SAAc,EACd,UAA4B;IAE5B,iEAAiE;IACjE,IAAI,IAAA,qBAAc,EAAC,UAAU,CAAC,GAAG,CAAC,KAAK,wBAAiB,EAAE,CAAC;QACzD,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,8DAA8D;IAC9D,mEAAmE;IACnE,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjD,8EAA8E;IAC9E,wEAAwE;IACxE,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAW,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAzBD,0DAyBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,gBAAgB;IACpC,IAAI,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACtC,mBAAmB,GAAG,MAAM,uBAAuB,CACjD,YAAY,EAAE,EACd,aAAa,EAAE,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AARD,4CAQC;AAED;;GAEG;AACI,KAAK,UAAU,uBAAuB;IAC3C,MAAM,QAAQ,GAAG,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAA,0BAAmB,EAAC,eAAe,CAAC,CAAC,CAAC;IAE5D,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,OAAO,CAC1C,yEAAyE,EACzE;QACE,KAAK;QACL,IAAI;QACJ,MAAM;KACP,CACF,CAAC;IACF,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC;IAEnD,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC;IAEvE,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;AACpC,CAAC;AApBD,0DAoBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc;IAClC,MAAM,iBAAiB,GAAG,4BAA4B,CAAC;IAEvD,IAAI,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,uBAAuB,EAAE,CAAC;IACrD,MAAM,OAAO,GAAG,IAAA,0BAAmB,EAAC,YAAY,CAAC,CAAC;IAElD,WAAW,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;IACpD,OAAO,WAAW,CAAC;AACrB,CAAC;AAdD,wCAcC;AAEM,KAAK,UAAU,eAAe;IACnC,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAA,+BAAgB,EAAC,QAAQ,CAAC,CAAC;IAE/C,OAAO,mBAAmB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;AACxD,CAAC;AALD,0CAKC;AAED,SAAgB,mBAAmB,CACjC,YAAoB,EACpB,WAA+B;IAE/B,IAAI,YAAY,GAAG,GAAG,YAAY,GAAG,CAAC;IAEtC,MAAM,MAAM,GAAG,IAAA,uBAAgB,EAAC,WAAW,CAAC,CAAC;IAC7C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,uDAAuD;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAClD,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACjC,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,qDAAqD;gBACrD,6CAA6C;gBAC7C,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AArBD,kDAqBC;AAED,SAAgB,yBAAyB,CAAC,CAAU;IAClD,IAAI,IAAA,kBAAW,EAAC,CAAC,CAAC,EAAE,CAAC;QACnB,IACE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kDAAkD,CAAC;YACtE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACtC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EACvD,CAAC;YACD,OAAO,IAAI,yBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAXD,8DAWC"}
|
||||
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AACtC,uEAAyD;AACzD,6DAA+C;AAC/C,0EAAgD;AAEhD,iDAAoE;AACpE,6CAAkD;AAClD,iCASgB;AAEhB,MAAM,gCAAgC,GAAG,6BAA6B,CAAC;AAEvE,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,0CAA1B,0BAA0B,QAGrC;AAiBD,SAAS,0BAA0B,CACjC,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,IAAI,eAAe,CACxB,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,UAAU,CAAC,MAAM;QAC1B,SAAS,EAAE,iBAAiB,IAAA,+BAAgB,GAAE,EAAE;QAChD,GAAG,EAAE,IAAA,2BAAe,EAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa;IAC3B,OAAO;QACL,IAAI,EAAE,IAAA,+BAAgB,EAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;QAC7C,MAAM,EAAE,IAAA,0BAAmB,EAAC,gBAAgB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAND,sCAMC;AAED,SAAgB,YAAY;IAC1B,OAAO,0BAA0B,CAAC,aAAa,EAAE,CAAC,CAAC;AACrD,CAAC;AAFD,oCAEC;AAED,SAAgB,4BAA4B,CAC1C,UAAoC;IAEpC,OAAO,0BAA0B,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;AACzE,CAAC;AAJD,oEAIC;AAED,IAAI,mBAAmB,GAA8B,SAAS,CAAC;AAExD,KAAK,UAAU,uBAAuB,CAC3C,SAAc,EACd,UAA4B;IAE5B,iEAAiE;IACjE,IAAI,IAAA,qBAAc,EAAC,UAAU,CAAC,GAAG,CAAC,KAAK,wBAAiB,EAAE,CAAC;QACzD,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,8DAA8D;IAC9D,mEAAmE;IACnE,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjD,8EAA8E;IAC9E,wEAAwE;IACxE,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAC,KAAK,SAAS,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,gCAAgC,CAAW,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,oBAAa,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAzBD,0DAyBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,gBAAgB;IACpC,IAAI,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACtC,mBAAmB,GAAG,MAAM,uBAAuB,CACjD,YAAY,EAAE,EACd,aAAa,EAAE,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AARD,4CAQC;AAED;;GAEG;AACI,KAAK,UAAU,uBAAuB;IAC3C,MAAM,QAAQ,GAAG,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAA,0BAAmB,EAAC,eAAe,CAAC,CAAC,CAAC;IAE5D,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,OAAO,CAC1C,yEAAyE,EACzE;QACE,KAAK;QACL,IAAI;QACJ,MAAM;KACP,CACF,CAAC;IACF,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC;IAEnD,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC;IAEvE,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;AACpC,CAAC;AApBD,0DAoBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc;IAClC,MAAM,iBAAiB,GAAG,4BAA4B,CAAC;IAEvD,IAAI,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,uBAAuB,EAAE,CAAC;IACrD,MAAM,OAAO,GAAG,IAAA,0BAAmB,EAAC,YAAY,CAAC,CAAC;IAElD,WAAW,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;IACpD,OAAO,WAAW,CAAC;AACrB,CAAC;AAdD,wCAcC;AAEM,KAAK,UAAU,eAAe;IACnC,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAA,+BAAgB,EAAC,QAAQ,CAAC,CAAC;IAE/C,OAAO,mBAAmB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;AACxD,CAAC;AALD,0CAKC;AAED,SAAgB,mBAAmB,CACjC,YAAoB,EACpB,WAA+B;IAE/B,IAAI,YAAY,GAAG,GAAG,YAAY,GAAG,CAAC;IAEtC,MAAM,MAAM,GAAG,IAAA,uBAAgB,EAAC,WAAW,CAAC,CAAC;IAC7C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,uDAAuD;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAClD,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACjC,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,qDAAqD;gBACrD,6CAA6C;gBAC7C,YAAY,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AArBD,kDAqBC;AASD,wEAAwE;AACjE,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,GAAW;IAEX,MAAM,aAAa,GAAG,IAAA,+BAAkB,EACtC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CACzC,CAAC;IAEF,OAAO,MAAM,YAAY,EAAE,CAAC,QAAQ,CAClC,0CAA0C,EAC1C;QACE,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,GAAG;QACH,GAAG;KACJ,CACF,CAAC;AACJ,CAAC;AAjBD,8CAiBC;AAED,8CAA8C;AACvC,KAAK,UAAU,kBAAkB,CAAC,EAAU;IACjD,MAAM,aAAa,GAAG,IAAA,+BAAkB,EACtC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CACzC,CAAC;IAEF,MAAM,YAAY,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC;QACvD,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;AACL,CAAC;AAVD,gDAUC;AAED,SAAgB,yBAAyB,CAAC,CAAU;IAClD,IAAI,IAAA,kBAAW,EAAC,CAAC,CAAC,EAAE,CAAC;QACnB,IACE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kDAAkD,CAAC;YACtE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACtC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EACvD,CAAC;YACD,OAAO,IAAI,yBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAXD,8DAWC"}
|
||||
6
lib/feature-flags.js
generated
6
lib/feature-flags.js
generated
|
|
@ -50,6 +50,7 @@ exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = "2.15.1";
|
|||
var Feature;
|
||||
(function (Feature) {
|
||||
Feature["AutobuildDirectTracing"] = "autobuild_direct_tracing";
|
||||
Feature["CleanupTrapCaches"] = "cleanup_trap_caches";
|
||||
Feature["CppDependencyInstallation"] = "cpp_dependency_installation_enabled";
|
||||
Feature["CppTrapCachingEnabled"] = "cpp_trap_caching_enabled";
|
||||
Feature["DisableJavaBuildlessEnabled"] = "disable_java_buildless_enabled";
|
||||
|
|
@ -64,6 +65,11 @@ exports.featureConfig = {
|
|||
minimumVersion: undefined,
|
||||
toolsFeature: tools_features_1.ToolsFeature.TraceCommandUseBuildMode,
|
||||
},
|
||||
[Feature.CleanupTrapCaches]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.CppDependencyInstallation]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_EXTRACTOR_CPP_AUTOINSTALL_DEPENDENCIES",
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
76
lib/trap-caching.js
generated
76
lib/trap-caching.js
generated
|
|
@ -23,11 +23,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getTotalCacheSize = exports.getLanguagesSupportingCaching = exports.uploadTrapCaches = exports.downloadTrapCaches = void 0;
|
||||
exports.getTotalCacheSize = exports.getLanguagesSupportingCaching = exports.cleanupTrapCaches = exports.uploadTrapCaches = exports.downloadTrapCaches = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const cache = __importStar(require("@actions/cache"));
|
||||
const actionsCache = __importStar(require("@actions/cache"));
|
||||
const actionsUtil = __importStar(require("./actions-util"));
|
||||
const apiClient = __importStar(require("./api-client"));
|
||||
const feature_flags_1 = require("./feature-flags");
|
||||
const util_1 = require("./util");
|
||||
// This constant should be bumped if we make a breaking change
|
||||
// to how the CodeQL Action stores or retrieves the TRAP cache,
|
||||
|
|
@ -35,6 +37,7 @@ const util_1 = require("./util");
|
|||
// this for CLI/extractor changes, since the CLI version also
|
||||
// goes into the cache key.
|
||||
const CACHE_VERSION = 1;
|
||||
const CODEQL_TRAP_CACHE_PREFIX = "codeql-trap";
|
||||
// This constant sets the minimum size in megabytes of a TRAP
|
||||
// cache for us to consider it worth uploading.
|
||||
const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
|
||||
|
|
@ -81,7 +84,7 @@ async function downloadTrapCaches(codeql, languages, logger) {
|
|||
// The SHA from the base of the PR is the most similar commit we might have a cache for
|
||||
const preferredKey = await cacheKey(codeql, language, baseSha);
|
||||
logger.info(`Looking in Actions cache for TRAP cache with key ${preferredKey}`);
|
||||
const found = await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, cache.restoreCache([cacheDir], preferredKey, [
|
||||
const found = await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, actionsCache.restoreCache([cacheDir], preferredKey, [
|
||||
// Fall back to any cache with the right key prefix
|
||||
await cachePrefix(codeql, language),
|
||||
]), () => {
|
||||
|
|
@ -123,13 +126,76 @@ async function uploadTrapCaches(codeql, config, logger) {
|
|||
}
|
||||
const key = await cacheKey(codeql, language, process.env.GITHUB_SHA || "unknown");
|
||||
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
|
||||
await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, cache.saveCache([cacheDir], key), () => {
|
||||
await (0, util_1.withTimeout)(MAX_CACHE_OPERATION_MS, actionsCache.saveCache([cacheDir], key), () => {
|
||||
logger.info(`Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading`);
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
exports.uploadTrapCaches = uploadTrapCaches;
|
||||
async function cleanupTrapCaches(config, features, logger) {
|
||||
if (!(await features.getValue(feature_flags_1.Feature.CleanupTrapCaches))) {
|
||||
return {
|
||||
trap_cache_cleanup_skipped_because: "feature disabled",
|
||||
};
|
||||
}
|
||||
if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
|
||||
return {
|
||||
trap_cache_cleanup_skipped_because: "not analyzing default branch",
|
||||
};
|
||||
}
|
||||
try {
|
||||
let totalBytesCleanedUp = 0;
|
||||
const allCaches = await apiClient.listActionsCaches(CODEQL_TRAP_CACHE_PREFIX, await actionsUtil.getRef());
|
||||
for (const language of config.languages) {
|
||||
if (config.trapCaches[language]) {
|
||||
const cachesToRemove = await getTrapCachesForLanguage(allCaches, language, logger);
|
||||
// Dates returned by the API are in ISO 8601 format, so we can sort them lexicographically
|
||||
cachesToRemove.sort((a, b) => a.created_at.localeCompare(b.created_at));
|
||||
// Keep the most recent cache
|
||||
const mostRecentCache = cachesToRemove.pop();
|
||||
logger.debug(`Keeping most recent TRAP cache (${JSON.stringify(mostRecentCache)})`);
|
||||
if (cachesToRemove.length === 0) {
|
||||
logger.info(`No TRAP caches to clean up for ${language}.`);
|
||||
continue;
|
||||
}
|
||||
for (const cache of cachesToRemove) {
|
||||
logger.debug(`Cleaning up TRAP cache (${JSON.stringify(cache)})`);
|
||||
await apiClient.deleteActionsCache(cache.id);
|
||||
}
|
||||
const bytesCleanedUp = cachesToRemove.reduce((acc, item) => acc + item.size_in_bytes, 0);
|
||||
totalBytesCleanedUp += bytesCleanedUp;
|
||||
const megabytesCleanedUp = (bytesCleanedUp / (1024 * 1024)).toFixed(2);
|
||||
logger.info(`Cleaned up ${megabytesCleanedUp} MiB of old TRAP caches for ${language}.`);
|
||||
}
|
||||
}
|
||||
return { trap_cache_cleanup_size_bytes: totalBytesCleanedUp };
|
||||
}
|
||||
catch (e) {
|
||||
if ((0, util_1.isHTTPError)(e) && e.status === 403) {
|
||||
logger.warning("Could not cleanup TRAP caches as the token did not have the required permissions. " +
|
||||
'To clean up TRAP caches, ensure the token has the "actions:write" permission. ' +
|
||||
"For more information, see https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs");
|
||||
}
|
||||
else {
|
||||
logger.info(`Failed to cleanup TRAP caches, continuing. Details: ${e}`);
|
||||
}
|
||||
return { trap_cache_cleanup_error: (0, util_1.wrapError)(e).message };
|
||||
}
|
||||
}
|
||||
exports.cleanupTrapCaches = cleanupTrapCaches;
|
||||
async function getTrapCachesForLanguage(allCaches, language, logger) {
|
||||
logger.debug(`Listing TRAP caches for ${language}`);
|
||||
for (const cache of allCaches) {
|
||||
if (!cache.created_at || !cache.id || !cache.key || !cache.size_in_bytes) {
|
||||
throw new Error("An unexpected cache item was returned from the API that was missing one or " +
|
||||
`more required fields: ${JSON.stringify(cache)}`);
|
||||
}
|
||||
}
|
||||
return allCaches.filter((cache) => {
|
||||
return cache.key?.includes(`-${language}-`);
|
||||
});
|
||||
}
|
||||
async function getLanguagesSupportingCaching(codeql, languages, logger) {
|
||||
const result = [];
|
||||
const resolveResult = await codeql.betterResolveLanguages();
|
||||
|
|
@ -169,6 +235,6 @@ async function cacheKey(codeql, language, baseSha) {
|
|||
return `${await cachePrefix(codeql, language)}${baseSha}`;
|
||||
}
|
||||
async function cachePrefix(codeql, language) {
|
||||
return `codeql-trap-${CACHE_VERSION}-${(await codeql.getVersion()).version}-${language}-`;
|
||||
return `${CODEQL_TRAP_CACHE_PREFIX}-${CACHE_VERSION}-${(await codeql.getVersion()).version}-${language}-`;
|
||||
}
|
||||
//# sourceMappingURL=trap-caching.js.map
|
||||
File diff suppressed because one or more lines are too long
73
lib/trap-caching.test.js
generated
73
lib/trap-caching.test.js
generated
|
|
@ -32,8 +32,11 @@ const cache = __importStar(require("@actions/cache"));
|
|||
const ava_1 = __importDefault(require("ava"));
|
||||
const sinon = __importStar(require("sinon"));
|
||||
const actionsUtil = __importStar(require("./actions-util"));
|
||||
const apiClient = __importStar(require("./api-client"));
|
||||
const codeql_1 = require("./codeql");
|
||||
const feature_flags_1 = require("./feature-flags");
|
||||
const languages_1 = require("./languages");
|
||||
const logging_1 = require("./logging");
|
||||
const testing_utils_1 = require("./testing-utils");
|
||||
const trap_caching_1 = require("./trap-caching");
|
||||
const util = __importStar(require("./util"));
|
||||
|
|
@ -168,4 +171,74 @@ function getTestConfigWithTempDir(tempDir) {
|
|||
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
|
||||
});
|
||||
});
|
||||
(0, ava_1.default)("cleanup removes only old CodeQL TRAP caches", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
// This config specifies that we are analyzing JavaScript and Ruby, but not Swift.
|
||||
const config = getTestConfigWithTempDir(tmpDir);
|
||||
sinon.stub(actionsUtil, "getRef").resolves("refs/heads/main");
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
|
||||
const listStub = sinon.stub(apiClient, "listActionsCaches").resolves([
|
||||
// Should be kept, since it's not relevant to CodeQL. In reality, the API shouldn't return
|
||||
// this in the first place, but this is a defensive check.
|
||||
{
|
||||
id: 1,
|
||||
key: "some-other-key",
|
||||
created_at: "2024-05-23T14:25:00Z",
|
||||
size_in_bytes: 100 * 1024 * 1024,
|
||||
},
|
||||
// Should be kept, since it's the newest TRAP cache for JavaScript
|
||||
{
|
||||
id: 2,
|
||||
key: "codeql-trap-1-2.0.0-javascript-newest",
|
||||
created_at: "2024-04-23T14:25:00Z",
|
||||
size_in_bytes: 50 * 1024 * 1024,
|
||||
},
|
||||
// Should be cleaned up
|
||||
{
|
||||
id: 3,
|
||||
key: "codeql-trap-1-2.0.0-javascript-older",
|
||||
created_at: "2024-03-22T14:25:00Z",
|
||||
size_in_bytes: 200 * 1024 * 1024,
|
||||
},
|
||||
// Should be cleaned up
|
||||
{
|
||||
id: 4,
|
||||
key: "codeql-trap-1-2.0.0-javascript-oldest",
|
||||
created_at: "2024-02-21T14:25:00Z",
|
||||
size_in_bytes: 300 * 1024 * 1024,
|
||||
},
|
||||
// Should be kept, since it's the newest TRAP cache for Ruby
|
||||
{
|
||||
id: 5,
|
||||
key: "codeql-trap-1-2.0.0-ruby-newest",
|
||||
created_at: "2024-02-20T14:25:00Z",
|
||||
size_in_bytes: 300 * 1024 * 1024,
|
||||
},
|
||||
// Should be kept, since we aren't analyzing Swift
|
||||
{
|
||||
id: 6,
|
||||
key: "codeql-trap-1-2.0.0-swift-newest",
|
||||
created_at: "2024-02-22T14:25:00Z",
|
||||
size_in_bytes: 300 * 1024 * 1024,
|
||||
},
|
||||
// Should be kept, since we aren't analyzing Swift
|
||||
{
|
||||
id: 7,
|
||||
key: "codeql-trap-1-2.0.0-swift-older",
|
||||
created_at: "2024-02-21T14:25:00Z",
|
||||
size_in_bytes: 300 * 1024 * 1024,
|
||||
},
|
||||
]);
|
||||
const deleteStub = sinon.stub(apiClient, "deleteActionsCache").resolves();
|
||||
const statusReport = await (0, trap_caching_1.cleanupTrapCaches)(config, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CleanupTrapCaches]), (0, logging_1.getRunnerLogger)(true));
|
||||
t.is(listStub.callCount, 1);
|
||||
t.assert(listStub.calledWithExactly("codeql-trap", "refs/heads/main"));
|
||||
t.deepEqual(statusReport, {
|
||||
trap_cache_cleanup_size_bytes: 500 * 1024 * 1024,
|
||||
});
|
||||
t.is(deleteStub.callCount, 2);
|
||||
t.assert(deleteStub.calledWithExactly(3));
|
||||
t.assert(deleteStub.calledWithExactly(4));
|
||||
});
|
||||
});
|
||||
//# sourceMappingURL=trap-caching.test.js.map
|
||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue