robojerk/codeql-action

Add telemetry for TRAP cache cleanup

Browse source
This commit is contained in:
Henry Mercer 2024-05-22 12:38:40 +01:00
parent 59af9fc5ab
commit 4f2b1826e9
6 changed files with 57 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

16
lib/analyze-action.js generated
View file

@ -48,7 +48,7 @@ const status_report_1 = require("./status-report");
const trap_caching_1 = require("./trap-caching");
const uploadLib = __importStar(require("./upload-lib"));
const util = __importStar(require("./util"));
async function sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger) {
async function sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, logger) {
const status = (0, status_report_1.getActionsStatus)(error, stats?.analyze_failure_language);
const statusReportBase = await (0, status_report_1.createStatusReportBase)(status_report_1.ActionName.Analyze, status, startedAt, config, await util.checkDiskUsage(), logger, error?.message, error?.stack);
if (statusReportBase !== undefined) {
@ -56,6 +56,7 @@ async function sendStatusReport(startedAt, config, stats, error, trapCacheUpload
...statusReportBase,
...(stats || {}),
...(dbCreationTimings || {}),
...(trapCacheCleanup || {}),
};
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport = {
@ -141,6 +142,7 @@ async function run() {
let uploadResult = undefined;
let runStats = undefined;
let config = undefined;
let trapCacheCleanupTelemetry = undefined;
let trapCacheUploadTime = undefined;
let dbCreationTimings = undefined;
let didUploadTrapCaches = false;
@ -197,7 +199,7 @@ async function run() {
didUploadTrapCaches = await (0, trap_caching_1.uploadTrapCaches)(codeql, config, logger);
trapCacheUploadTime = perf_hooks_1.performance.now() - trapCacheUploadStartTime;
// Clean up TRAP caches
await (0, trap_caching_1.cleanupTrapCaches)(config, logger);
trapCacheCleanupTelemetry = await (0, trap_caching_1.cleanupTrapCaches)(config, logger);
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
logger.debug("In test mode. Waiting for processing is disabled.");
@ -220,10 +222,10 @@ async function run() {
}
if (error instanceof analyze_1.CodeQLAnalysisError) {
const stats = { ...error.queriesStatusReport };
await sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, stats, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
else {
await sendStatusReport(startedAt, config, undefined, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, undefined, error, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
return;
}
@ -231,13 +233,13 @@ async function run() {
await sendStatusReport(startedAt, config, {
...runStats,
...uploadResult.statusReport,
}, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
}, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
else if (runStats) {
await sendStatusReport(startedAt, config, { ...runStats }, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, { ...runStats }, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
else {
await sendStatusReport(startedAt, config, undefined, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, logger);
await sendStatusReport(startedAt, config, undefined, undefined, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, logger);
}
}
exports.runPromise = run();

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

18
lib/trap-caching.js generated
View file

@ -133,9 +133,13 @@ async function uploadTrapCaches(codeql, config, logger) {
}
exports.uploadTrapCaches = uploadTrapCaches;
async function cleanupTrapCaches(config, logger) {
if (!(await actionsUtil.isAnalyzingDefaultBranch()))
return;
if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
return {
trap_cache_cleanup_skipped_because: "not analyzing default branch",
};
}
try {
let totalBytesCleanedUp = 0;
for (const language of config.languages) {
if (config.trapCaches[language]) {
const cachesToRemove = await getTrapCachesForLanguage(language, logger);
@ -148,12 +152,13 @@ async function cleanupTrapCaches(config, logger) {
logger.debug(`Deleting old TRAP cache (${JSON.stringify(cache)})`);
await apiClient.deleteActionsCache(cache.id);
}
const totalBytesCleanedUp = cachesToRemove.reduce((acc, item) => acc + item.size_in_bytes, 0);
const totalMegabytesCleanedUp = (totalBytesCleanedUp /
(1024 * 1024)).toFixed(2);
logger.info(`Cleaned up ${totalMegabytesCleanedUp} MiB of old TRAP caches for ${language}.`);
const bytesCleanedUp = cachesToRemove.reduce((acc, item) => acc + item.size_in_bytes, 0);
totalBytesCleanedUp += bytesCleanedUp;
const megabytesCleanedUp = (bytesCleanedUp / (1024 * 1024)).toFixed(2);
logger.info(`Cleaned up ${megabytesCleanedUp} MiB of old TRAP caches for ${language}.`);
}
}
return { trap_cache_cleanup_size_bytes: totalBytesCleanedUp };
}
catch (e) {
if ((0, util_1.isHTTPError)(e) && e.status === 403) {
@ -164,6 +169,7 @@ async function cleanupTrapCaches(config, logger) {
else {
logger.info(`Failed to cleanup TRAP caches, continuing. Details: ${e}`);
}
return { trap_cache_cleanup_error: (0, util_1.wrapError)(e).message };
}
}
exports.cleanupTrapCaches = cleanupTrapCaches;

2
lib/trap-caching.js.map
View file

File diff suppressed because one or more lines are too long

12
src/analyze-action.ts
View file

@ -35,6 +35,7 @@ import {
import {
cleanupTrapCaches,
getTotalCacheSize,
TrapCacheCleanupStatusReport,
uploadTrapCaches,
} from "./trap-caching";
import * as uploadLib from "./upload-lib";
@ -65,6 +66,7 @@ async function sendStatusReport(
trapCacheUploadTime: number | undefined,
dbCreationTimings: DatabaseCreationTimings | undefined,
didUploadTrapCaches: boolean,
trapCacheCleanup: TrapCacheCleanupStatusReport | undefined,
logger: Logger,
) {
const status = getActionsStatus(error, stats?.analyze_failure_language);
@ -83,6 +85,7 @@ async function sendStatusReport(
...statusReportBase,
...(stats || {}),
...(dbCreationTimings || {}),
...(trapCacheCleanup || {}),
};
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = {
@ -193,6 +196,8 @@ async function run() {
let uploadResult: UploadResult | undefined = undefined;
let runStats: QueriesStatusReport | undefined = undefined;
let config: Config | undefined = undefined;
let trapCacheCleanupTelemetry: TrapCacheCleanupStatusReport | undefined =
undefined;
let trapCacheUploadTime: number | undefined = undefined;
let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
let didUploadTrapCaches = false;
@ -316,7 +321,7 @@ async function run() {
trapCacheUploadTime = performance.now() - trapCacheUploadStartTime;
// Clean up TRAP caches
await cleanupTrapCaches(config, logger);
trapCacheCleanupTelemetry = await cleanupTrapCaches(config, logger);
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
@ -357,6 +362,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
} else {
@ -368,6 +374,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
}
@ -387,6 +394,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
} else if (runStats) {
@ -398,6 +406,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
} else {
@ -409,6 +418,7 @@ async function run() {
trapCacheUploadTime,
dbCreationTimings,
didUploadTrapCaches,
trapCacheCleanupTelemetry,
logger,
);
}

32
src/trap-caching.ts
View file

@ -9,7 +9,7 @@ import { CodeQL } from "./codeql";
import type { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import { isHTTPError, tryGetFolderBytes, withTimeout } from "./util";
import { isHTTPError, tryGetFolderBytes, withTimeout, wrapError } from "./util";
// This constant should be bumped if we make a breaking change
// to how the CodeQL Action stores or retrieves the TRAP cache,
@ -161,10 +161,24 @@ export async function uploadTrapCaches(
return true;
}
export async function cleanupTrapCaches(config: Config, logger: Logger) {
if (!(await actionsUtil.isAnalyzingDefaultBranch())) return;
export interface TrapCacheCleanupStatusReport {
trap_cache_cleanup_error?: string;
trap_cache_cleanup_size_bytes?: number;
trap_cache_cleanup_skipped_because?: string;
}
export async function cleanupTrapCaches(
config: Config,
logger: Logger,
): Promise<TrapCacheCleanupStatusReport> {
if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
return {
trap_cache_cleanup_skipped_because: "not analyzing default branch",
};
}
try {
let totalBytesCleanedUp = 0;
for (const language of config.languages) {
if (config.trapCaches[language]) {
const cachesToRemove = await getTrapCachesForLanguage(language, logger);
@ -179,19 +193,18 @@ export async function cleanupTrapCaches(config: Config, logger: Logger) {
logger.debug(`Deleting old TRAP cache (${JSON.stringify(cache)})`);
await apiClient.deleteActionsCache(cache.id);
}
const totalBytesCleanedUp = cachesToRemove.reduce(
const bytesCleanedUp = cachesToRemove.reduce(
(acc, item) => acc + item.size_in_bytes,
0,
);
const totalMegabytesCleanedUp = (
totalBytesCleanedUp /
(1024 * 1024)
).toFixed(2);
totalBytesCleanedUp += bytesCleanedUp;
const megabytesCleanedUp = (bytesCleanedUp / (1024 * 1024)).toFixed(2);
logger.info(
`Cleaned up ${totalMegabytesCleanedUp} MiB of old TRAP caches for ${language}.`,
`Cleaned up ${megabytesCleanedUp} MiB of old TRAP caches for ${language}.`,
);
}
}
return { trap_cache_cleanup_size_bytes: totalBytesCleanedUp };
} catch (e) {
if (isHTTPError(e) && e.status === 403) {
logger.warning(
@ -202,6 +215,7 @@ export async function cleanupTrapCaches(config: Config, logger: Logger) {
} else {
logger.info(`Failed to cleanup TRAP caches, continuing. Details: ${e}`);
}
return { trap_cache_cleanup_error: wrapError(e).message };
}
}