robojerk/codeql-action

Merge remote-tracking branch 'origin/main' into angelapwen/post-init-cleanup

Browse source
This commit is contained in:
Angela P Wen 2022-08-10 15:08:04 +02:00
commit 90676d9cb9
52 changed files with 985 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/analysis-paths.test.ts
View file

@ -25,6 +25,7 @@ test("emptyPaths", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@ -50,6 +51,7 @@ test("nonEmptyPaths", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@ -78,6 +80,7 @@ test("exclude temp dir", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);

6
src/analyze-action.ts
View file

@ -5,8 +5,8 @@ import {
CodeQLAnalysisError,
QueriesStatusReport,
runCleanup,
runQueries,
runFinalize,
runQueries,
} from "./analyze";
import { getGitHubVersionActionsOnly } from "./api-client";
import { getCodeQL } from "./codeql";
@ -15,6 +15,7 @@ import { uploadDatabases } from "./database-upload";
import { GitHubFeatureFlags } from "./feature-flags";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { uploadTrapCaches } from "./trap-caching";
import * as upload_lib from "./upload-lib";
import { UploadResult } from "./upload-lib";
import * as util from "./util";
@ -160,6 +161,9 @@ async function run() {
// Possibly upload the database bundles for remote queries
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
await uploadTrapCaches(codeql, config, logger);
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
core.debug("In test mode. Waiting for processing is disabled.");

2
src/analyze.test.ts
View file

@ -113,6 +113,7 @@ test("status report fields and search path setting", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
@ -268,6 +269,7 @@ const stubConfig: Config = {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
for (const options of [

6
src/analyze.ts
View file

@ -137,11 +137,7 @@ export async function createdDBForScannedLanguages(
await setupPythonExtractor(logger);
}
await codeql.extractScannedLanguage(
util.getCodeQLDatabasePath(config, language),
language,
featureFlags
);
await codeql.extractScannedLanguage(config, language, featureFlags);
logger.endGroup();
}
}

1
src/codeql.test.ts
View file

@ -435,6 +435,7 @@ const stubConfig: Config = {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
test("databaseInitCluster() Lua feature flag enabled, but old CLI", async (t) => {

58
src/codeql.ts
View file

@ -9,15 +9,19 @@ import { default as queryString } from "query-string";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
import { getRelativeScriptPath, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { FeatureFlags, FeatureFlag } from "./feature-flags";
import { FeatureFlag, FeatureFlags } from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import {
getTrapCachingExtractorConfigArgs,
getTrapCachingExtractorConfigArgsForLang,
} from "./trap-caching";
import * as util from "./util";
import { isGoodVersion } from "./util";
@ -97,7 +101,7 @@ export interface CodeQL {
* and running the language extractor.
*/
extractScannedLanguage(
database: string,
config: Config,
language: Language,
featureFlags: FeatureFlags
): Promise<void>;
@ -113,6 +117,10 @@ export interface CodeQL {
* Run 'codeql resolve languages'.
*/
resolveLanguages(): Promise<ResolveLanguagesOutput>;
/**
* Run 'codeql resolve languages' with '--format=betterjson'.
*/
betterResolveLanguages(): Promise<BetterResolveLanguagesOutput>;
/**
* Run 'codeql resolve queries'.
*/
@ -170,6 +178,17 @@ export interface ResolveLanguagesOutput {
[language: string]: [string];
}
export interface BetterResolveLanguagesOutput {
extractors: {
[language: string]: [
{
extractor_root: string;
extractor_options?: any;
}
];
};
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
@ -248,6 +267,12 @@ export const CODEQL_VERSION_NEW_TRACING = "2.7.0";
*/
export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
/**
* Previous versions had the option already, but were missing the
* --extractor-options-verbosity that we need.
*/
export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
function getCodeQLBundleName(): string {
let platform: string;
if (process.platform === "win32") {
@ -585,6 +610,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
),
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
betterResolveLanguages: resolveFunction(
partialCodeql,
"betterResolveLanguages"
),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
@ -730,6 +759,7 @@ async function getCodeQLForCmd(
);
if (config.languages.filter(isTracedLanguage).length > 0) {
extraArgs.push("--begin-tracing");
extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
if (processName !== undefined) {
extraArgs.push(`--trace-process-name=${processName}`);
} else {
@ -797,10 +827,11 @@ async function getCodeQLForCmd(
await runTool(autobuildCmd);
},
async extractScannedLanguage(
databasePath: string,
config: Config,
language: Language,
featureFlags: FeatureFlags
) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Get extractor location
let extractorPath = "";
await new toolrunner.ToolRunner(
@ -850,6 +881,7 @@ async function getCodeQLForCmd(
"database",
"trace-command",
...extraArgs,
...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
...getExtraOptionsFromEnv(["database", "trace-command"]),
databasePath,
"--",
@ -892,6 +924,24 @@ async function getCodeQLForCmd(
);
}
},
async betterResolveLanguages() {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
try {
return JSON.parse(output);
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`
);
}
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined

24
src/config-utils.test.ts
View file

@ -86,6 +86,7 @@ test("load empty config", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -106,6 +107,7 @@ test("load empty config", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -151,6 +153,7 @@ test("loading config saves config", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -185,6 +188,7 @@ test("load input outside of workspace", async (t) => {
"../input",
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -223,6 +227,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -262,6 +267,7 @@ test("load non-existent input", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -350,6 +356,7 @@ test("load non-empty input", async (t) => {
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
injectedMlQueries: false,
trapCaches: {},
};
const languages = "javascript";
@ -362,6 +369,7 @@ test("load non-empty input", async (t) => {
configFilePath,
undefined,
false,
false,
"my-artifact",
"my-db",
{ owner: "github", repo: "example " },
@ -428,6 +436,7 @@ test("Default queries are used", async (t) => {
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -502,6 +511,7 @@ test("Queries can be specified in config file", async (t) => {
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -575,6 +585,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -646,6 +657,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -708,6 +720,7 @@ test("Multiple queries can be specified in workflow file, no config file require
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -791,6 +804,7 @@ test("Queries in workflow file can be added to the set of queries without overri
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -868,6 +882,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -935,6 +950,7 @@ test("API client used when reading remote config", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -964,6 +980,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
repoReference,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -1001,6 +1018,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
repoReference,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -1039,6 +1057,7 @@ test("No detected languages", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -1069,6 +1088,7 @@ test("Unknown languages", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -1121,6 +1141,7 @@ test("Config specifies packages", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -1177,6 +1198,7 @@ test("Config specifies packages for multiple languages", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
@ -1244,6 +1266,7 @@ function doInvalidInputTest(
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
@ -1766,6 +1789,7 @@ const mlPoweredQueriesMacro = test.macro({
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },

17
src/config-utils.ts
View file

@ -16,6 +16,7 @@ import { FeatureFlag, FeatureFlags } from "./feature-flags";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { downloadTrapCaches } from "./trap-caching";
import {
codeQlVersionAbove,
getMlPoweredJsQueriesPack,
@ -148,6 +149,11 @@ export interface Config {
* Whether we injected ML queries into this configuration.
*/
injectedMlQueries: boolean;
/**
* Partial map from languages to locations of TRAP caches for that language.
* If a key is omitted, then TRAP caching should not be used for that language.
*/
trapCaches: Partial<Record<Language, string>>;
}
export type Packs = Partial<Record<Language, string[]>>;
@ -878,6 +884,7 @@ export async function getDefaultConfig(
queriesInput: string | undefined,
packsInput: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
@ -937,6 +944,9 @@ export async function getDefaultConfig(
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
};
}
@ -949,6 +959,7 @@ async function loadConfig(
packsInput: string | undefined,
configFile: string,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
@ -1117,6 +1128,9 @@ async function loadConfig(
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
};
}
@ -1358,6 +1372,7 @@ export async function initConfig(
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
@ -1380,6 +1395,7 @@ export async function initConfig(
queriesInput,
packsInput,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
@ -1399,6 +1415,7 @@ export async function initConfig(
packsInput,
configFile,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,

1
src/database-upload.test.ts
View file

@ -56,6 +56,7 @@ function getTestConfig(tmpDir: string): Config {
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
}

1
src/feature-flags.ts
View file

@ -10,6 +10,7 @@ export interface FeatureFlags {
export enum FeatureFlag {
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
LuaTracerConfigEnabled = "lua_tracer_config_enabled",
TrapCachingEnabled = "trap_caching_enabled",
}
/**

25
src/init-action.ts
View file

@ -15,7 +15,7 @@ import {
import { getGitHubVersionActionsOnly } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { GitHubFeatureFlags } from "./feature-flags";
import { FeatureFlag, FeatureFlags, GitHubFeatureFlags } from "./feature-flags";
import {
initCodeQL,
initConfig,
@ -27,18 +27,18 @@ import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
getRequiredEnvParam,
initializeEnvironment,
Mode,
checkActionVersion,
checkGitHubVersionInRange,
codeQlVersionAbove,
enrichEnvironment,
getMemoryFlagValue,
getThreadsFlagValue,
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
enrichEnvironment,
getMemoryFlagValue,
getMlPoweredJsQueriesStatus,
checkActionVersion,
getRequiredEnvParam,
getThreadsFlagValue,
initializeEnvironment,
Mode,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -183,6 +183,7 @@ async function run() {
getOptionalInput("packs"),
getOptionalInput("config-file"),
getOptionalInput("db-location"),
await getTrapCachingEnabled(featureFlags),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@ -299,6 +300,14 @@ async function run() {
await sendSuccessStatusReport(startedAt, config, toolsVersion);
}
async function getTrapCachingEnabled(
featureFlags: FeatureFlags
): Promise<boolean> {
const trapCaching = getOptionalInput("trap-caching");
if (trapCaching !== undefined) return trapCaching === "true";
return await featureFlags.getValue(FeatureFlag.TrapCachingEnabled);
}
async function runWrapper() {
try {
await run();

2
src/init.ts
View file

@ -42,6 +42,7 @@ export async function initConfig(
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
@ -61,6 +62,7 @@ export async function initConfig(
packsInput,
configFile,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,

1
src/runner.ts
View file

@ -241,6 +241,7 @@ program
cmd.configFile,
undefined,
false,
false,
"",
"",
parseRepositoryNwo(cmd.repository),

1
src/tracer-config.test.ts
View file

@ -32,6 +32,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
}

210
src/trap-caching.test.ts Normal file
View file

@ -0,0 +1,210 @@
import * as fs from "fs";
import * as path from "path";
import * as cache from "@actions/cache";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { getRecordingLogger, setupTests } from "./testing-utils";
import {
downloadTrapCaches,
getLanguagesSupportingCaching,
getTrapCachingExtractorConfigArgs,
getTrapCachingExtractorConfigArgsForLang,
uploadTrapCaches,
} from "./trap-caching";
import * as util from "./util";
setupTests(test);
const stubCodeql = setCodeQL({
async getVersion() {
return "2.10.3";
},
async betterResolveLanguages() {
return {
extractors: {
[Language.javascript]: [
{
extractor_root: "some_root",
extractor_options: {
trap: {
properties: {
cache: {
properties: {
dir: {
title: "Cache directory",
},
bound: {
title: "Cache bound",
},
write: {
title: "Cache write",
},
},
},
},
},
},
},
],
[Language.cpp]: [
{
extractor_root: "other_root",
},
],
},
};
},
});
const testConfigWithoutTmpDir: Config = {
languages: [Language.javascript, Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {
javascript: "/some/cache/dir",
},
};
function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
return {
languages: [Language.javascript, Language.ruby],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {
javascript: path.resolve(tmpDir, "jsCache"),
ruby: path.resolve(tmpDir, "rubyCache"),
},
};
}
test("check flags for JS, analyzing default branch", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfigWithTempDir(tmpDir);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const result = await getTrapCachingExtractorConfigArgsForLang(
config,
Language.javascript
);
t.deepEqual(result, [
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
"-O=javascript.trap.cache.bound=1024",
"-O=javascript.trap.cache.write=true",
]);
});
});
test("check flags for all, not analyzing default branch", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfigWithTempDir(tmpDir);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
const result = await getTrapCachingExtractorConfigArgs(config);
t.deepEqual(result, [
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
"-O=javascript.trap.cache.bound=1024",
"-O=javascript.trap.cache.write=false",
`-O=ruby.trap.cache.dir=${path.resolve(tmpDir, "rubyCache")}`,
"-O=ruby.trap.cache.bound=1024",
"-O=ruby.trap.cache.write=false",
]);
});
});
test("get languages that support TRAP caching", async (t) => {
const loggedMessages = [];
const logger = getRecordingLogger(loggedMessages);
const languagesSupportingCaching = await getLanguagesSupportingCaching(
stubCodeql,
[Language.javascript, Language.cpp],
logger
);
t.deepEqual(languagesSupportingCaching, [Language.javascript]);
});
test("upload cache key contains right fields", async (t) => {
const loggedMessages = [];
const logger = getRecordingLogger(loggedMessages);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const stubSave = sinon.stub(cache, "saveCache");
process.env.GITHUB_SHA = "somesha";
await uploadTrapCaches(stubCodeql, testConfigWithoutTmpDir, logger);
t.assert(
stubSave.calledOnceWith(
sinon.match.array.contains(["/some/cache/dir"]),
sinon
.match("somesha")
.and(sinon.match("2.10.3"))
.and(sinon.match("javascript"))
)
);
});
test("download cache looks for the right key and creates dir", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const loggedMessages = [];
const logger = getRecordingLogger(loggedMessages);
sinon.stub(actionsUtil, "getTemporaryDirectory").returns(tmpDir);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
const stubRestore = sinon.stub(cache, "restoreCache").resolves("found");
const eventFile = path.resolve(tmpDir, "event.json");
process.env.GITHUB_EVENT_NAME = "pull_request";
process.env.GITHUB_EVENT_PATH = eventFile;
fs.writeFileSync(
eventFile,
JSON.stringify({
pull_request: {
base: {
sha: "somesha",
},
},
})
);
await downloadTrapCaches(
stubCodeql,
[Language.javascript, Language.cpp],
logger
);
t.assert(
stubRestore.calledOnceWith(
sinon.match.array.contains([
path.resolve(tmpDir, "trapCaches", "javascript"),
]),
sinon
.match("somesha")
.and(sinon.match("2.10.3"))
.and(sinon.match("javascript"))
)
);
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
});
});

191
src/trap-caching.ts Normal file
View file

@ -0,0 +1,191 @@
import * as fs from "fs";
import * as path from "path";
import * as cache from "@actions/cache";
import * as actionsUtil from "./actions-util";
import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import { codeQlVersionAbove } from "./util";
// This constant should be bumped if we make a breaking change
// to how the CodeQL Action stores or retrieves the TRAP cache,
// and will invalidate previous caches. We don't need to bump
// this for CLI/extractor changes, since the CLI version also
// goes into the cache key.
const CACHE_VERSION = 1;
// This constant sets the size of each TRAP cache in megabytes.
const CACHE_SIZE_MB = 1024;
export async function getTrapCachingExtractorConfigArgs(
config: Config
): Promise<string[]> {
const result: string[][] = [];
for (const language of config.languages)
result.push(
await getTrapCachingExtractorConfigArgsForLang(config, language)
);
return result.flat();
}
export async function getTrapCachingExtractorConfigArgsForLang(
config: Config,
language: Language
): Promise<string[]> {
const cacheDir = config.trapCaches[language];
if (cacheDir === undefined) return [];
const write = await actionsUtil.isAnalyzingDefaultBranch();
return [
`-O=${language}.trap.cache.dir=${cacheDir}`,
`-O=${language}.trap.cache.bound=${CACHE_SIZE_MB}`,
`-O=${language}.trap.cache.write=${write}`,
];
}
/**
* Download TRAP caches from the Actions cache.
* @param codeql The CodeQL instance to use.
* @param languages The languages being analyzed.
* @param logger A logger to record some informational messages to.
* @returns A partial map from languages to TRAP cache paths on disk, with
* languages for which we shouldn't use TRAP caching omitted.
*/
export async function downloadTrapCaches(
codeql: CodeQL,
languages: Language[],
logger: Logger
): Promise<Partial<Record<Language, string>>> {
const result = {};
const languagesSupportingCaching = await getLanguagesSupportingCaching(
codeql,
languages,
logger
);
logger.info(
`Found ${languagesSupportingCaching.length} languages that support TRAP caching`
);
if (languagesSupportingCaching.length === 0) return result;
const cachesDir = path.join(
actionsUtil.getTemporaryDirectory(),
"trapCaches"
);
for (const language of languagesSupportingCaching) {
const cacheDir = path.join(cachesDir, language);
fs.mkdirSync(cacheDir, { recursive: true });
result[language] = cacheDir;
}
if (await actionsUtil.isAnalyzingDefaultBranch()) {
logger.info(
"Analyzing default branch. Skipping downloading of TRAP caches."
);
return result;
}
let baseSha = "unknown";
const eventPath = process.env.GITHUB_EVENT_PATH;
if (
process.env.GITHUB_EVENT_NAME === "pull_request" &&
eventPath !== undefined
) {
const event = JSON.parse(fs.readFileSync(path.resolve(eventPath), "utf-8"));
baseSha = event.pull_request?.base?.sha || baseSha;
}
for (const language of languages) {
const cacheDir = result[language];
if (cacheDir === undefined) continue;
// The SHA from the base of the PR is the most similar commit we might have a cache for
const preferredKey = await cacheKey(codeql, language, baseSha);
logger.info(
`Looking in Actions cache for TRAP cache with key ${preferredKey}`
);
const found = await cache.restoreCache([cacheDir], preferredKey, [
await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
]);
if (found === undefined) {
// We didn't find a TRAP cache in the Actions cache, so the directory on disk is
// still just an empty directory. There's no reason to tell the extractor to use it,
// so let's unset the entry in the map so we don't set any extractor options.
logger.info(`No TRAP cache found in Actions cache for ${language}`);
result[language] = undefined;
}
}
return result;
}
export async function uploadTrapCaches(
codeql: CodeQL,
config: Config,
logger: Logger
): Promise<void> {
if (!(await actionsUtil.isAnalyzingDefaultBranch())) return; // Only upload caches from the default branch
const toAwait: Array<Promise<number>> = [];
for (const language of config.languages) {
const cacheDir = config.trapCaches[language];
if (cacheDir === undefined) continue;
const key = await cacheKey(
codeql,
language,
process.env.GITHUB_SHA || "unknown"
);
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
toAwait.push(cache.saveCache([cacheDir], key));
}
await Promise.all(toAwait);
}
export async function getLanguagesSupportingCaching(
codeql: CodeQL,
languages: Language[],
logger: Logger
): Promise<Language[]> {
const result: Language[] = [];
if (
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES))
)
return result;
const resolveResult = await codeql.betterResolveLanguages();
outer: for (const lang of languages) {
if (resolveResult.extractors[lang].length !== 1) continue;
const extractor = resolveResult.extractors[lang][0];
const trapCacheOptions =
extractor.extractor_options?.trap?.properties?.cache?.properties;
if (trapCacheOptions === undefined) {
logger.info(
`${lang} does not support TRAP caching (missing option group)`
);
continue;
}
for (const requiredOpt of ["dir", "bound", "write"]) {
if (!(requiredOpt in trapCacheOptions)) {
logger.info(
`${lang} does not support TRAP caching (missing ${requiredOpt} option)`
);
continue outer;
}
}
result.push(lang);
}
return result;
}
async function cacheKey(
codeql: CodeQL,
language: Language,
baseSha: string
): Promise<string> {
return `${await cachePrefix(codeql, language)}-${baseSha}`;
}
async function cachePrefix(
codeql: CodeQL,
language: Language
): Promise<string> {
return `codeql-trap-${CACHE_VERSION}-${await codeql.getVersion()}-${language}-`;
}

1
src/util.test.ts
View file

@ -348,6 +348,7 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);