robojerk/codeql-action

Merge branch 'main' into aeisenberg/unrevert-query-filters

Browse source
This commit is contained in:
Andrew Eisenberg 2022-08-10 15:22:40 -07:00
commit 44f42da9ca
1787 changed files with 206897 additions and 1744 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/actions-util.ts
View file

@ -57,13 +57,6 @@ export function getTemporaryDirectory(): string {
: getRequiredEnvParam("RUNNER_TEMP");
}
export function getToolCacheDirectory(): string {
const value = process.env["CODEQL_ACTION_TOOL_CACHE"];
return value !== undefined && value !== ""
? value
: getRequiredEnvParam("RUNNER_TOOL_CACHE");
}
/**
* Gets the SHA of the commit that is currently checked out.
*/
@ -674,12 +667,7 @@ export async function createStatusReportBase(
}
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const codeQlCliVersion = getCachedCodeQlVersion();
// If running locally then the GITHUB_ACTION_REF cannot be trusted as it may be for the previous action
// See https://github.com/actions/runner/issues/803
const actionRef = isRunningLocalAction()
? undefined
: process.env["GITHUB_ACTION_REF"];
const actionRef = process.env["GITHUB_ACTION_REF"];
const statusReport: StatusReportBase = {
workflow_run_id: workflowRunID,

50
src/analysis-paths.test.ts
View file

@ -17,7 +17,6 @@ test("emptyPaths", async (t) => {
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
@ -26,6 +25,7 @@ test("emptyPaths", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@ -43,7 +43,6 @@ test("nonEmptyPaths", async (t) => {
pathsIgnore: ["path4", "path5", "path6/**"],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
@ -52,6 +51,7 @@ test("nonEmptyPaths", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@ -64,28 +64,26 @@ test("nonEmptyPaths", async (t) => {
});
test("exclude temp dir", async (t) => {
return await util.withTmpDir(async (toolCacheDir) => {
const tempDir = path.join(process.cwd(), "codeql-runner-temp");
const config = {
languages: [],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir,
toolCacheDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tempDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
t.is(process.env["LGTM_INDEX_EXCLUDE"], "codeql-runner-temp");
t.is(process.env["LGTM_INDEX_FILTERS"], undefined);
});
const tempDir = path.join(process.cwd(), "codeql-runner-temp");
const config = {
languages: [],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tempDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
t.is(process.env["LGTM_INDEX_EXCLUDE"], "codeql-runner-temp");
t.is(process.env["LGTM_INDEX_FILTERS"], undefined);
});

12
src/analysis-paths.ts
View file

@ -54,17 +54,13 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
}
// If the temporary or tools directory is in the working directory ignore that too.
const tempRelativeToWorking = path.relative(process.cwd(), config.tempDir);
const toolsRelativeToWorking = path.relative(
process.cwd(),
config.toolCacheDir
);
let pathsIgnore = config.pathsIgnore;
if (!tempRelativeToWorking.startsWith("..")) {
if (
!tempRelativeToWorking.startsWith("..") &&
!path.isAbsolute(tempRelativeToWorking)
) {
pathsIgnore = pathsIgnore.concat(tempRelativeToWorking);
}
if (!toolsRelativeToWorking.startsWith("..")) {
pathsIgnore = pathsIgnore.concat(toolsRelativeToWorking);
}
if (pathsIgnore.length !== 0) {
process.env["LGTM_INDEX_EXCLUDE"] = buildIncludeExcludeEnvVar(pathsIgnore);
}

37
src/analyze-action.ts
View file

@ -9,14 +9,17 @@ import {
CodeQLAnalysisError,
QueriesStatusReport,
runCleanup,
runQueries,
runFinalize,
runQueries,
} from "./analyze";
import { getGitHubVersionActionsOnly } from "./api-client";
import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { GitHubFeatureFlags } from "./feature-flags";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { uploadTrapCaches } from "./trap-caching";
import * as upload_lib from "./upload-lib";
import { UploadResult } from "./upload-lib";
import * as util from "./util";
@ -112,7 +115,16 @@ async function run() {
util.getRequiredEnvParam("GITHUB_REPOSITORY")
);
await runFinalize(outputDir, threads, memory, config, logger);
const gitHubVersion = await getGitHubVersionActionsOnly();
const featureFlags = new GitHubFeatureFlags(
gitHubVersion,
apiDetails,
repositoryNwo,
logger
);
await runFinalize(outputDir, threads, memory, config, logger, featureFlags);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await runQueries(
outputDir,
@ -140,17 +152,19 @@ async function run() {
if (config.debugMode) {
// Upload the logs as an Actions artifact for debugging
const toUpload: string[] = [];
let toUpload: string[] = [];
for (const language of config.languages) {
toUpload.push(
...listFolder(
toUpload = toUpload.concat(
listFolder(
path.resolve(util.getCodeQLDatabasePath(config, language), "log")
)
);
}
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Multilanguage tracing: there are additional logs in the root of the cluster
toUpload.push(...listFolder(path.resolve(config.dbLocation, "log")));
toUpload = toUpload.concat(
listFolder(path.resolve(config.dbLocation, "log"))
);
}
await uploadDebugArtifacts(
toUpload,
@ -196,6 +210,9 @@ async function run() {
// Possibly upload the database bundles for remote queries
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
await uploadTrapCaches(codeql, config, logger);
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
core.debug("In test mode. Waiting for processing is disabled.");
@ -225,7 +242,7 @@ async function run() {
return;
} finally {
if (config !== undefined && config.debugMode) {
if (config?.debugMode) {
try {
// Upload the database bundles as an Actions artifact for debugging
const toUpload: string[] = [];
@ -249,7 +266,7 @@ async function run() {
}
}
if (core.isDebug() && config !== undefined) {
if (config?.debugMode) {
core.info("Debug mode is on. Printing CodeQL debug logs...");
for (const language of config.languages) {
const databaseDirectory = util.getCodeQLDatabasePath(config, language);
@ -308,12 +325,12 @@ async function uploadDebugArtifacts(
function listFolder(dir: string): string[] {
const entries = fs.readdirSync(dir, { withFileTypes: true });
const files: string[] = [];
let files: string[] = [];
for (const entry of entries) {
if (entry.isFile()) {
files.push(path.resolve(dir, entry.name));
} else if (entry.isDirectory()) {
files.push(...listFolder(path.resolve(dir, entry.name)));
files = files.concat(listFolder(path.resolve(dir, entry.name)));
}
}
return files;

103
src/analyze.test.ts
View file

@ -7,13 +7,16 @@ import * as sinon from "sinon";
import {
convertPackToQuerySuiteEntry,
createdDBForScannedLanguages,
createQuerySuiteContents,
runQueries,
validateQueryFilters,
} from "./analyze";
import { setCodeQL } from "./codeql";
import { setCodeQL, getCodeQLForTesting } from "./codeql";
import { stubToolRunnerConstructor } from "./codeql.test";
import { Config } from "./config-utils";
import * as count from "./count-loc";
import { createFeatureFlags, FeatureFlag } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars } from "./testing-utils";
@ -106,7 +109,6 @@ test("status report fields and search path setting", async (t) => {
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
@ -117,6 +119,7 @@ test("status report fields and search path setting", async (t) => {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
@ -412,3 +415,99 @@ test("createQuerySuiteContents", (t) => {
t.deepEqual(yamlResult, expected);
});
const stubConfig: Config = {
languages: [Language.cpp, Language.go],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
for (const options of [
{
name: "Lua feature flag enabled, but old CLI",
version: "2.9.0",
featureFlags: [FeatureFlag.LuaTracerConfigEnabled],
yesFlagSet: false,
noFlagSet: false,
},
{
name: "Lua feature flag disabled, with old CLI",
version: "2.9.0",
featureFlags: [],
yesFlagSet: false,
noFlagSet: false,
},
{
name: "Lua feature flag enabled, with new CLI",
version: "2.10.0",
featureFlags: [FeatureFlag.LuaTracerConfigEnabled],
yesFlagSet: true,
noFlagSet: false,
},
{
name: "Lua feature flag disabled, with new CLI",
version: "2.10.0",
featureFlags: [],
yesFlagSet: false,
noFlagSet: true,
},
]) {
test(`createdDBForScannedLanguages() ${options.name}`, async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await getCodeQLForTesting("codeql/for-testing");
sinon.stub(codeqlObject, "getVersion").resolves(options.version);
const promise = createdDBForScannedLanguages(
codeqlObject,
stubConfig,
getRunnerLogger(true),
createFeatureFlags(options.featureFlags)
);
// call listener on `codeql resolve extractor`
const mockToolRunner = runnerConstructorStub.getCall(0);
mockToolRunner.args[2].listeners.stdout('"/path/to/extractor"');
await promise;
if (options.yesFlagSet)
t.true(
runnerConstructorStub.secondCall.args[1].includes(
"--internal-use-lua-tracing"
),
"--internal-use-lua-tracing should be present, but it is absent"
);
else
t.false(
runnerConstructorStub.secondCall.args[1].includes(
"--internal-use-lua-tracing"
),
"--internal-use-lua-tracing should be absent, but it is present"
);
if (options.noFlagSet)
t.true(
runnerConstructorStub.secondCall.args[1].includes(
"--no-internal-use-lua-tracing"
),
"--no-internal-use-lua-tracing should be present, but it is absent"
);
else
t.false(
runnerConstructorStub.secondCall.args[1].includes(
"--no-internal-use-lua-tracing"
),
"--no-internal-use-lua-tracing should be absent, but it is present"
);
});
}

72
src/analyze.ts
View file

@ -7,12 +7,14 @@ import * as yaml from "js-yaml";
import * as analysisPaths from "./analysis-paths";
import {
CodeQL,
CODEQL_VERSION_COUNTS_LINES,
CODEQL_VERSION_NEW_TRACING,
getCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { countLoc } from "./count-loc";
import { FeatureFlags } from "./feature-flags";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
@ -114,15 +116,16 @@ async function setupPythonExtractor(logger: Logger) {
process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
}
async function createdDBForScannedLanguages(
export async function createdDBForScannedLanguages(
codeql: CodeQL,
config: configUtils.Config,
logger: Logger
logger: Logger,
featureFlags: FeatureFlags
) {
// Insert the LGTM_INDEX_X env vars at this point so they are set when
// we extract any scanned languages.
analysisPaths.includeAndExcludeAnalysisPaths(config);
const codeql = await getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (
isScannedLanguage(language) &&
@ -134,10 +137,7 @@ async function createdDBForScannedLanguages(
await setupPythonExtractor(logger);
}
await codeql.extractScannedLanguage(
util.getCodeQLDatabasePath(config, language),
language
);
await codeql.extractScannedLanguage(config, language, featureFlags);
logger.endGroup();
}
}
@ -166,11 +166,12 @@ async function finalizeDatabaseCreation(
config: configUtils.Config,
threadsFlag: string,
memoryFlag: string,
logger: Logger
logger: Logger,
featureFlags: FeatureFlags
) {
await createdDBForScannedLanguages(config, logger);
const codeql = await getCodeQL(config.codeQLCmd);
await createdDBForScannedLanguages(codeql, config, logger, featureFlags);
for (const language of config.languages) {
if (dbIsFinalized(config, language, logger)) {
logger.info(
@ -204,11 +205,9 @@ export async function runQueries(
{}
);
const cliCanCountBaseline = await cliCanCountLoC();
const debugMode =
process.env["INTERNAL_CODEQL_ACTION_DEBUG_LOC"] ||
process.env["ACTIONS_RUNNER_DEBUG"] ||
process.env["ACTIONS_STEP_DEBUG"];
if (!cliCanCountBaseline || debugMode) {
const countLocDebugMode =
process.env["INTERNAL_CODEQL_ACTION_DEBUG_LOC"] || config.debugMode;
if (!cliCanCountBaseline || countLocDebugMode) {
// count the number of lines in the background
locPromise = countLoc(
path.resolve(),
@ -306,7 +305,8 @@ export async function runQueries(
const analysisSummary = await runInterpretResults(
language,
querySuitePaths,
sarifFile
sarifFile,
config.debugMode
);
if (!cliCanCountBaseline)
await injectLinesOfCode(sarifFile, language, locPromise);
@ -314,7 +314,7 @@ export async function runQueries(
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
if (!cliCanCountBaseline || debugMode)
if (!cliCanCountBaseline || countLocDebugMode)
printLinesOfCodeSummary(logger, language, await locPromise);
if (cliCanCountBaseline) logger.info(await runPrintLinesOfCode(language));
} catch (e) {
@ -335,7 +335,8 @@ export async function runQueries(
async function runInterpretResults(
language: Language,
queries: string[],
sarifFile: string
sarifFile: string,
enableDebugLogging: boolean
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = await getCodeQL(config.codeQLCmd);
@ -345,6 +346,7 @@ export async function runQueries(
sarifFile,
addSnippetsFlag,
threadsFlag,
enableDebugLogging ? "-vv" : "-v",
automationDetailsId
);
}
@ -455,17 +457,9 @@ export async function runFinalize(
threadsFlag: string,
memoryFlag: string,
config: configUtils.Config,
logger: Logger
logger: Logger,
featureFlags: FeatureFlags
) {
const codeql = await getCodeQL(config.codeQLCmd);
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Delete variables as specified by the end-tracing script
await endTracingForCluster(config);
} else {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
}
try {
await del(outputDir, { force: true });
} catch (error: any) {
@ -475,7 +469,27 @@ export async function runFinalize(
}
await fs.promises.mkdir(outputDir, { recursive: true });
await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
await finalizeDatabaseCreation(
config,
threadsFlag,
memoryFlag,
logger,
featureFlags
);
const codeql = await getCodeQL(config.codeQLCmd);
// WARNING: This does not _really_ end tracing, as the tracer will restore its
// critical environment variables and it'll still be active for all processes
// launched from this build step.
// However, it will stop tracing for all steps past the codeql-action/analyze
// step.
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Delete variables as specified by the end-tracing script
await endTracingForCluster(config);
} else {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
}
}
export async function runCleanup(

18
src/codeql.test.ts
View file

@ -52,7 +52,6 @@ test("download codeql bundle cache", async (t) => {
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -82,7 +81,6 @@ test("download codeql bundle cache explicitly requested with pinned different ve
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -101,7 +99,6 @@ test("download codeql bundle cache explicitly requested with pinned different ve
"https://example.com/download/codeql-bundle-20200610/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -126,7 +123,6 @@ test("don't download codeql bundle cache with pinned different version cached",
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -138,7 +134,6 @@ test("don't download codeql bundle cache with pinned different version cached",
undefined,
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -165,7 +160,6 @@ test("download codeql bundle cache with different version cached (not pinned)",
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -192,7 +186,6 @@ test("download codeql bundle cache with different version cached (not pinned)",
undefined,
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -219,7 +212,6 @@ test('download codeql bundle cache with pinned different version cached if "late
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -247,7 +239,6 @@ test('download codeql bundle cache with pinned different version cached if "late
"latest",
sampleApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
@ -302,7 +293,6 @@ test("download codeql bundle from github ae endpoint", async (t) => {
undefined,
sampleGHAEApiDetails,
tmpDir,
tmpDir,
util.GitHubVariant.GHAE,
getRunnerLogger(true),
false
@ -410,7 +400,7 @@ test("databaseInterpretResults() does not set --sarif-add-query-help for 2.7.0",
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.7.0");
await codeqlObject.databaseInterpretResults("", [], "", "", "", "");
await codeqlObject.databaseInterpretResults("", [], "", "", "", "-v", "");
t.false(
runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"),
"--sarif-add-query-help should be absent, but it is present"
@ -421,7 +411,7 @@ test("databaseInterpretResults() sets --sarif-add-query-help for 2.7.1", async (
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.7.1");
await codeqlObject.databaseInterpretResults("", [], "", "", "", "");
await codeqlObject.databaseInterpretResults("", [], "", "", "", "-v", "");
t.true(
runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"),
"--sarif-add-query-help should be present, but it is absent"
@ -435,7 +425,6 @@ const stubConfig: Config = {
paths: [],
originalUserInput: {},
tempDir: "",
toolCacheDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
@ -446,6 +435,7 @@ const stubConfig: Config = {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
test("databaseInitCluster() Lua feature flag enabled, but old CLI", async (t) => {
@ -540,7 +530,7 @@ test("databaseInitCluster() Lua feature flag disabled, compatible CLI", async (t
);
});
function stubToolRunnerConstructor(): sinon.SinonStub<
export function stubToolRunnerConstructor(): sinon.SinonStub<
any[],
toolrunner.ToolRunner
> {

149
src/codeql.ts
View file

@ -3,20 +3,25 @@ import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import { default as queryString } from "query-string";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
import { getRelativeScriptPath, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { FeatureFlags, FeatureFlag } from "./feature-flags";
import { FeatureFlag, FeatureFlags } from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as toolcache from "./toolcache";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import {
getTrapCachingExtractorConfigArgs,
getTrapCachingExtractorConfigArgsForLang,
} from "./trap-caching";
import * as util from "./util";
import { isGoodVersion } from "./util";
@ -95,7 +100,11 @@ export interface CodeQL {
* Extract code for a scanned language using 'codeql database trace-command'
* and running the language extractor.
*/
extractScannedLanguage(database: string, language: Language): Promise<void>;
extractScannedLanguage(
config: Config,
language: Language,
featureFlags: FeatureFlags
): Promise<void>;
/**
* Finalize a database using 'codeql database finalize'.
*/
@ -108,6 +117,10 @@ export interface CodeQL {
* Run 'codeql resolve languages'.
*/
resolveLanguages(): Promise<ResolveLanguagesOutput>;
/**
* Run 'codeql resolve languages' with '--format=betterjson'.
*/
betterResolveLanguages(): Promise<BetterResolveLanguagesOutput>;
/**
* Run 'codeql resolve queries'.
*/
@ -152,6 +165,7 @@ export interface CodeQL {
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
verbosityFlag: string | undefined,
automationDetailsId: string | undefined
): Promise<string>;
/**
@ -164,6 +178,17 @@ export interface ResolveLanguagesOutput {
[language: string]: [string];
}
export interface BetterResolveLanguagesOutput {
extractors: {
[language: string]: [
{
extractor_root: string;
extractor_options?: any;
}
];
};
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
@ -242,6 +267,12 @@ export const CODEQL_VERSION_NEW_TRACING = "2.7.0";
*/
export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
/**
* Previous versions had the option already, but were missing the
* --extractor-options-verbosity that we need.
*/
export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
function getCodeQLBundleName(): string {
let platform: string;
if (process.platform === "win32") {
@ -381,7 +412,6 @@ async function getCodeQLBundleDownloadURL(
* @param codeqlURL
* @param apiDetails
* @param tempDir
* @param toolCacheDir
* @param variant
* @param logger
* @param checkVersion Whether to check that CodeQL CLI meets the minimum
@ -392,7 +422,6 @@ export async function setupCodeQL(
codeqlURL: string | undefined,
apiDetails: api.GitHubApiDetails,
tempDir: string,
toolCacheDir: string,
variant: util.GitHubVariant,
logger: Logger,
checkVersion: boolean
@ -407,7 +436,7 @@ export async function setupCodeQL(
let codeqlFolder: string;
let codeqlURLVersion: string;
if (codeqlURL && !codeqlURL.startsWith("http")) {
codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
codeqlFolder = await toolcache.extractTar(codeqlURL);
codeqlURLVersion = "local";
} else {
codeqlURLVersion = getCodeQLURLVersion(
@ -416,29 +445,15 @@ export async function setupCodeQL(
const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
// If we find the specified version, we always use that.
codeqlFolder = toolcache.find(
"CodeQL",
codeqlURLSemVer,
toolCacheDir,
logger
);
codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer);
// If we don't find the requested version, in some cases we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (!codeqlFolder && !codeqlURL && !forceLatest) {
const codeqlVersions = toolcache.findAllVersions(
"CodeQL",
toolCacheDir,
logger
);
const codeqlVersions = toolcache.findAllVersions("CodeQL");
if (codeqlVersions.length === 1 && isGoodVersion(codeqlVersions[0])) {
const tmpCodeqlFolder = toolcache.find(
"CodeQL",
codeqlVersions[0],
toolCacheDir,
logger
);
const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0]);
if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
logger.debug(
`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
@ -480,24 +495,25 @@ export async function setupCodeQL(
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const dest = path.join(tempDir, uuidV4());
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers
);
const codeqlPath = await toolcache.downloadTool(
codeqlURL,
tempDir,
headers
dest,
undefined,
finalHeaders
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(
codeqlPath,
tempDir,
logger
);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(
codeqlExtracted,
"CodeQL",
codeqlURLSemVer,
toolCacheDir,
logger
codeqlURLSemVer
);
}
}
@ -594,6 +610,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
),
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
betterResolveLanguages: resolveFunction(
partialCodeql,
"betterResolveLanguages"
),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
@ -630,8 +650,10 @@ export function getCachedCodeQL(): CodeQL {
* a non-existent placeholder codeql command, so tests that use this function
* should also stub the toolrunner.ToolRunner constructor.
*/
export async function getCodeQLForTesting(): Promise<CodeQL> {
return getCodeQLForCmd("codeql-for-testing", false);
export async function getCodeQLForTesting(
cmd = "codeql-for-testing"
): Promise<CodeQL> {
return getCodeQLForCmd(cmd, false);
}
/**
@ -737,6 +759,7 @@ async function getCodeQLForCmd(
);
if (config.languages.filter(isTracedLanguage).length > 0) {
extraArgs.push("--begin-tracing");
extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
if (processName !== undefined) {
extraArgs.push(`--trace-process-name=${processName}`);
} else {
@ -787,9 +810,28 @@ async function getCodeQLForCmd(
"-Dmaven.wagon.http.pool=false",
].join(" ");
// On macOS, System Integrity Protection (SIP) typically interferes with
// CodeQL build tracing of protected binaries.
// The usual workaround is to prefix `$CODEQL_RUNNER` to build commands:
// `$CODEQL_RUNNER` (not to be confused with the deprecated CodeQL Runner tool)
// points to a simple wrapper binary included with the CLI, and the extra layer of
// process indirection helps the tracer bypass SIP.
// The above SIP workaround is *not* needed here.
// At the `autobuild` step in the Actions workflow, we assume the `init` step
// has successfully run, and will have exported `DYLD_INSERT_LIBRARIES`
// into the environment of subsequent steps, to activate the tracer.
// When `DYLD_INSERT_LIBRARIES` is set in the environment for a step,
// the Actions runtime introduces its own workaround for SIP
// (https://github.com/actions/runner/pull/416).
await runTool(autobuildCmd);
},
async extractScannedLanguage(databasePath: string, language: Language) {
async extractScannedLanguage(
config: Config,
language: Language,
featureFlags: FeatureFlags
) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Get extractor location
let extractorPath = "";
await new toolrunner.ToolRunner(
@ -821,6 +863,16 @@ async function getCodeQLForCmd(
"tools",
`autobuild${ext}`
);
const extraArgs: string[] = [];
if (
await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)
) {
if (await featureFlags.getValue(FeatureFlag.LuaTracerConfigEnabled)) {
extraArgs.push("--internal-use-lua-tracing");
} else {
extraArgs.push("--no-internal-use-lua-tracing");
}
}
// Run trace command
await toolrunnerErrorCatcher(
@ -828,6 +880,8 @@ async function getCodeQLForCmd(
[
"database",
"trace-command",
...extraArgs,
...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
...getExtraOptionsFromEnv(["database", "trace-command"]),
databasePath,
"--",
@ -870,6 +924,24 @@ async function getCodeQLForCmd(
);
}
},
async betterResolveLanguages() {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
try {
return JSON.parse(output);
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`
);
}
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined
@ -921,6 +993,7 @@ async function getCodeQLForCmd(
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
verbosityFlag: string,
automationDetailsId: string | undefined
): Promise<string> {
const codeqlArgs = [
@ -928,7 +1001,7 @@ async function getCodeQLForCmd(
"interpret-results",
threadsFlag,
"--format=sarif-latest",
"-v",
verbosityFlag,
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "interpret-results"]),

147
src/config-utils.test.ts
View file

@ -86,11 +86,11 @@ test("load empty config", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -107,11 +107,11 @@ test("load empty config", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -153,11 +153,11 @@ test("loading config saves config", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -188,11 +188,11 @@ test("load input outside of workspace", async (t) => {
"../input",
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
@ -227,11 +227,11 @@ test("load non-local input with invalid repo syntax", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
@ -267,11 +267,11 @@ test("load non-existent input", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
@ -348,7 +348,6 @@ test("load non-empty input", async (t) => {
paths: ["c/d"],
},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
@ -357,6 +356,7 @@ test("load non-empty input", async (t) => {
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
injectedMlQueries: false,
trapCaches: {},
};
const languages = "javascript";
@ -369,11 +369,11 @@ test("load non-empty input", async (t) => {
configFilePath,
undefined,
false,
false,
"my-artifact",
"my-db",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -436,11 +436,11 @@ test("Default queries are used", async (t) => {
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -511,11 +511,11 @@ test("Queries can be specified in config file", async (t) => {
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -529,16 +529,21 @@ test("Queries can be specified in config file", async (t) => {
// and once for `./foo` from the config file.
t.deepEqual(resolveQueriesArgs.length, 2);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/foo$/);
t.true(resolveQueriesArgs[1].queries[0].endsWith(`${path.sep}foo`));
// Now check that the end result contains the default queries and the query from config
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
t.true(
config.queries["javascript"].builtin[0].endsWith(
"javascript-code-scanning.qls"
)
);
t.true(
config.queries["javascript"].custom[0].queries[0].endsWith(
`${path.sep}foo`
)
);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/foo$/);
});
});
@ -580,11 +585,11 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -598,16 +603,21 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
// but won't be called for './foo' from the config file.
t.deepEqual(resolveQueriesArgs.length, 2);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override$/);
t.true(resolveQueriesArgs[1].queries[0].endsWith(`${path.sep}override`));
// Now check that the end result contains only the default queries and the override query
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
t.true(
config.queries["javascript"].builtin[0].endsWith(
"javascript-code-scanning.qls"
)
);
t.true(
config.queries["javascript"].custom[0].queries[0].endsWith(
`${path.sep}override`
)
);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/override$/);
});
});
@ -647,11 +657,11 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -665,14 +675,17 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
// but won't be called for the default one since that was disabled
t.deepEqual(resolveQueriesArgs.length, 1);
t.deepEqual(resolveQueriesArgs[0].queries.length, 1);
t.regex(resolveQueriesArgs[0].queries[0], /.*\/workflow-query$/);
t.true(
resolveQueriesArgs[0].queries[0].endsWith(`${path.sep}workflow-query`)
);
// Now check that the end result contains only the workflow query, and not the default one
t.deepEqual(config.queries["javascript"].builtin.length, 0);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/workflow-query$/
t.true(
config.queries["javascript"].custom[0].queries[0].endsWith(
`${path.sep}workflow-query`
)
);
});
});
@ -707,11 +720,11 @@ test("Multiple queries can be specified in workflow file, no config file require
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -726,23 +739,26 @@ test("Multiple queries can be specified in workflow file, no config file require
t.deepEqual(resolveQueriesArgs.length, 3);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.deepEqual(resolveQueriesArgs[2].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override1$/);
t.regex(resolveQueriesArgs[2].queries[0], /.*\/override2$/);
t.true(resolveQueriesArgs[1].queries[0].endsWith(`${path.sep}override1`));
t.true(resolveQueriesArgs[2].queries[0].endsWith(`${path.sep}override2`));
// Now check that the end result contains both the queries from the workflow, as well as the defaults
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 2);
t.regex(
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
t.true(
config.queries["javascript"].builtin[0].endsWith(
"javascript-code-scanning.qls"
)
);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/override1$/
t.true(
config.queries["javascript"].custom[0].queries[0].endsWith(
`${path.sep}override1`
)
);
t.regex(
config.queries["javascript"].custom[1].queries[0],
/.*\/override2$/
t.true(
config.queries["javascript"].custom[1].queries[0].endsWith(
`${path.sep}override2`
)
);
});
});
@ -788,11 +804,11 @@ test("Queries in workflow file can be added to the set of queries without overri
configFilePath,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -807,28 +823,35 @@ test("Queries in workflow file can be added to the set of queries without overri
// and once for './foo' from the config file
t.deepEqual(resolveQueriesArgs.length, 4);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/additional1$/);
t.true(resolveQueriesArgs[1].queries[0].endsWith(`${path.sep}additional1`));
t.deepEqual(resolveQueriesArgs[2].queries.length, 1);
t.regex(resolveQueriesArgs[2].queries[0], /.*\/additional2$/);
t.true(resolveQueriesArgs[2].queries[0].endsWith(`${path.sep}additional2`));
t.deepEqual(resolveQueriesArgs[3].queries.length, 1);
t.regex(resolveQueriesArgs[3].queries[0], /.*\/foo$/);
t.true(resolveQueriesArgs[3].queries[0].endsWith(`${path.sep}foo`));
// Now check that the end result contains all the queries
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 3);
t.regex(
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
t.true(
config.queries["javascript"].builtin[0].endsWith(
"javascript-code-scanning.qls"
)
);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/additional1$/
t.true(
config.queries["javascript"].custom[0].queries[0].endsWith(
`${path.sep}additional1`
)
);
t.regex(
config.queries["javascript"].custom[1].queries[0],
/.*\/additional2$/
t.true(
config.queries["javascript"].custom[1].queries[0].endsWith(
`${path.sep}additional2`
)
);
t.true(
config.queries["javascript"].custom[2].queries[0].endsWith(
`${path.sep}foo`
)
);
t.regex(config.queries["javascript"].custom[2].queries[0], /.*\/foo$/);
});
});
@ -859,11 +882,11 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -927,11 +950,11 @@ test("API client used when reading remote config", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -957,11 +980,11 @@ test("Remote config handles the case where a directory is provided", async (t) =
repoReference,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
@ -995,11 +1018,11 @@ test("Invalid format of remote config handled correctly", async (t) => {
repoReference,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
@ -1034,11 +1057,11 @@ test("No detected languages", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -1065,11 +1088,11 @@ test("Unknown languages", async (t) => {
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
@ -1118,11 +1141,11 @@ test("Config specifies packages", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -1175,11 +1198,11 @@ test("Config specifies packages for multiple languages", async (t) => {
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -1243,11 +1266,11 @@ function doInvalidInputTest(
configFile,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
@ -1820,11 +1843,11 @@ const mlPoweredQueriesMacro = test.macro({
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,

37
src/config-utils.ts
View file

@ -16,6 +16,7 @@ import { FeatureFlag, FeatureFlags } from "./feature-flags";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { downloadTrapCaches } from "./trap-caching";
import {
codeQlVersionAbove,
getMlPoweredJsQueriesPack,
@ -144,11 +145,6 @@ export interface Config {
* deleted at the end of the job.
*/
tempDir: string;
/**
* Directory to use for the tool cache.
* This may be persisted between jobs but this is not guaranteed.
*/
toolCacheDir: string;
/**
* Path of the CodeQL executable.
*/
@ -183,6 +179,11 @@ export interface Config {
* Whether we injected ML queries into this configuration.
*/
injectedMlQueries: boolean;
/**
* Partial map from languages to locations of TRAP caches for that language.
* If a key is omitted, then TRAP caching should not be used for that language.
*/
trapCaches: Partial<Record<Language, string>>;
}
export type Packs = Partial<Record<Language, string[]>>;
@ -915,12 +916,12 @@ export async function getDefaultConfig(
queriesInput: string | undefined,
packsInput: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: GitHubVersion,
@ -968,7 +969,6 @@ export async function getDefaultConfig(
packs,
originalUserInput: {},
tempDir,
toolCacheDir,
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
@ -976,6 +976,9 @@ export async function getDefaultConfig(
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
};
}
@ -988,12 +991,12 @@ async function loadConfig(
packsInput: string | undefined,
configFile: string,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: GitHubVersion,
@ -1150,7 +1153,6 @@ async function loadConfig(
packs,
originalUserInput: parsedYAML,
tempDir,
toolCacheDir,
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
@ -1158,6 +1160,9 @@ async function loadConfig(
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
};
}
@ -1316,7 +1321,13 @@ export function parsePacksSpecification(
if (
packPath &&
(path.isAbsolute(packPath) || path.normalize(packPath) !== packPath)
(path.isAbsolute(packPath) ||
// Permit using "/" instead of "\" on Windows
// Use `x.split(y).join(z)` as a polyfill for `x.replaceAll(y, z)` since
// if we used a regex we'd need to escape the path separator on Windows
// which seems more awkward.
path.normalize(packPath).split(path.sep).join("/") !==
packPath.split(path.sep).join("/"))
) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
@ -1405,12 +1416,12 @@ export async function initConfig(
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: GitHubVersion,
@ -1428,12 +1439,12 @@ export async function initConfig(
queriesInput,
packsInput,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
toolCacheDir,
codeQL,
workspacePath,
gitHubVersion,
@ -1448,12 +1459,12 @@ export async function initConfig(
packsInput,
configFile,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
toolCacheDir,
codeQL,
workspacePath,
gitHubVersion,

2
src/database-upload.test.ts
View file

@ -48,7 +48,6 @@ function getTestConfig(tmpDir: string): Config {
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "foo",
gitHubVersion: { type: GitHubVariant.DOTCOM },
dbLocation: tmpDir,
@ -57,6 +56,7 @@ function getTestConfig(tmpDir: string): Config {
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
}

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20220623"
"bundleVersion": "codeql-bundle-20220728"
}

31
src/feature-flags.test.ts
View file

@ -67,6 +67,37 @@ for (const variant of ALL_FEATURE_FLAGS_DISABLED_VARIANTS) {
});
}
test("API response missing", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const loggedMessages = [];
const featureFlags = new GitHubFeatureFlags(
{ type: GitHubVariant.DOTCOM },
testApiDetails,
testRepositoryNwo,
getRecordingLogger(loggedMessages)
);
mockFeatureFlagApiEndpoint(403, {});
for (const flag of Object.values(FeatureFlag)) {
t.assert((await featureFlags.getValue(flag)) === false);
}
for (const featureFlag of ["ml_powered_queries_enabled"]) {
t.assert(
loggedMessages.find(
(v: LoggedMessage) =>
v.type === "debug" &&
v.message ===
`No feature flags API response for ${featureFlag}, considering it disabled.`
) !== undefined
);
}
});
});
test("Feature flags are disabled if they're not returned in API response", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);

12
src/feature-flags.ts
View file

@ -10,6 +10,7 @@ export interface FeatureFlags {
export enum FeatureFlag {
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
LuaTracerConfigEnabled = "lua_tracer_config_enabled",
TrapCachingEnabled = "trap_caching_enabled",
}
/**
@ -31,14 +32,21 @@ export class GitHubFeatureFlags implements FeatureFlags {
) {}
async getValue(flag: FeatureFlag): Promise<boolean> {
const response = (await this.getApiResponse())[flag];
const response = await this.getApiResponse();
if (response === undefined) {
this.logger.debug(
`No feature flags API response for ${flag}, considering it disabled.`
);
return false;
}
const flagValue = response[flag];
if (flagValue === undefined) {
this.logger.debug(
`Feature flag '${flag}' undefined in API response, considering it disabled.`
);
return false;
}
return response;
return flagValue;
}
private async getApiResponse(): Promise<FeatureFlagsApiResponse> {

20
src/fingerprints.test.ts
View file

@ -139,19 +139,27 @@ test("resolveUriToFile", (t) => {
// so we need to give it real files to look at. We will use this file as an example.
// For this to work we require the current working directory to be a parent, but this
// should generally always be the case so this is fine.
const cwd = process.cwd();
const filepath = __filename;
t.true(filepath.startsWith(`${cwd}/`));
const relativeFilepath = filepath.substring(cwd.length + 1);
const filepath = __filename.split(path.sep).join("/");
const relativeFilepath = path
.relative(process.cwd(), __filename)
.split(path.sep)
.join("/");
// Absolute paths are unmodified
t.is(testResolveUriToFile(filepath, undefined, []), filepath);
t.is(testResolveUriToFile(`file://${filepath}`, undefined, []), filepath);
// Relative paths are made absolute
t.is(testResolveUriToFile(relativeFilepath, undefined, []), filepath);
t.is(
testResolveUriToFile(`file://${relativeFilepath}`, undefined, []),
testResolveUriToFile(relativeFilepath, undefined, [])
?.split(path.sep)
.join("/"),
filepath
);
t.is(
testResolveUriToFile(`file://${relativeFilepath}`, undefined, [])
?.split(path.sep)
.join("/"),
filepath
);

3
src/fingerprints.ts
View file

@ -1,4 +1,5 @@
import * as fs from "fs";
import path from "path";
import Long from "long";
@ -226,7 +227,7 @@ export function resolveUriToFile(
// Just assume a relative path is relative to the src root.
// This is not necessarily true but should be a good approximation
// and here we likely want to err on the side of handling more cases.
if (!uri.startsWith("/")) {
if (!path.isAbsolute(uri)) {
uri = srcRootPrefix + uri;
}

34
src/init-action.ts
View file

@ -8,7 +8,6 @@ import {
getOptionalInput,
getRequiredInput,
getTemporaryDirectory,
getToolCacheDirectory,
sendStatusReport,
StatusReportBase,
validateWorkflow,
@ -16,7 +15,7 @@ import {
import { getGitHubVersionActionsOnly } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { GitHubFeatureFlags } from "./feature-flags";
import { FeatureFlag, FeatureFlags, GitHubFeatureFlags } from "./feature-flags";
import {
initCodeQL,
initConfig,
@ -28,18 +27,18 @@ import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
getRequiredEnvParam,
initializeEnvironment,
Mode,
checkActionVersion,
checkGitHubVersionInRange,
codeQlVersionAbove,
enrichEnvironment,
getMemoryFlagValue,
getThreadsFlagValue,
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
enrichEnvironment,
getMemoryFlagValue,
getMlPoweredJsQueriesStatus,
checkActionVersion,
getRequiredEnvParam,
getThreadsFlagValue,
initializeEnvironment,
Mode,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -171,7 +170,6 @@ async function run() {
getOptionalInput("tools"),
apiDetails,
getTemporaryDirectory(),
getToolCacheDirectory(),
gitHubVersion.type,
logger
);
@ -185,12 +183,16 @@ async function run() {
getOptionalInput("packs"),
getOptionalInput("config-file"),
getOptionalInput("db-location"),
getOptionalInput("debug") === "true",
await getTrapCachingEnabled(featureFlags),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
// or by setting the `ACTIONS_STEP_DEBUG` secret to `true`).
getOptionalInput("debug") === "true" || core.isDebug(),
getOptionalInput("debug-artifact-name") || DEFAULT_DEBUG_ARTIFACT_NAME,
getOptionalInput("debug-database-name") || DEFAULT_DEBUG_DATABASE_NAME,
repositoryNwo,
getTemporaryDirectory(),
getRequiredEnvParam("RUNNER_TOOL_CACHE"),
codeql,
getRequiredEnvParam("GITHUB_WORKSPACE"),
gitHubVersion,
@ -298,6 +300,14 @@ async function run() {
await sendSuccessStatusReport(startedAt, config, toolsVersion);
}
async function getTrapCachingEnabled(
featureFlags: FeatureFlags
): Promise<boolean> {
const trapCaching = getOptionalInput("trap-caching");
if (trapCaching !== undefined) return trapCaching === "true";
return await featureFlags.getValue(FeatureFlag.TrapCachingEnabled);
}
async function runWrapper() {
try {
await run();

11
src/init.ts
View file

@ -19,7 +19,6 @@ export async function initCodeQL(
codeqlURL: string | undefined,
apiDetails: GitHubApiDetails,
tempDir: string,
toolCacheDir: string,
variant: util.GitHubVariant,
logger: Logger
): Promise<{ codeql: CodeQL; toolsVersion: string }> {
@ -28,7 +27,6 @@ export async function initCodeQL(
codeqlURL,
apiDetails,
tempDir,
toolCacheDir,
variant,
logger,
true
@ -44,12 +42,12 @@ export async function initConfig(
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: util.GitHubVersion,
@ -64,12 +62,12 @@ export async function initConfig(
packsInput,
configFile,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
toolCacheDir,
codeQL,
workspacePath,
gitHubVersion,
@ -253,11 +251,14 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("py"), [
"-3",
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
} else {
await new toolrunner.ToolRunner(path.join(scriptsFolder, script), [
await new toolrunner.ToolRunner(await safeWhich.safeWhich("python3"), [
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
}

22
src/runner.ts
View file

@ -1,5 +1,4 @@
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import { Command } from "commander";
@ -47,14 +46,6 @@ function getTempDir(userInput: string | undefined): string {
return tempDir;
}
function getToolsDir(userInput: string | undefined): string {
const toolsDir = userInput || path.join(os.homedir(), "codeql-runner-tools");
if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true });
}
return toolsDir;
}
const codeqlEnvJsonFilename = "codeql-env.json";
function loadTracerEnvironment(config: Config): { [name: string]: string } {
@ -194,7 +185,6 @@ program
try {
const tempDir = getTempDir(cmd.tempDir);
const toolsDir = getToolsDir(cmd.toolsDir);
const checkoutPath = cmd.checkoutPath || process.cwd();
// Wipe the temp dir
@ -237,7 +227,6 @@ program
undefined,
apiDetails,
tempDir,
toolsDir,
gitHubVersion.type,
logger
)
@ -252,11 +241,11 @@ program
cmd.configFile,
undefined,
false,
false,
"",
"",
parseRepositoryNwo(cmd.repository),
tempDir,
toolsDir,
codeql,
workspacePath,
gitHubVersion,
@ -501,7 +490,14 @@ program
logger
);
const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
await runFinalize(outputDir, threads, memory, config, logger);
await runFinalize(
outputDir,
threads,
memory,
config,
logger,
createFeatureFlags([])
);
await runQueries(
outputDir,
memory,

10
src/testing-utils.ts
View file

@ -63,6 +63,16 @@ export function setupTests(test: TestFn<any>) {
t.context.stderrWrite = processStderrWrite;
process.stderr.write = wrapOutput(t.context) as any;
// Workaround an issue in tests where the case insensitivity of the `$PATH`
// environment variable on Windows isn't preserved, i.e. `process.env.PATH`
// is not the same as `process.env.Path`.
const pathKeys = Object.keys(process.env).filter(
(k) => k.toLowerCase() === "path"
);
if (pathKeys.length > 0) {
process.env.PATH = process.env[pathKeys[0]];
}
// Many tests modify environment variables. Take a copy now so that
// we reset them after the test to keep tests independent of each other.
// process.env only has strings fields, so a shallow copy is fine.

337
src/toolcache.ts
View file

@ -1,337 +0,0 @@
import * as fs from "fs";
import { OutgoingHttpHeaders } from "http";
import * as os from "os";
import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as io from "@actions/io";
import * as actionsToolcache from "@actions/tool-cache";
import * as safeWhich from "@chrisgavin/safe-which";
import del from "del";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { Logger } from "./logging";
import { isActions } from "./util";
/*
* This file acts as an interface to the functionality of the actions toolcache.
* That library is not safe to use outside of actions as it makes assumptions about
* the state of the filesystem and available environment variables.
*
* On actions we can just delegate to the toolcache library, however outside of
* actions we provide our own implementation.
*/
/**
* Extract a compressed tar archive.
*
* See extractTar function from node_modules/@actions/tool-cache/lib/tool-cache.d.ts
*
* @param file path to the tar
* @param mode should run the actions or runner implementation
* @param tempDir path to the temporary directory
* @param logger logger to use
* @returns path to the destination directory
*/
export async function extractTar(
file: string,
tempDir: string,
logger: Logger
): Promise<string> {
if (isActions()) {
return await actionsToolcache.extractTar(file);
} else {
// Initial implementation copied from node_modules/@actions/tool-cache/lib/tool-cache.js
if (!file) {
throw new Error("parameter 'file' is required");
}
// Create dest
const dest = createExtractFolder(tempDir);
// Determine whether GNU tar
logger.debug("Checking tar --version");
let versionOutput = "";
await new toolrunner.ToolRunner(
await safeWhich.safeWhich("tar"),
["--version"],
{
ignoreReturnCode: true,
silent: true,
listeners: {
stdout: (data) => (versionOutput += data.toString()),
stderr: (data) => (versionOutput += data.toString()),
},
}
).exec();
logger.debug(versionOutput.trim());
const isGnuTar = versionOutput.toUpperCase().includes("GNU TAR");
// Initialize args
const args = ["xz"];
if (logger.isDebug()) {
args.push("-v");
}
let destArg = dest;
let fileArg = file;
if (process.platform === "win32" && isGnuTar) {
args.push("--force-local");
destArg = dest.replace(/\\/g, "/");
// Technically only the dest needs to have `/` but for aesthetic consistency
// convert slashes in the file arg too.
fileArg = file.replace(/\\/g, "/");
}
if (isGnuTar) {
// Suppress warnings when using GNU tar to extract archives created by BSD tar
args.push("--warning=no-unknown-keyword");
}
args.push("-C", destArg, "-f", fileArg);
await new toolrunner.ToolRunner(`tar`, args).exec();
return dest;
}
}
/**
* Caches a directory and installs it into the tool cacheDir.
*
* Also see cacheDir function from node_modules/@actions/tool-cache/lib/tool-cache.d.ts
*
* @param sourceDir the directory to cache into tools
* @param tool tool name
* @param version version of the tool. semver format
* @param mode should run the actions or runner implementation
* @param toolCacheDir path to the tool cache directory
* @param logger logger to use
*/
export async function cacheDir(
sourceDir: string,
tool: string,
version: string,
toolCacheDir: string,
logger: Logger
): Promise<string> {
if (isActions()) {
return await actionsToolcache.cacheDir(sourceDir, tool, version);
} else {
// Initial implementation copied from node_modules/@actions/tool-cache/lib/tool-cache.js
version = semver.clean(version) || version;
const arch = os.arch();
logger.debug(`Caching tool ${tool} ${version} ${arch}`);
logger.debug(`source dir: ${sourceDir}`);
if (!fs.statSync(sourceDir).isDirectory()) {
throw new Error("sourceDir is not a directory");
}
// Create the tool dir
const destPath = await createToolPath(
tool,
version,
arch,
toolCacheDir,
logger
);
// copy each child item. do not move. move can fail on Windows
// due to anti-virus software having an open handle on a file.
for (const itemName of fs.readdirSync(sourceDir)) {
const s = path.join(sourceDir, itemName);
await io.cp(s, destPath, { recursive: true });
}
// write .complete
completeToolPath(tool, version, arch, toolCacheDir, logger);
return destPath;
}
}
/**
* Finds the path to a tool version in the local installed tool cache.
*
* Also see find function from node_modules/@actions/tool-cache/lib/tool-cache.d.ts
*
* @param toolName name of the tool
* @param versionSpec version of the tool
* @param mode should run the actions or runner implementation
* @param toolCacheDir path to the tool cache directory
* @param logger logger to use
*/
export function find(
toolName: string,
versionSpec: string,
toolCacheDir: string,
logger: Logger
): string {
if (isActions()) {
return actionsToolcache.find(toolName, versionSpec);
} else {
// Initial implementation copied from node_modules/@actions/tool-cache/lib/tool-cache.js
if (!toolName) {
throw new Error("toolName parameter is required");
}
if (!versionSpec) {
throw new Error("versionSpec parameter is required");
}
const arch = os.arch();
// attempt to resolve an explicit version
if (!isExplicitVersion(versionSpec, logger)) {
const localVersions = findAllVersions(toolName, toolCacheDir, logger);
const match = evaluateVersions(localVersions, versionSpec, logger);
versionSpec = match;
}
// check for the explicit version in the cache
let toolPath = "";
if (versionSpec) {
versionSpec = semver.clean(versionSpec) || "";
const cachePath = path.join(toolCacheDir, toolName, versionSpec, arch);
logger.debug(`checking cache: ${cachePath}`);
if (fs.existsSync(cachePath) && fs.existsSync(`${cachePath}.complete`)) {
logger.debug(`Found tool in cache ${toolName} ${versionSpec} ${arch}`);
toolPath = cachePath;
} else {
logger.debug("not found");
}
}
return toolPath;
}
}
/**
* Finds the paths to all versions of a tool that are installed in the local tool cache.
*
* Also see findAllVersions function from node_modules/@actions/tool-cache/lib/tool-cache.d.ts
*
* @param toolName name of the tool
* @param toolCacheDir path to the tool cache directory
* @param logger logger to use
*/
export function findAllVersions(
toolName: string,
toolCacheDir: string,
logger: Logger
): string[] {
if (isActions()) {
return actionsToolcache.findAllVersions(toolName);
} else {
// Initial implementation copied from node_modules/@actions/tool-cache/lib/tool-cache.js
const versions: string[] = [];
const arch = os.arch();
const toolPath = path.join(toolCacheDir, toolName);
if (fs.existsSync(toolPath)) {
const children = fs.readdirSync(toolPath);
for (const child of children) {
if (isExplicitVersion(child, logger)) {
const fullPath = path.join(toolPath, child, arch || "");
if (
fs.existsSync(fullPath) &&
fs.existsSync(`${fullPath}.complete`)
) {
versions.push(child);
}
}
}
}
return versions;
}
}
export async function downloadTool(
url: string,
tempDir: string,
headers: OutgoingHttpHeaders
): Promise<string> {
const dest = path.join(tempDir, uuidV4());
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers
);
return await actionsToolcache.downloadTool(
url,
dest,
undefined,
finalHeaders
);
}
function createExtractFolder(tempDir: string): string {
// create a temp dir
const dest = path.join(tempDir, "toolcache-temp");
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest);
}
return dest;
}
async function createToolPath(
tool: string,
version: string,
arch: string,
toolCacheDir: string,
logger: Logger
): Promise<string> {
const folderPath = path.join(
toolCacheDir,
tool,
semver.clean(version) || version,
arch || ""
);
logger.debug(`destination ${folderPath}`);
const markerPath = `${folderPath}.complete`;
await del(folderPath, { force: true });
await del(markerPath, { force: true });
fs.mkdirSync(folderPath, { recursive: true });
return folderPath;
}
function completeToolPath(
tool: string,
version: string,
arch: string,
toolCacheDir: string,
logger: Logger
) {
const folderPath = path.join(
toolCacheDir,
tool,
semver.clean(version) || version,
arch || ""
);
const markerPath = `${folderPath}.complete`;
fs.writeFileSync(markerPath, "");
logger.debug("finished caching tool");
}
function isExplicitVersion(versionSpec: string, logger: Logger) {
const c = semver.clean(versionSpec) || "";
logger.debug(`isExplicit: ${c}`);
const valid = semver.valid(c) != null;
logger.debug(`explicit? ${valid}`);
return valid;
}
function evaluateVersions(
versions: string[],
versionSpec: string,
logger: Logger
): string {
let version = "";
logger.debug(`evaluating ${versions.length} versions`);
versions = versions.sort((a, b) => {
if (semver.gt(a, b)) {
return 1;
}
return -1;
});
for (let i = versions.length - 1; i >= 0; i--) {
const potential = versions[i];
const satisfied = semver.satisfies(potential, versionSpec);
if (satisfied) {
version = potential;
break;
}
}
if (version) {
logger.debug(`matched: ${version}`);
} else {
logger.debug("match not found");
}
return version;
}

2
src/tracer-config.test.ts
View file

@ -24,7 +24,6 @@ function getTestConfig(tmpDir: string): configUtils.Config {
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
@ -33,6 +32,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
}

210
src/trap-caching.test.ts Normal file
View file

@ -0,0 +1,210 @@
import * as fs from "fs";
import * as path from "path";
import * as cache from "@actions/cache";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { getRecordingLogger, setupTests } from "./testing-utils";
import {
downloadTrapCaches,
getLanguagesSupportingCaching,
getTrapCachingExtractorConfigArgs,
getTrapCachingExtractorConfigArgsForLang,
uploadTrapCaches,
} from "./trap-caching";
import * as util from "./util";
setupTests(test);
const stubCodeql = setCodeQL({
async getVersion() {
return "2.10.3";
},
async betterResolveLanguages() {
return {
extractors: {
[Language.javascript]: [
{
extractor_root: "some_root",
extractor_options: {
trap: {
properties: {
cache: {
properties: {
dir: {
title: "Cache directory",
},
bound: {
title: "Cache bound",
},
write: {
title: "Cache write",
},
},
},
},
},
},
},
],
[Language.cpp]: [
{
extractor_root: "other_root",
},
],
},
};
},
});
const testConfigWithoutTmpDir: Config = {
languages: [Language.javascript, Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {
javascript: "/some/cache/dir",
},
};
function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
return {
languages: [Language.javascript, Language.ruby],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {
javascript: path.resolve(tmpDir, "jsCache"),
ruby: path.resolve(tmpDir, "rubyCache"),
},
};
}
test("check flags for JS, analyzing default branch", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfigWithTempDir(tmpDir);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const result = await getTrapCachingExtractorConfigArgsForLang(
config,
Language.javascript
);
t.deepEqual(result, [
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
"-O=javascript.trap.cache.bound=1024",
"-O=javascript.trap.cache.write=true",
]);
});
});
test("check flags for all, not analyzing default branch", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfigWithTempDir(tmpDir);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
const result = await getTrapCachingExtractorConfigArgs(config);
t.deepEqual(result, [
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
"-O=javascript.trap.cache.bound=1024",
"-O=javascript.trap.cache.write=false",
`-O=ruby.trap.cache.dir=${path.resolve(tmpDir, "rubyCache")}`,
"-O=ruby.trap.cache.bound=1024",
"-O=ruby.trap.cache.write=false",
]);
});
});
test("get languages that support TRAP caching", async (t) => {
const loggedMessages = [];
const logger = getRecordingLogger(loggedMessages);
const languagesSupportingCaching = await getLanguagesSupportingCaching(
stubCodeql,
[Language.javascript, Language.cpp],
logger
);
t.deepEqual(languagesSupportingCaching, [Language.javascript]);
});
test("upload cache key contains right fields", async (t) => {
const loggedMessages = [];
const logger = getRecordingLogger(loggedMessages);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const stubSave = sinon.stub(cache, "saveCache");
process.env.GITHUB_SHA = "somesha";
await uploadTrapCaches(stubCodeql, testConfigWithoutTmpDir, logger);
t.assert(
stubSave.calledOnceWith(
sinon.match.array.contains(["/some/cache/dir"]),
sinon
.match("somesha")
.and(sinon.match("2.10.3"))
.and(sinon.match("javascript"))
)
);
});
test("download cache looks for the right key and creates dir", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const loggedMessages = [];
const logger = getRecordingLogger(loggedMessages);
sinon.stub(actionsUtil, "getTemporaryDirectory").returns(tmpDir);
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
const stubRestore = sinon.stub(cache, "restoreCache").resolves("found");
const eventFile = path.resolve(tmpDir, "event.json");
process.env.GITHUB_EVENT_NAME = "pull_request";
process.env.GITHUB_EVENT_PATH = eventFile;
fs.writeFileSync(
eventFile,
JSON.stringify({
pull_request: {
base: {
sha: "somesha",
},
},
})
);
await downloadTrapCaches(
stubCodeql,
[Language.javascript, Language.cpp],
logger
);
t.assert(
stubRestore.calledOnceWith(
sinon.match.array.contains([
path.resolve(tmpDir, "trapCaches", "javascript"),
]),
sinon
.match("somesha")
.and(sinon.match("2.10.3"))
.and(sinon.match("javascript"))
)
);
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
});
});

191
src/trap-caching.ts Normal file
View file

@ -0,0 +1,191 @@
import * as fs from "fs";
import * as path from "path";
import * as cache from "@actions/cache";
import * as actionsUtil from "./actions-util";
import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import { codeQlVersionAbove } from "./util";
// This constant should be bumped if we make a breaking change
// to how the CodeQL Action stores or retrieves the TRAP cache,
// and will invalidate previous caches. We don't need to bump
// this for CLI/extractor changes, since the CLI version also
// goes into the cache key.
const CACHE_VERSION = 1;
// This constant sets the size of each TRAP cache in megabytes.
const CACHE_SIZE_MB = 1024;
export async function getTrapCachingExtractorConfigArgs(
config: Config
): Promise<string[]> {
const result: string[][] = [];
for (const language of config.languages)
result.push(
await getTrapCachingExtractorConfigArgsForLang(config, language)
);
return result.flat();
}
export async function getTrapCachingExtractorConfigArgsForLang(
config: Config,
language: Language
): Promise<string[]> {
const cacheDir = config.trapCaches[language];
if (cacheDir === undefined) return [];
const write = await actionsUtil.isAnalyzingDefaultBranch();
return [
`-O=${language}.trap.cache.dir=${cacheDir}`,
`-O=${language}.trap.cache.bound=${CACHE_SIZE_MB}`,
`-O=${language}.trap.cache.write=${write}`,
];
}
/**
* Download TRAP caches from the Actions cache.
* @param codeql The CodeQL instance to use.
* @param languages The languages being analyzed.
* @param logger A logger to record some informational messages to.
* @returns A partial map from languages to TRAP cache paths on disk, with
* languages for which we shouldn't use TRAP caching omitted.
*/
export async function downloadTrapCaches(
codeql: CodeQL,
languages: Language[],
logger: Logger
): Promise<Partial<Record<Language, string>>> {
const result = {};
const languagesSupportingCaching = await getLanguagesSupportingCaching(
codeql,
languages,
logger
);
logger.info(
`Found ${languagesSupportingCaching.length} languages that support TRAP caching`
);
if (languagesSupportingCaching.length === 0) return result;
const cachesDir = path.join(
actionsUtil.getTemporaryDirectory(),
"trapCaches"
);
for (const language of languagesSupportingCaching) {
const cacheDir = path.join(cachesDir, language);
fs.mkdirSync(cacheDir, { recursive: true });
result[language] = cacheDir;
}
if (await actionsUtil.isAnalyzingDefaultBranch()) {
logger.info(
"Analyzing default branch. Skipping downloading of TRAP caches."
);
return result;
}
let baseSha = "unknown";
const eventPath = process.env.GITHUB_EVENT_PATH;
if (
process.env.GITHUB_EVENT_NAME === "pull_request" &&
eventPath !== undefined
) {
const event = JSON.parse(fs.readFileSync(path.resolve(eventPath), "utf-8"));
baseSha = event.pull_request?.base?.sha || baseSha;
}
for (const language of languages) {
const cacheDir = result[language];
if (cacheDir === undefined) continue;
// The SHA from the base of the PR is the most similar commit we might have a cache for
const preferredKey = await cacheKey(codeql, language, baseSha);
logger.info(
`Looking in Actions cache for TRAP cache with key ${preferredKey}`
);
const found = await cache.restoreCache([cacheDir], preferredKey, [
await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
]);
if (found === undefined) {
// We didn't find a TRAP cache in the Actions cache, so the directory on disk is
// still just an empty directory. There's no reason to tell the extractor to use it,
// so let's unset the entry in the map so we don't set any extractor options.
logger.info(`No TRAP cache found in Actions cache for ${language}`);
result[language] = undefined;
}
}
return result;
}
export async function uploadTrapCaches(
codeql: CodeQL,
config: Config,
logger: Logger
): Promise<void> {
if (!(await actionsUtil.isAnalyzingDefaultBranch())) return; // Only upload caches from the default branch
const toAwait: Array<Promise<number>> = [];
for (const language of config.languages) {
const cacheDir = config.trapCaches[language];
if (cacheDir === undefined) continue;
const key = await cacheKey(
codeql,
language,
process.env.GITHUB_SHA || "unknown"
);
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
toAwait.push(cache.saveCache([cacheDir], key));
}
await Promise.all(toAwait);
}
export async function getLanguagesSupportingCaching(
codeql: CodeQL,
languages: Language[],
logger: Logger
): Promise<Language[]> {
const result: Language[] = [];
if (
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES))
)
return result;
const resolveResult = await codeql.betterResolveLanguages();
outer: for (const lang of languages) {
if (resolveResult.extractors[lang].length !== 1) continue;
const extractor = resolveResult.extractors[lang][0];
const trapCacheOptions =
extractor.extractor_options?.trap?.properties?.cache?.properties;
if (trapCacheOptions === undefined) {
logger.info(
`${lang} does not support TRAP caching (missing option group)`
);
continue;
}
for (const requiredOpt of ["dir", "bound", "write"]) {
if (!(requiredOpt in trapCacheOptions)) {
logger.info(
`${lang} does not support TRAP caching (missing ${requiredOpt} option)`
);
continue outer;
}
}
result.push(lang);
}
return result;
}
async function cacheKey(
codeql: CodeQL,
language: Language,
baseSha: string
): Promise<string> {
return `${await cachePrefix(codeql, language)}-${baseSha}`;
}
async function cachePrefix(
codeql: CodeQL,
language: Language
): Promise<string> {
return `codeql-trap-${CACHE_VERSION}-${await codeql.getVersion()}-${language}-`;
}

3
src/upload-lib.ts
View file

@ -524,7 +524,8 @@ export function validateUniqueCategory(sarif: SarifFile): void {
if (process.env[sentinelEnvVar]) {
throw new Error(
"Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job per tool/category. " +
"The easiest fix is to specify a unique value for the `category` input. " +
"The easiest fix is to specify a unique value for the `category` input. If .runs[].automationDetails.id is specified " +
"in the sarif file, that will take precedence over your configured `category`. " +
`Category: (${id ? id : "none"}) Tool: (${tool ? tool : "none"})`
);
}

2
src/util.test.ts
View file

@ -335,7 +335,6 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
pathsIgnore: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
@ -348,6 +347,7 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);

6
src/util.ts
View file

@ -116,11 +116,7 @@ export async function withTmpDir<T>(
body: (tmpDir: string) => Promise<T>
): Promise<T> {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "codeql-action-"));
const realSubdir = path.join(tmpDir, "real");
fs.mkdirSync(realSubdir);
const symlinkSubdir = path.join(tmpDir, "symlink");
fs.symlinkSync(realSubdir, symlinkSubdir, "dir");
const result = await body(symlinkSubdir);
const result = await body(tmpDir);
await del(tmpDir, { force: true });
return result;
}