robojerk/codeql-action

Merge pull request #1547 from github/update-v2.2.5-237a258d2

Browse source 
Merge main into releases/v2
This commit is contained in:
Henry Mercer 2023-02-24 19:26:08 +00:00 • committed by GitHub
commit 32dc499307
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 482 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/script/check-node-modules.sh vendored
View file

@ -10,6 +10,10 @@ fi
# When updating this, make sure to update the npm version in
# `.github/workflows/update-dependencies.yml` too.
sudo npm install --force -g npm@9.2.0
# clean the npm cache to ensure we don't have any files owned by root
sudo npm cache clean --force
# Reinstall modules and then clean to remove absolute paths
# Use 'npm ci' instead of 'npm install' as this is intended to be reproducible
npm ci

4
CHANGELOG.md
View file

@ -1,5 +1,9 @@
# CodeQL Action Changelog
## 2.2.5 - 24 Feb 2023
- Update default CodeQL bundle version to 2.12.3. [#1543](https://github.com/github/codeql-action/pull/1543)
## 2.2.4 - 10 Feb 2023
No user facing changes.

6
CONTRIBUTING.md
View file

@ -67,12 +67,8 @@ Here are a few things you can do that will increase the likelihood of your pull
This mergeback incorporates the changelog updates into `main`, tags the release using the merge commit of the "Merge main into releases/v2" pull request, and bumps the patch version of the CodeQL Action.
Approve the mergeback PR and automerge it.
1. When the "Merge main into releases/v2" pull request is merged into the `releases/v2` branch, the "Update release branch" workflow will create a "Merge releases/v2 into releases/v1" pull request to merge the changes since the last release into the `releases/v1` release branch.
This ensures we keep both the `releases/v1` and `releases/v2` release branches up to date and fully supported.
Review the checklist items in the pull request description.
Once you've checked off all the items, approve the PR and automerge it.
1. Once the mergeback has been merged to `main` and the "Merge releases/v2 into releases/v1" PR has been merged to `releases/v1`, the release is complete.
Once the mergeback has been merged to `main`, the release is complete.
## Keeping the PR checks up to date (admin access required)

32
lib/analyze.js generated
View file

@ -126,6 +126,7 @@ async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger)
async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger, featureEnablement) {
const statusReport = {};
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
const queryFlags = [memoryFlag, threadsFlag];
await util.logCodeScanningConfigInCli(codeql, featureEnablement, logger);
for (const language of config.languages) {
const queries = config.queries[language];
@ -140,7 +141,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
// another to interpret the results.
logger.startGroup(`Running queries for ${language}`);
const startTimeBuiltIn = new Date().getTime();
await runQueryGroup(language, "all", undefined, undefined);
await runQueryGroup(language, "all", undefined, undefined, true);
// TODO should not be using `builtin` here. We should be using `all` instead.
// The status report does not support `all` yet.
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@ -164,24 +165,29 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
!hasPackWithCustomQueries) {
throw new Error(`Unable to analyze ${language} as no queries were selected for this language`);
}
const customQueryIndices = [];
for (let i = 0; i < queries.custom.length; ++i) {
if (queries.custom[i].queries.length > 0) {
customQueryIndices.push(i);
}
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths = [];
if (queries["builtin"].length > 0) {
if (queries.builtin.length > 0) {
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"], queryFilters), undefined)));
querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries.builtin, queryFilters), undefined, customQueryIndices.length === 0 && packsWithVersion.length === 0)));
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries, queryFilters), queries["custom"][i].searchPath)));
ranCustom = true;
}
for (const i of customQueryIndices) {
querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries.custom[i].queries, queryFilters), queries.custom[i].searchPath, i === customQueryIndices[customQueryIndices.length - 1] &&
packsWithVersion.length === 0)));
ranCustom = true;
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters));
querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters, true));
ranCustom = true;
}
if (ranCustom) {
@ -218,7 +224,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const databasePath = util.getCodeQLDatabasePath(config, language);
return await codeql.databasePrintBaseline(databasePath);
}
async function runQueryGroup(language, type, querySuiteContents, searchPath) {
async function runQueryGroup(language, type, querySuiteContents, searchPath, optimizeForLastQueryRun) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
@ -229,11 +235,11 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
}
await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, memoryFlag, threadsFlag);
await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
async function runQueryPacks(language, type, packs, queryFilters) {
async function runQueryPacks(language, type, packs, queryFilters, optimizeForLastQueryRun) {
const databasePath = util.getCodeQLDatabasePath(config, language);
for (const pack of packs) {
logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
@ -243,7 +249,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, memoryFlag, threadsFlag);
await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, queryFlags, optimizeForLastQueryRun);
return querySuitePath;
}
}

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

122
lib/analyze.test.js generated
View file

@ -30,8 +30,10 @@ const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const ava_1 = __importDefault(require("ava"));
const yaml = __importStar(require("js-yaml"));
const sinon = __importStar(require("sinon"));
const analyze_1 = require("./analyze");
const codeql_1 = require("./codeql");
const feature_flags_1 = require("./feature-flags");
const languages_1 = require("./languages");
const logging_1 = require("./logging");
const testing_utils_1 = require("./testing-utils");
@ -188,6 +190,126 @@ const util = __importStar(require("./util"));
}
}
});
function mockCodeQL() {
return {
getVersion: async () => "2.12.2",
databaseRunQueries: sinon.spy(),
databaseInterpretResults: async () => "",
databasePrintBaseline: async () => "",
};
}
function createBaseConfig(tmpDir) {
return {
languages: [],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "tempDir",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
},
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
}
function createQueryConfig(builtin, custom) {
return {
builtin,
custom: custom.map((c) => ({ searchPath: "/search", queries: [c] })),
};
}
async function runQueriesWithConfig(config, features) {
for (const language of config.languages) {
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
});
}
return (0, analyze_1.runQueries)("sarif-folder", "--memFlag", "--addSnippetsFlag", "--threadsFlag", undefined, config, (0, logging_1.getRunnerLogger)(true), (0, testing_utils_1.createFeatures)(features));
}
function getDatabaseRunQueriesCalls(mock) {
return mock.databaseRunQueries.getCalls();
}
(0, ava_1.default)("optimizeForLastQueryRun for one language", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp];
config.queries.cpp = createQueryConfig(["foo.ql"], []);
await runQueriesWithConfig(config, []);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true]);
});
});
(0, ava_1.default)("optimizeForLastQueryRun for two languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp, languages_1.Language.java];
config.queries.cpp = createQueryConfig(["foo.ql"], []);
config.queries.java = createQueryConfig(["bar.ql"], []);
await runQueriesWithConfig(config, []);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true, true]);
});
});
(0, ava_1.default)("optimizeForLastQueryRun for two languages, with custom queries", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp, languages_1.Language.java];
config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
await runQueriesWithConfig(config, []);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [false, false, true, false, true]);
});
});
(0, ava_1.default)("optimizeForLastQueryRun for two languages, with custom queries and packs", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp, languages_1.Language.java];
config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
config.packs.cpp = ["a/cpp-pack1@0.1.0"];
config.packs.java = ["b/java-pack1@0.2.0", "b/java-pack2@0.3.3"];
await runQueriesWithConfig(config, []);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [false, false, false, true, false, false, true]);
});
});
(0, ava_1.default)("optimizeForLastQueryRun for one language, CliConfigFileEnabled", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp];
await runQueriesWithConfig(config, [feature_flags_1.Feature.CliConfigFileEnabled]);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true]);
});
});
(0, ava_1.default)("optimizeForLastQueryRun for two languages, CliConfigFileEnabled", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp, languages_1.Language.java];
await runQueriesWithConfig(config, [feature_flags_1.Feature.CliConfigFileEnabled]);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true, true]);
});
});
(0, ava_1.default)("validateQueryFilters", (t) => {
t.notThrows(() => (0, analyze_1.validateQueryFilters)([]));
t.notThrows(() => (0, analyze_1.validateQueryFilters)(undefined));

2
lib/analyze.test.js.map
View file

File diff suppressed because one or more lines are too long

9
lib/codeql.js generated
View file

@ -472,17 +472,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, memoryFlag, threadsFlag) {
async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun) {
const codeqlArgs = [
"database",
"run-queries",
memoryFlag,
threadsFlag,
...flags,
databasePath,
"--min-disk-free=1024",
"-v",
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (optimizeForLastQueryRun &&
(await util.supportExpectDiscardedCache(this))) {
codeqlArgs.push("--expect-discarded-cache");
}
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

8
lib/defaults.json
View file

@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-20230207",
"cliVersion": "2.12.2",
"priorBundleVersion": "codeql-bundle-20230120",
"priorCliVersion": "2.12.1"
"bundleVersion": "codeql-bundle-20230217",
"cliVersion": "2.12.3",
"priorBundleVersion": "codeql-bundle-20230207",
"priorCliVersion": "2.12.2"
}

5
lib/feature-flags.js generated
View file

@ -37,7 +37,6 @@ var Feature;
Feature["CliConfigFileEnabled"] = "cli_config_file_enabled";
Feature["DisableKotlinAnalysisEnabled"] = "disable_kotlin_analysis_enabled";
Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
Feature["TrapCachingEnabled"] = "trap_caching_enabled";
Feature["UploadFailedSarifEnabled"] = "upload_failed_sarif_enabled";
})(Feature = exports.Feature || (exports.Feature = {}));
exports.featureConfig = {
@ -53,10 +52,6 @@ exports.featureConfig = {
envVar: "CODEQL_ML_POWERED_QUERIES",
minimumVersion: "2.7.5",
},
[Feature.TrapCachingEnabled]: {
envVar: "CODEQL_TRAP_CACHING",
minimumVersion: undefined,
},
[Feature.UploadFailedSarifEnabled]: {
envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
minimumVersion: "2.11.3",

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

21
lib/init-action.js generated
View file

@ -46,12 +46,13 @@ async function sendInitStatusReport(actionStatus, startedAt, config, toolsDownlo
tools_source: toolsSource || init_1.ToolsSource.Unknown,
workflow_languages: workflowLanguages || "",
};
let initToolsDownloadFields = {};
if (toolsSource === init_1.ToolsSource.Download) {
initToolsDownloadFields = {
tools_download_duration_ms: toolsDownloadDurationMs,
tools_feature_flags_valid: toolsFeatureFlagsValid,
};
const initToolsDownloadFields = {};
if (toolsDownloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadDurationMs;
}
if (toolsFeatureFlagsValid !== undefined) {
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
}
if (config !== undefined) {
const languages = config.languages.join(",");
@ -128,7 +129,7 @@ async function run() {
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
await (0, util_1.enrichEnvironment)(codeql);
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), await getTrapCachingEnabled(features),
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), getTrapCachingEnabled(),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@ -192,7 +193,7 @@ async function run() {
}
await sendInitStatusReport("success", startedAt, config, toolsDownloadDurationMs, toolsFeatureFlagsValid, toolsSource, toolsVersion, logger);
}
async function getTrapCachingEnabled(featureEnablement) {
function getTrapCachingEnabled() {
// If the workflow specified something always respect that
const trapCaching = (0, actions_util_1.getOptionalInput)("trap-caching");
if (trapCaching !== undefined)
@ -200,8 +201,8 @@ async function getTrapCachingEnabled(featureEnablement) {
// On self-hosted runners which may have slow network access, disable TRAP caching by default
if (!(0, util_1.isHostedRunner)())
return false;
// On hosted runners, respect the feature flag
return await featureEnablement.getValue(feature_flags_1.Feature.TrapCachingEnabled);
// On hosted runners, enable TRAP caching by default
return true;
}
async function runWrapper() {
try {

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

4
lib/upload-lib.js generated
View file

@ -330,7 +330,9 @@ async function waitForProcessing(repositoryNwo, sarifID, logger, options = {
else {
util.assertNever(status);
}
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS, {
allowProcessExit: false,
});
}
}
finally {

2
lib/upload-lib.js.map
View file

File diff suppressed because one or more lines are too long

31
lib/util.js generated
View file

@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@ -455,16 +455,33 @@ async function bundleDb(config, language, codeql, dbName) {
return databaseBundlePath;
}
exports.bundleDb = bundleDb;
async function delay(milliseconds) {
// Immediately `unref` the timer such that it only prevents the process from exiting if the
// surrounding promise is being awaited.
return new Promise((resolve) => setTimeout(resolve, milliseconds).unref());
/**
* @param milliseconds time to delay
* @param opts options
* @param opts.allowProcessExit if true, the timer will not prevent the process from exiting
*/
async function delay(milliseconds, { allowProcessExit }) {
return new Promise((resolve) => {
const timer = setTimeout(resolve, milliseconds);
if (allowProcessExit) {
// Immediately `unref` the timer such that it only prevents the process from exiting if the
// surrounding promise is being awaited.
timer.unref();
}
});
}
exports.delay = delay;
function isGoodVersion(versionSpec) {
return !BROKEN_VERSIONS.includes(versionSpec);
}
exports.isGoodVersion = isGoodVersion;
/**
* Checks whether the CodeQL CLI supports the `--expect-discarded-cache` command-line flag.
*/
async function supportExpectDiscardedCache(codeQL) {
return codeQlVersionAbove(codeQL, "2.12.1");
}
exports.supportExpectDiscardedCache = supportExpectDiscardedCache;
exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-queries";
/**
* Gets the ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
@ -636,7 +653,7 @@ async function withTimeout(timeoutMs, promise, onTimeout) {
return result;
};
const timeoutTask = async () => {
await delay(timeoutMs);
await delay(timeoutMs, { allowProcessExit: true });
if (!finished) {
// Workaround: While the promise racing below will allow the main code
// to continue, the process won't normally exit until the asynchronous
@ -659,7 +676,7 @@ exports.withTimeout = withTimeout;
async function checkForTimeout() {
if (hadTimeout === true) {
core.info("A timeout occurred, force exiting the process after 30 seconds to prevent hanging.");
await delay(30000);
await delay(30000, { allowProcessExit: true });
process.exit();
}
}

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

2
node_modules/.package-lock.json generated vendored
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "2.2.4",
"version": "2.2.5",
"lockfileVersion": 3,
"requires": true,
"packages": {

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "codeql",
"version": "2.2.4",
"version": "2.2.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "codeql",
"version": "2.2.4",
"version": "2.2.5",
"license": "MIT",
"dependencies": {
"@actions/artifact": "^1.1.0",

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "2.2.4",
"version": "2.2.5",
"private": true,
"description": "CodeQL action",
"scripts": {

176
src/analyze.test.ts
View file

@ -3,15 +3,18 @@ import * as path from "path";
import test, { ExecutionContext } from "ava";
import * as yaml from "js-yaml";
import * as sinon from "sinon";
import {
convertPackToQuerySuiteEntry,
createQuerySuiteContents,
runQueries,
validateQueryFilters,
QueriesStatusReport,
} from "./analyze";
import { setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { CodeQL, setCodeQL } from "./codeql";
import { Config, QueriesWithSearchPath } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
@ -229,6 +232,175 @@ test("status report fields and search path setting", async (t) => {
}
});
function mockCodeQL(): Partial<CodeQL> {
return {
getVersion: async () => "2.12.2",
databaseRunQueries: sinon.spy(),
databaseInterpretResults: async () => "",
databasePrintBaseline: async () => "",
};
}
function createBaseConfig(tmpDir: string): Config {
return {
languages: [],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "tempDir",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
}
function createQueryConfig(
builtin: string[],
custom: string[]
): { builtin: string[]; custom: QueriesWithSearchPath[] } {
return {
builtin,
custom: custom.map((c) => ({ searchPath: "/search", queries: [c] })),
};
}
async function runQueriesWithConfig(
config: Config,
features: Feature[]
): Promise<QueriesStatusReport> {
for (const language of config.languages) {
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
});
}
return runQueries(
"sarif-folder",
"--memFlag",
"--addSnippetsFlag",
"--threadsFlag",
undefined,
config,
getRunnerLogger(true),
createFeatures(features)
);
}
function getDatabaseRunQueriesCalls(mock: Partial<CodeQL>) {
return (mock.databaseRunQueries as sinon.SinonSpy).getCalls();
}
test("optimizeForLastQueryRun for one language", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp];
config.queries.cpp = createQueryConfig(["foo.ql"], []);
await runQueriesWithConfig(config, []);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[true]
);
});
});
test("optimizeForLastQueryRun for two languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp, Language.java];
config.queries.cpp = createQueryConfig(["foo.ql"], []);
config.queries.java = createQueryConfig(["bar.ql"], []);
await runQueriesWithConfig(config, []);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[true, true]
);
});
});
test("optimizeForLastQueryRun for two languages, with custom queries", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp, Language.java];
config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
await runQueriesWithConfig(config, []);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[false, false, true, false, true]
);
});
});
test("optimizeForLastQueryRun for two languages, with custom queries and packs", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp, Language.java];
config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
config.packs.cpp = ["a/cpp-pack1@0.1.0"];
config.packs.java = ["b/java-pack1@0.2.0", "b/java-pack2@0.3.3"];
await runQueriesWithConfig(config, []);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[false, false, false, true, false, false, true]
);
});
});
test("optimizeForLastQueryRun for one language, CliConfigFileEnabled", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp];
await runQueriesWithConfig(config, [Feature.CliConfigFileEnabled]);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[true]
);
});
});
test("optimizeForLastQueryRun for two languages, CliConfigFileEnabled", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp, Language.java];
await runQueriesWithConfig(config, [Feature.CliConfigFileEnabled]);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[true, true]
);
});
});
test("validateQueryFilters", (t) => {
t.notThrows(() => validateQueryFilters([]));
t.notThrows(() => validateQueryFilters(undefined));

61
src/analyze.ts
View file

@ -212,6 +212,7 @@ export async function runQueries(
const statusReport: QueriesStatusReport = {};
const codeql = await getCodeQL(config.codeQLCmd);
const queryFlags = [memoryFlag, threadsFlag];
await util.logCodeScanningConfigInCli(codeql, featureEnablement, logger);
@ -231,7 +232,7 @@ export async function runQueries(
// another to interpret the results.
logger.startGroup(`Running queries for ${language}`);
const startTimeBuiltIn = new Date().getTime();
await runQueryGroup(language, "all", undefined, undefined);
await runQueryGroup(language, "all", undefined, undefined, true);
// TODO should not be using `builtin` here. We should be using `all` instead.
// The status report does not support `all` yet.
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@ -267,16 +268,24 @@ export async function runQueries(
);
}
const customQueryIndices: number[] = [];
for (let i = 0; i < queries.custom.length; ++i) {
if (queries.custom[i].queries.length > 0) {
customQueryIndices.push(i);
}
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {
if (queries.builtin.length > 0) {
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push(
(await runQueryGroup(
language,
"builtin",
createQuerySuiteContents(queries["builtin"], queryFilters),
undefined
createQuerySuiteContents(queries.builtin, queryFilters),
undefined,
customQueryIndices.length === 0 && packsWithVersion.length === 0
)) as string
);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@ -284,21 +293,18 @@ export async function runQueries(
}
const startTimeCustom = new Date().getTime();
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
querySuitePaths.push(
(await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(
queries["custom"][i].queries,
queryFilters
),
queries["custom"][i].searchPath
)) as string
);
ranCustom = true;
}
for (const i of customQueryIndices) {
querySuitePaths.push(
(await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(queries.custom[i].queries, queryFilters),
queries.custom[i].searchPath,
i === customQueryIndices[customQueryIndices.length - 1] &&
packsWithVersion.length === 0
)) as string
);
ranCustom = true;
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(
@ -306,7 +312,8 @@ export async function runQueries(
language,
"packs",
packsWithVersion,
queryFilters
queryFilters,
true
)
);
ranCustom = true;
@ -373,7 +380,8 @@ export async function runQueries(
language: Language,
type: string,
querySuiteContents: string | undefined,
searchPath: string | undefined
searchPath: string | undefined,
optimizeForLastQueryRun: boolean
): Promise<string | undefined> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
@ -391,8 +399,8 @@ export async function runQueries(
databasePath,
searchPath,
querySuitePath,
memoryFlag,
threadsFlag
queryFlags,
optimizeForLastQueryRun
);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
@ -402,7 +410,8 @@ export async function runQueries(
language: Language,
type: string,
packs: string[],
queryFilters: configUtils.QueryFilter[]
queryFilters: configUtils.QueryFilter[],
optimizeForLastQueryRun: boolean
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
@ -424,8 +433,8 @@ export async function runQueries(
databasePath,
undefined,
querySuitePath,
memoryFlag,
threadsFlag
queryFlags,
optimizeForLastQueryRun
);
return querySuitePath;

23
src/codeql.ts
View file

@ -148,13 +148,19 @@ export interface CodeQL {
): Promise<void>;
/**
* Run 'codeql database run-queries'.
*
* @param optimizeForLastQueryRun Whether to apply additional optimization for
* the last database query run in the action.
* It is always safe to set it to false.
* It should be set to true only for the very
* last databaseRunQueries() call.
*/
databaseRunQueries(
databasePath: string,
extraSearchPath: string | undefined,
querySuitePath: string | undefined,
memoryFlag: string,
threadsFlag: string
flags: string[],
optimizeForLastQueryRun: boolean
): Promise<void>;
/**
* Run 'codeql database interpret-results'.
@ -789,19 +795,24 @@ export async function getCodeQLForCmd(
databasePath: string,
extraSearchPath: string | undefined,
querySuitePath: string | undefined,
memoryFlag: string,
threadsFlag: string
flags: string[],
optimizeForLastQueryRun: boolean
): Promise<void> {
const codeqlArgs = [
"database",
"run-queries",
memoryFlag,
threadsFlag,
...flags,
databasePath,
"--min-disk-free=1024", // Try to leave at least 1GB free
"-v",
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (
optimizeForLastQueryRun &&
(await util.supportExpectDiscardedCache(this))
) {
codeqlArgs.push("--expect-discarded-cache");
}
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}

8
src/defaults.json
View file

@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-20230207",
"cliVersion": "2.12.2",
"priorBundleVersion": "codeql-bundle-20230120",
"priorCliVersion": "2.12.1"
"bundleVersion": "codeql-bundle-20230217",
"cliVersion": "2.12.3",
"priorBundleVersion": "codeql-bundle-20230207",
"priorCliVersion": "2.12.2"
}

5
src/feature-flags.ts
View file

@ -37,7 +37,6 @@ export enum Feature {
CliConfigFileEnabled = "cli_config_file_enabled",
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
TrapCachingEnabled = "trap_caching_enabled",
UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
}
@ -57,10 +56,6 @@ export const featureConfig: Record<
envVar: "CODEQL_ML_POWERED_QUERIES",
minimumVersion: "2.7.5",
},
[Feature.TrapCachingEnabled]: {
envVar: "CODEQL_TRAP_CACHING",
minimumVersion: undefined,
},
[Feature.UploadFailedSarifEnabled]: {
envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
minimumVersion: "2.11.3",

25
src/init-action.ts
View file

@ -16,7 +16,7 @@ import {
import { getGitHubVersion } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { Feature, FeatureEnablement, Features } from "./feature-flags";
import { Feature, Features } from "./feature-flags";
import {
initCodeQL,
initConfig,
@ -118,13 +118,14 @@ async function sendInitStatusReport(
workflow_languages: workflowLanguages || "",
};
let initToolsDownloadFields: InitToolsDownloadFields = {};
const initToolsDownloadFields: InitToolsDownloadFields = {};
if (toolsSource === ToolsSource.Download) {
initToolsDownloadFields = {
tools_download_duration_ms: toolsDownloadDurationMs,
tools_feature_flags_valid: toolsFeatureFlagsValid,
};
if (toolsDownloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadDurationMs;
}
if (toolsFeatureFlagsValid !== undefined) {
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
}
if (config !== undefined) {
@ -253,7 +254,7 @@ async function run() {
getOptionalInput("registries"),
getOptionalInput("config-file"),
getOptionalInput("db-location"),
await getTrapCachingEnabled(features),
getTrapCachingEnabled(),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@ -385,9 +386,7 @@ async function run() {
);
}
async function getTrapCachingEnabled(
featureEnablement: FeatureEnablement
): Promise<boolean> {
function getTrapCachingEnabled(): boolean {
// If the workflow specified something always respect that
const trapCaching = getOptionalInput("trap-caching");
if (trapCaching !== undefined) return trapCaching === "true";
@ -395,8 +394,8 @@ async function getTrapCachingEnabled(
// On self-hosted runners which may have slow network access, disable TRAP caching by default
if (!isHostedRunner()) return false;
// On hosted runners, respect the feature flag
return await featureEnablement.getValue(Feature.TrapCachingEnabled);
// On hosted runners, enable TRAP caching by default
return true;
}
async function runWrapper() {

4
src/upload-lib.ts
View file

@ -463,7 +463,9 @@ export async function waitForProcessing(
util.assertNever(status);
}
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS, {
allowProcessExit: false,
});
}
} finally {
logger.endGroup();

34
src/util.ts
View file

@ -548,16 +548,38 @@ export async function bundleDb(
return databaseBundlePath;
}
export async function delay(milliseconds: number) {
// Immediately `unref` the timer such that it only prevents the process from exiting if the
// surrounding promise is being awaited.
return new Promise((resolve) => setTimeout(resolve, milliseconds).unref());
/**
* @param milliseconds time to delay
* @param opts options
* @param opts.allowProcessExit if true, the timer will not prevent the process from exiting
*/
export async function delay(
milliseconds: number,
{ allowProcessExit }: { allowProcessExit: boolean }
) {
return new Promise((resolve) => {
const timer = setTimeout(resolve, milliseconds);
if (allowProcessExit) {
// Immediately `unref` the timer such that it only prevents the process from exiting if the
// surrounding promise is being awaited.
timer.unref();
}
});
}
export function isGoodVersion(versionSpec: string) {
return !BROKEN_VERSIONS.includes(versionSpec);
}
/**
* Checks whether the CodeQL CLI supports the `--expect-discarded-cache` command-line flag.
*/
export async function supportExpectDiscardedCache(
codeQL: CodeQL
): Promise<boolean> {
return codeQlVersionAbove(codeQL, "2.12.1");
}
export const ML_POWERED_JS_QUERIES_PACK_NAME =
"codeql/javascript-experimental-atm-queries";
@ -748,7 +770,7 @@ export async function withTimeout<T>(
return result;
};
const timeoutTask = async () => {
await delay(timeoutMs);
await delay(timeoutMs, { allowProcessExit: true });
if (!finished) {
// Workaround: While the promise racing below will allow the main code
// to continue, the process won't normally exit until the asynchronous
@ -773,7 +795,7 @@ export async function checkForTimeout() {
core.info(
"A timeout occurred, force exiting the process after 30 seconds to prevent hanging."
);
await delay(30_000);
await delay(30_000, { allowProcessExit: true });
process.exit();
}
}