robojerk/codeql-action

Merge pull request #455 from github/update-v1-896b4ff1

Browse source 
Merge main into v1
This commit is contained in:
Henning Makholm 2021-04-21 20:50:55 +02:00 • committed by GitHub
commit 429ece1037
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 246 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

57
lib/analyze.js generated
View file

@ -10,10 +10,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const actionsUtil = __importStar(require("./actions-util"));
const analysisPaths = __importStar(require("./analysis-paths"));
const codeql_1 = require("./codeql");
const languages_1 = require("./languages");
const sharedEnv = __importStar(require("./shared-environment"));
const upload_lib_1 = require("./upload-lib");
const util = __importStar(require("./util"));
class CodeQLAnalysisError extends Error {
constructor(queriesStatusReport, message) {
@ -83,29 +85,26 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
}
try {
for (const type of ["builtin", "custom"]) {
if (queries[type].length > 0) {
const startTime = new Date().getTime();
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries[type]
.map((q) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
const sarifFile = path.join(sarifFolder, `${language}-${type}.sarif`);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
await codeql.databaseAnalyze(databasePath, sarifFile, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag);
logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
logger.endGroup();
// Record the performance
const endTime = new Date().getTime();
statusReport[`analyze_${type}_queries_${language}_duration_ms`] =
endTime - startTime;
if (queries["builtin"].length > 0) {
const startTimeBuliltIn = new Date().getTime();
await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuliltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = actionsUtil.getTemporaryDirectory();
const temporarySarifFiles = [];
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
temporarySarifFiles.push(path.join(temporarySarifDir, `${language}-custom-${i}.sarif`));
}
}
if (temporarySarifFiles.length > 0) {
fs.writeFileSync(path.join(sarifFolder, `${language}-custom.sarif`), upload_lib_1.combineSarifFiles(temporarySarifFiles));
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
}
catch (e) {
logger.info(e);
@ -114,6 +113,22 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
}
}
return statusReport;
async function runQueryGroup(language, type, queries, destinationFolder, searchPath) {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries
.map((q) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag);
logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
logger.endGroup();
}
}
exports.runQueries = runQueries;
async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, config, logger) {

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

26
lib/analyze.test.js generated
View file

@ -20,16 +20,24 @@ const testing_utils_1 = require("./testing-utils");
const util = __importStar(require("./util"));
testing_utils_1.setupTests(ava_1.default);
// Checks that the duration fields are populated for the correct language
// and correct case of builtin or custom.
ava_1.default("status report fields", async (t) => {
// and correct case of builtin or custom. Also checks the correct search
// paths are set in the database analyze invocation.
ava_1.default("status report fields and search path setting", async (t) => {
let searchPathsUsed = [];
return await util.withTmpDir(async (tmpDir) => {
codeql_1.setCodeQL({
databaseAnalyze: async () => undefined,
databaseAnalyze: async (_, sarifFile, searchPath) => {
fs.writeFileSync(sarifFile, JSON.stringify({
runs: [],
}));
searchPathsUsed.push(searchPath);
},
});
const memoryFlag = "";
const addSnippetsFlag = "";
const threadsFlag = "";
for (const language of Object.values(languages_1.Language)) {
searchPathsUsed = [];
const config = {
languages: [language],
queries: {},
@ -55,11 +63,21 @@ ava_1.default("status report fields", async (t) => {
t.true(`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport);
config.queries[language] = {
builtin: [],
custom: ["foo.ql"],
custom: [
{
queries: ["foo.ql"],
searchPath: "/1",
},
{
queries: ["bar.ql"],
searchPath: "/2",
},
],
};
const customStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, config, logging_1.getRunnerLogger(true));
t.deepEqual(Object.keys(customStatusReport).length, 1);
t.true(`analyze_custom_queries_${language}_duration_ms` in customStatusReport);
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
}
});
});

2
lib/analyze.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze.test.js","sourceRoot":"","sources":["../src/analyze.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uCAAyB;AAEzB,8CAAuB;AAEvB,uCAAuC;AACvC,qCAAqC;AAErC,2CAAuC;AACvC,uCAA4C;AAC5C,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,yEAAyE;AACzE,yCAAyC;AACzC,aAAI,CAAC,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,kBAAS,CAAC;YACR,eAAe,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;SACvC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,EAAE,CAAC;QACtB,MAAM,eAAe,GAAG,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,EAAE;YAC9C,MAAM,MAAM,GAAW;gBACrB,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACrB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE,EAAE;gBACT,iBAAiB,EAAE,EAAE;gBACrB,OAAO,EAAE,MAAM;gBACf,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;iBACV;aACxB,CAAC;YACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACjE,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACnB,MAAM,EAAE,EAAE;aACX,CAAC;YACF,MAAM,mBAAmB,GAAG,MAAM,oBAAU,CAC1C,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,IAAI,CACJ,2BAA2B,QAAQ,cAAc,IAAI,mBAAmB,CACzE,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,CAAC,QAAQ,CAAC;aACnB,CAAC;YACF,MAAM,kBAAkB,GAAG,MAAM,oBAAU,CACzC,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC,CAAC,IAAI,CACJ,0BAA0B,QAAQ,cAAc,IAAI,kBAAkB,CACvE,CAAC;SACH;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
{"version":3,"file":"analyze.test.js","sourceRoot":"","sources":["../src/analyze.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uCAAyB;AAEzB,8CAAuB;AAEvB,uCAAuC;AACvC,qCAAqC;AAErC,2CAAuC;AACvC,uCAA4C;AAC5C,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,yEAAyE;AACzE,wEAAwE;AACxE,oDAAoD;AACpD,aAAI,CAAC,8CAA8C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/D,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,kBAAS,CAAC;YACR,eAAe,EAAE,KAAK,EACpB,CAAC,EACD,SAAiB,EACjB,UAA8B,EAC9B,EAAE;gBACF,EAAE,CAAC,aAAa,CACd,SAAS,EACT,IAAI,CAAC,SAAS,CAAC;oBACb,IAAI,EAAE,EAAE;iBACT,CAAC,CACH,CAAC;gBACF,eAAe,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;YACpC,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,EAAE,CAAC;QACtB,MAAM,eAAe,GAAG,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,EAAE;YAC9C,eAAe,GAAG,EAAE,CAAC;YACrB,MAAM,MAAM,GAAW;gBACrB,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACrB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE,EAAE;gBACT,iBAAiB,EAAE,EAAE;gBACrB,OAAO,EAAE,MAAM;gBACf,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;iBACV;aACxB,CAAC;YACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACjE,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACnB,MAAM,EAAE,EAAE;aACX,CAAC;YACF,MAAM,mBAAmB,GAAG,MAAM,oBAAU,CAC1C,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,IAAI,CACJ,2BAA2B,QAAQ,cAAc,IAAI,mBAAmB,CACzE,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE;oBACN;wBACE,OAAO,EAAE,CAAC,QAAQ,CAAC;wBACnB,UAAU,EAAE,IAAI;qBACjB;oBACD;wBACE,OAAO,EAAE,CAAC,QAAQ,CAAC;wBACnB,UAAU,EAAE,IAAI;qBACjB;iBACF;aACF,CAAC;YACF,MAAM,kBAAkB,GAAG,MAAM,oBAAU,CACzC,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC,CAAC,IAAI,CACJ,0BAA0B,QAAQ,cAAc,IAAI,kBAAkB,CACvE,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;SACvD;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

12
lib/codeql.js generated
View file

@ -427,8 +427,8 @@ function getCodeQLForCmd(cmd) {
}).exec();
return JSON.parse(output);
},
async databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, addSnippetsFlag, threadsFlag) {
await new toolrunner.ToolRunner(cmd, [
async databaseAnalyze(databasePath, sarifFile, extraSearchPath, querySuite, memoryFlag, addSnippetsFlag, threadsFlag) {
const args = [
"database",
"analyze",
memoryFlag,
@ -440,8 +440,12 @@ function getCodeQLForCmd(cmd) {
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "analyze"]),
querySuite,
]).exec();
];
if (extraSearchPath !== undefined) {
args.push("--search-path", extraSearchPath);
}
args.push(querySuite);
await new toolrunner.ToolRunner(cmd, args).exec();
},
};
}

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

5
lib/config-utils.js generated
View file

@ -78,7 +78,10 @@ async function runResolveQueries(codeQL, resultMap, toResolve, extraSearchPath)
}
const queries = Object.keys(queryPaths).filter((q) => !queryIsDisabled(language, q));
if (extraSearchPath !== undefined) {
resultMap[language].custom.push(...queries);
resultMap[language].custom.push({
searchPath: extraSearchPath,
queries,
});
}
else {
resultMap[language].builtin.push(...queries);

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

23
lib/config-utils.test.js generated
View file

@ -172,7 +172,12 @@ ava_1.default("load non-empty input", async (t) => {
queries: {
javascript: {
builtin: [],
custom: ["/foo/a.ql", "/bar/b.ql"],
custom: [
{
queries: ["/foo/a.ql", "/bar/b.ql"],
searchPath: tmpDir,
},
],
},
},
pathsIgnore: ["a", "b"],
@ -281,7 +286,7 @@ ava_1.default("Queries can be specified in config file", async (t) => {
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
t.regex(config.queries["javascript"].custom[0], /.*\/foo$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/foo$/);
});
});
ava_1.default("Queries from config file can be overridden in workflow file", async (t) => {
@ -314,7 +319,7 @@ ava_1.default("Queries from config file can be overridden in workflow file", asy
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
t.regex(config.queries["javascript"].custom[0], /.*\/override$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/override$/);
});
});
ava_1.default("Queries in workflow file can be used in tandem with the 'disable default queries' option", async (t) => {
@ -345,7 +350,7 @@ ava_1.default("Queries in workflow file can be used in tandem with the 'disable
// Now check that the end result contains only the workflow query, and not the default one
t.deepEqual(config.queries["javascript"].builtin.length, 0);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(config.queries["javascript"].custom[0], /.*\/workflow-query$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/workflow-query$/);
});
});
ava_1.default("Multiple queries can be specified in workflow file, no config file required", async (t) => {
@ -374,8 +379,8 @@ ava_1.default("Multiple queries can be specified in workflow file, no config fil
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 2);
t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
t.regex(config.queries["javascript"].custom[0], /.*\/override1$/);
t.regex(config.queries["javascript"].custom[1], /.*\/override2$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/override1$/);
t.regex(config.queries["javascript"].custom[1].queries[0], /.*\/override2$/);
});
});
ava_1.default("Queries in workflow file can be added to the set of queries without overriding config file", async (t) => {
@ -416,9 +421,9 @@ ava_1.default("Queries in workflow file can be added to the set of queries witho
t.deepEqual(config.queries["javascript"].builtin.length, 1);
t.deepEqual(config.queries["javascript"].custom.length, 3);
t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
t.regex(config.queries["javascript"].custom[0], /.*\/additional1$/);
t.regex(config.queries["javascript"].custom[1], /.*\/additional2$/);
t.regex(config.queries["javascript"].custom[2], /.*\/foo$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/additional1$/);
t.regex(config.queries["javascript"].custom[1].queries[0], /.*\/additional2$/);
t.regex(config.queries["javascript"].custom[2].queries[0], /.*\/foo$/);
});
});
ava_1.default("Invalid queries in workflow file handled correctly", async (t) => {

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20210326"
"bundleVersion": "codeql-bundle-20210421"
}

33
src/analyze.test.ts
View file

@ -13,11 +13,25 @@ import * as util from "./util";
setupTests(test);
// Checks that the duration fields are populated for the correct language
// and correct case of builtin or custom.
test("status report fields", async (t) => {
// and correct case of builtin or custom. Also checks the correct search
// paths are set in the database analyze invocation.
test("status report fields and search path setting", async (t) => {
let searchPathsUsed: string[] = [];
return await util.withTmpDir(async (tmpDir) => {
setCodeQL({
databaseAnalyze: async () => undefined,
databaseAnalyze: async (
_,
sarifFile: string,
searchPath: string | undefined
) => {
fs.writeFileSync(
sarifFile,
JSON.stringify({
runs: [],
})
);
searchPathsUsed.push(searchPath!);
},
});
const memoryFlag = "";
@ -25,6 +39,7 @@ test("status report fields", async (t) => {
const threadsFlag = "";
for (const language of Object.values(Language)) {
searchPathsUsed = [];
const config: Config = {
languages: [language],
queries: {},
@ -61,7 +76,16 @@ test("status report fields", async (t) => {
config.queries[language] = {
builtin: [],
custom: ["foo.ql"],
custom: [
{
queries: ["foo.ql"],
searchPath: "/1",
},
{
queries: ["bar.ql"],
searchPath: "/2",
},
],
};
const customStatusReport = await runQueries(
tmpDir,
@ -75,6 +99,7 @@ test("status report fields", async (t) => {
t.true(
`analyze_custom_queries_${language}_duration_ms` in customStatusReport
);
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
}
});
});

109
src/analyze.ts
View file

@ -3,12 +3,14 @@ import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as actionsUtil from "./actions-util";
import * as analysisPaths from "./analysis-paths";
import { getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
import { combineSarifFiles } from "./upload-lib";
import * as util from "./util";
export class CodeQLAnalysisError extends Error {
@ -154,48 +156,43 @@ export async function runQueries(
}
try {
for (const type of ["builtin", "custom"]) {
if (queries[type].length > 0) {
const startTime = new Date().getTime();
const databasePath = util.getCodeQLDatabasePath(
config.tempDir,
language
if (queries["builtin"].length > 0) {
const startTimeBuliltIn = new Date().getTime();
await runQueryGroup(
language,
"builtin",
queries["builtin"],
sarifFolder,
undefined
);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuliltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = actionsUtil.getTemporaryDirectory();
const temporarySarifFiles: string[] = [];
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
await runQueryGroup(
language,
`custom-${i}`,
queries["custom"][i].queries,
temporarySarifDir,
queries["custom"][i].searchPath
);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries[type]
.map((q: string) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(
`Query suite file for ${language}...\n${querySuiteContents}`
temporarySarifFiles.push(
path.join(temporarySarifDir, `${language}-custom-${i}.sarif`)
);
const sarifFile = path.join(sarifFolder, `${language}-${type}.sarif`);
const codeql = getCodeQL(config.codeQLCmd);
await codeql.databaseAnalyze(
databasePath,
sarifFile,
querySuitePath,
memoryFlag,
addSnippetsFlag,
threadsFlag
);
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
logger.endGroup();
// Record the performance
const endTime = new Date().getTime();
statusReport[`analyze_${type}_queries_${language}_duration_ms`] =
endTime - startTime;
}
}
if (temporarySarifFiles.length > 0) {
fs.writeFileSync(
path.join(sarifFolder, `${language}-custom.sarif`),
combineSarifFiles(temporarySarifFiles)
);
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
} catch (e) {
logger.info(e);
statusReport.analyze_failure_language = language;
@ -207,6 +204,42 @@ export async function runQueries(
}
return statusReport;
async function runQueryGroup(
language: Language,
type: string,
queries: string[],
destinationFolder: string,
searchPath: string | undefined
): Promise<void> {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries
.map((q: string) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
const codeql = getCodeQL(config.codeQLCmd);
await codeql.databaseAnalyze(
databasePath,
sarifFile,
searchPath,
querySuitePath,
memoryFlag,
addSnippetsFlag,
threadsFlag
);
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
logger.endGroup();
}
}
export async function runAnalyze(

12
src/codeql.ts
View file

@ -90,6 +90,7 @@ export interface CodeQL {
databaseAnalyze(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
@ -640,12 +641,13 @@ function getCodeQLForCmd(cmd: string): CodeQL {
async databaseAnalyze(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string
) {
await new toolrunner.ToolRunner(cmd, [
const args = [
"database",
"analyze",
memoryFlag,
@ -657,8 +659,12 @@ function getCodeQLForCmd(cmd: string): CodeQL {
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "analyze"]),
querySuite,
]).exec();
];
if (extraSearchPath !== undefined) {
args.push("--search-path", extraSearchPath);
}
args.push(querySuite);
await new toolrunner.ToolRunner(cmd, args).exec();
},
};
}

38
src/config-utils.test.ts
View file

@ -284,7 +284,12 @@ test("load non-empty input", async (t) => {
queries: {
javascript: {
builtin: [],
custom: ["/foo/a.ql", "/bar/b.ql"],
custom: [
{
queries: ["/foo/a.ql", "/bar/b.ql"],
searchPath: tmpDir,
},
],
},
},
pathsIgnore: ["a", "b"],
@ -463,7 +468,7 @@ test("Queries can be specified in config file", async (t) => {
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/foo$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/foo$/);
});
});
@ -526,7 +531,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/override$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/override$/);
});
});
@ -583,7 +588,10 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
// Now check that the end result contains only the workflow query, and not the default one
t.deepEqual(config.queries["javascript"].builtin.length, 0);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(config.queries["javascript"].custom[0], /.*\/workflow-query$/);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/workflow-query$/
);
});
});
@ -640,8 +648,14 @@ test("Multiple queries can be specified in workflow file, no config file require
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/override1$/);
t.regex(config.queries["javascript"].custom[1], /.*\/override2$/);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/override1$/
);
t.regex(
config.queries["javascript"].custom[1].queries[0],
/.*\/override2$/
);
});
});
@ -712,9 +726,15 @@ test("Queries in workflow file can be added to the set of queries without overri
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/additional1$/);
t.regex(config.queries["javascript"].custom[1], /.*\/additional2$/);
t.regex(config.queries["javascript"].custom[2], /.*\/foo$/);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/additional1$/
);
t.regex(
config.queries["javascript"].custom[1].queries[0],
/.*\/additional2$/
);
t.regex(config.queries["javascript"].custom[2].queries[0], /.*\/foo$/);
});
});

19
src/config-utils.ts
View file

@ -46,11 +46,23 @@ type Queries = {
[language: string]: {
/** Queries from one of the builtin suites */
builtin: string[];
/** Custom queries, from a non-standard location */
custom: string[];
custom: QueriesWithSearchPath[];
};
};
/**
* Contains some information about a user-defined query.
*/
export interface QueriesWithSearchPath {
/** Additional search path to use when running these queries. */
searchPath: string;
/** Array of absolute paths to a .ql file containing the queries. */
queries: string[];
}
/**
* Format of the parsed config file.
*/
@ -188,7 +200,10 @@ async function runResolveQueries(
(q) => !queryIsDisabled(language, q)
);
if (extraSearchPath !== undefined) {
resultMap[language].custom.push(...queries);
resultMap[language].custom.push({
searchPath: extraSearchPath,
queries,
});
} else {
resultMap[language].builtin.push(...queries);
}

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20210326"
"bundleVersion": "codeql-bundle-20210421"
}