robojerk/codeql-action

Add external git repositories to search path for custom queries

Browse source
This commit is contained in:
Edoardo Pirovano 2021-04-01 12:38:13 +01:00 • committed by Edoardo Pirovano
parent 1fa35632f2
commit 578f9fc99e
15 changed files with 244 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/analyze.test.ts
View file

@ -13,11 +13,25 @@ import * as util from "./util";
setupTests(test);
// Checks that the duration fields are populated for the correct language
// and correct case of builtin or custom.
test("status report fields", async (t) => {
// and correct case of builtin or custom. Also checks the correct search
// paths are set in the database analyze invocation.
test("status report fields and search path setting", async (t) => {
let searchPathsUsed: string[] = [];
return await util.withTmpDir(async (tmpDir) => {
setCodeQL({
databaseAnalyze: async () => undefined,
databaseAnalyze: async (
_,
sarifFile: string,
searchPath: string | undefined
) => {
fs.writeFileSync(
sarifFile,
JSON.stringify({
runs: [],
})
);
searchPathsUsed.push(searchPath!);
},
});
const memoryFlag = "";
@ -25,6 +39,7 @@ test("status report fields", async (t) => {
const threadsFlag = "";
for (const language of Object.values(Language)) {
searchPathsUsed = [];
const config: Config = {
languages: [language],
queries: {},
@ -61,7 +76,16 @@ test("status report fields", async (t) => {
config.queries[language] = {
builtin: [],
custom: ["foo.ql"],
custom: [
{
queries: ["foo.ql"],
searchPath: "/1",
},
{
queries: ["bar.ql"],
searchPath: "/2",
},
],
};
const customStatusReport = await runQueries(
tmpDir,
@ -75,6 +99,7 @@ test("status report fields", async (t) => {
t.true(
`analyze_custom_queries_${language}_duration_ms` in customStatusReport
);
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
}
});
});

109
src/analyze.ts
View file

@ -3,12 +3,14 @@ import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as actionsUtil from "./actions-util";
import * as analysisPaths from "./analysis-paths";
import { getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
import { combineSarifFiles } from "./upload-lib";
import * as util from "./util";
export class CodeQLAnalysisError extends Error {
@ -154,48 +156,43 @@ export async function runQueries(
}
try {
for (const type of ["builtin", "custom"]) {
if (queries[type].length > 0) {
const startTime = new Date().getTime();
const databasePath = util.getCodeQLDatabasePath(
config.tempDir,
language
if (queries["builtin"].length > 0) {
const startTimeBuliltIn = new Date().getTime();
await runQueryGroup(
language,
"builtin",
queries["builtin"],
sarifFolder,
undefined
);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuliltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = actionsUtil.getTemporaryDirectory();
const temporarySarifFiles: string[] = [];
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
await runQueryGroup(
language,
`custom-${i}`,
queries["custom"][i].queries,
temporarySarifDir,
queries["custom"][i].searchPath
);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries[type]
.map((q: string) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(
`Query suite file for ${language}...\n${querySuiteContents}`
temporarySarifFiles.push(
path.join(temporarySarifDir, `${language}-custom-${i}.sarif`)
);
const sarifFile = path.join(sarifFolder, `${language}-${type}.sarif`);
const codeql = getCodeQL(config.codeQLCmd);
await codeql.databaseAnalyze(
databasePath,
sarifFile,
querySuitePath,
memoryFlag,
addSnippetsFlag,
threadsFlag
);
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
logger.endGroup();
// Record the performance
const endTime = new Date().getTime();
statusReport[`analyze_${type}_queries_${language}_duration_ms`] =
endTime - startTime;
}
}
if (temporarySarifFiles.length > 0) {
fs.writeFileSync(
path.join(sarifFolder, `${language}-custom.sarif`),
combineSarifFiles(temporarySarifFiles)
);
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
} catch (e) {
logger.info(e);
statusReport.analyze_failure_language = language;
@ -207,6 +204,42 @@ export async function runQueries(
}
return statusReport;
async function runQueryGroup(
language: Language,
type: string,
queries: string[],
destinationFolder: string,
searchPath: string | undefined
): Promise<void> {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries
.map((q: string) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
const codeql = getCodeQL(config.codeQLCmd);
await codeql.databaseAnalyze(
databasePath,
sarifFile,
searchPath,
querySuitePath,
memoryFlag,
addSnippetsFlag,
threadsFlag
);
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
logger.endGroup();
}
}
export async function runAnalyze(

12
src/codeql.ts
View file

@ -90,6 +90,7 @@ export interface CodeQL {
databaseAnalyze(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
@ -640,12 +641,13 @@ function getCodeQLForCmd(cmd: string): CodeQL {
async databaseAnalyze(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string
) {
await new toolrunner.ToolRunner(cmd, [
const args = [
"database",
"analyze",
memoryFlag,
@ -657,8 +659,12 @@ function getCodeQLForCmd(cmd: string): CodeQL {
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "analyze"]),
querySuite,
]).exec();
];
if (extraSearchPath !== undefined) {
args.push("--search-path", extraSearchPath);
}
args.push(querySuite);
await new toolrunner.ToolRunner(cmd, args).exec();
},
};
}

38
src/config-utils.test.ts
View file

@ -284,7 +284,12 @@ test("load non-empty input", async (t) => {
queries: {
javascript: {
builtin: [],
custom: ["/foo/a.ql", "/bar/b.ql"],
custom: [
{
queries: ["/foo/a.ql", "/bar/b.ql"],
searchPath: tmpDir,
},
],
},
},
pathsIgnore: ["a", "b"],
@ -463,7 +468,7 @@ test("Queries can be specified in config file", async (t) => {
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/foo$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/foo$/);
});
});
@ -526,7 +531,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/override$/);
t.regex(config.queries["javascript"].custom[0].queries[0], /.*\/override$/);
});
});
@ -583,7 +588,10 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
// Now check that the end result contains only the workflow query, and not the default one
t.deepEqual(config.queries["javascript"].builtin.length, 0);
t.deepEqual(config.queries["javascript"].custom.length, 1);
t.regex(config.queries["javascript"].custom[0], /.*\/workflow-query$/);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/workflow-query$/
);
});
});
@ -640,8 +648,14 @@ test("Multiple queries can be specified in workflow file, no config file require
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/override1$/);
t.regex(config.queries["javascript"].custom[1], /.*\/override2$/);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/override1$/
);
t.regex(
config.queries["javascript"].custom[1].queries[0],
/.*\/override2$/
);
});
});
@ -712,9 +726,15 @@ test("Queries in workflow file can be added to the set of queries without overri
config.queries["javascript"].builtin[0],
/javascript-code-scanning.qls$/
);
t.regex(config.queries["javascript"].custom[0], /.*\/additional1$/);
t.regex(config.queries["javascript"].custom[1], /.*\/additional2$/);
t.regex(config.queries["javascript"].custom[2], /.*\/foo$/);
t.regex(
config.queries["javascript"].custom[0].queries[0],
/.*\/additional1$/
);
t.regex(
config.queries["javascript"].custom[1].queries[0],
/.*\/additional2$/
);
t.regex(config.queries["javascript"].custom[2].queries[0], /.*\/foo$/);
});
});

19
src/config-utils.ts
View file

@ -46,11 +46,23 @@ type Queries = {
[language: string]: {
/** Queries from one of the builtin suites */
builtin: string[];
/** Custom queries, from a non-standard location */
custom: string[];
custom: QueriesWithSearchPath[];
};
};
/**
* Contains some information about a user-defined query.
*/
export interface QueriesWithSearchPath {
/** Additional search path to use when running these queries. */
searchPath: string;
/** Array of absolute paths to a .ql file containing the queries. */
queries: string[];
}
/**
* Format of the parsed config file.
*/
@ -188,7 +200,10 @@ async function runResolveQueries(
(q) => !queryIsDisabled(language, q)
);
if (extraSearchPath !== undefined) {
resultMap[language].custom.push(...queries);
resultMap[language].custom.push({
searchPath: extraSearchPath,
queries,
});
} else {
resultMap[language].builtin.push(...queries);
}