robojerk/codeql-action

Merge branch 'aeisenberg/unrevert-query-filters' into aeisenberg/fix-config-files

Browse source
This commit is contained in:
Andrew Eisenberg 2022-08-11 14:16:50 -07:00
commit fa2bc211fd
61 changed files with 882 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

65
lib/analyze.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.runCleanup = exports.runFinalize = exports.runQueries = exports.dbIsFinalized = exports.createdDBForScannedLanguages = exports.CodeQLAnalysisError = void 0;
exports.validateQueryFilters = exports.runCleanup = exports.runFinalize = exports.createQuerySuiteContents = exports.convertPackToQuerySuiteEntry = exports.runQueries = exports.dbIsFinalized = exports.createdDBForScannedLanguages = exports.CodeQLAnalysisError = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@ -30,6 +30,7 @@ const del_1 = __importDefault(require("del"));
const yaml = __importStar(require("js-yaml"));
const analysisPaths = __importStar(require("./analysis-paths"));
const codeql_1 = require("./codeql");
const configUtils = __importStar(require("./config-utils"));
const count_loc_1 = require("./count-loc");
const languages_1 = require("./languages");
const sharedEnv = __importStar(require("./shared-environment"));
@ -128,6 +129,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
for (const language of config.languages) {
const queries = config.queries[language];
const queryFilters = validateQueryFilters(config.originalUserInput["query-filters"]);
const packsWithVersion = config.packs[language] || [];
const hasBuiltinQueries = (queries === null || queries === void 0 ? void 0 : queries.builtin.length) > 0;
const hasCustomQueries = (queries === null || queries === void 0 ? void 0 : queries.custom.length) > 0;
@ -172,7 +174,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const querySuitePaths = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"]), undefined)));
querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"], queryFilters), undefined)));
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
@ -180,12 +182,12 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries), queries["custom"][i].searchPath)));
querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries, queryFilters), queries["custom"][i].searchPath)));
ranCustom = true;
}
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(...(await runQueryPacks(language, "packs", packsWithVersion, undefined)));
querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters));
ranCustom = true;
}
if (ranCustom) {
@ -248,22 +250,40 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
async function runQueryPacks(language, type, packs, searchPath) {
async function runQueryPacks(language, type, packs, queryFilters) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Run the queries individually instead of all at once to avoid command
// line length restrictions, particularly on windows.
for (const pack of packs) {
logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
await codeql.databaseRunQueries(databasePath, searchPath, pack, memoryFlag, threadsFlag);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
}
return packs;
// combine the list of packs into a query suite in order to run them all simultaneously.
const querySuite = packs.map(convertPackToQuerySuiteEntry).concat(queryFilters);
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, memoryFlag, threadsFlag);
return querySuitePath;
}
}
exports.runQueries = runQueries;
function createQuerySuiteContents(queries) {
return queries.map((q) => `- query: ${q}`).join("\n");
function convertPackToQuerySuiteEntry(packStr) {
var _a, _b, _c, _d;
const pack = configUtils.parsePacksSpecification(packStr);
return {
qlpack: !pack.path ? pack.name : undefined,
from: pack.path ? pack.name : undefined,
version: pack.version,
query: ((_a = pack.path) === null || _a === void 0 ? void 0 : _a.endsWith(".ql")) ? pack.path : undefined,
queries: !((_b = pack.path) === null || _b === void 0 ? void 0 : _b.endsWith(".ql")) && !((_c = pack.path) === null || _c === void 0 ? void 0 : _c.endsWith(".qls"))
? pack.path
: undefined,
apply: ((_d = pack.path) === null || _d === void 0 ? void 0 : _d.endsWith(".qls")) ? pack.path : undefined,
};
}
exports.convertPackToQuerySuiteEntry = convertPackToQuerySuiteEntry;
function createQuerySuiteContents(queries, queryFilters) {
return yaml.dump(queries.map((q) => ({ query: q })).concat(queryFilters));
}
exports.createQuerySuiteContents = createQuerySuiteContents;
async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, featureFlags) {
try {
await (0, del_1.default)(outputDir, { force: true });
@ -329,4 +349,25 @@ function printLinesOfCodeSummary(logger, language, lineCounts) {
logger.info(`Counted a baseline of ${lineCounts[language]} lines of code for ${language}.`);
}
}
// exported for testing
function validateQueryFilters(queryFilters) {
if (!queryFilters) {
return [];
}
const errors = [];
for (const qf of queryFilters) {
const keys = Object.keys(qf);
if (keys.length !== 1) {
errors.push(`Query filter must have exactly one key: ${JSON.stringify(qf)}`);
}
if (!["exclude", "include"].includes(keys[0])) {
errors.push(`Only "include" or "exclude" filters are allowed:\n${JSON.stringify(qf)}`);
}
}
if (errors.length) {
throw new Error(`Invalid query filter.\n${errors.join("\n")}`);
}
return queryFilters;
}
exports.validateQueryFilters = validateQueryFilters;
//# sourceMappingURL=analyze.js.map