Allow the codeql-action to run packages
This commit adds a `packs` option to the codeql-config.yml file. Users
can specify a list of ql packs to include in the analysis.
For a single language analysis, the packs property looks like this:
```yaml
packs:
- pack-scope/pack-name1@1.2.3
- pack-scope/pack-name2 # no explicit version means download the latest
```
For multi-language analysis, you must key the packs block by lanaguage:
```yaml
packs:
cpp:
- pack-scope/pack-name1@1.2.3
- pack-scope/pack-name2
java:
- pack-scope/pack-name3@1.2.3
- pack-scope/pack-name4
```
This implementation adds a new analysis run (alongside custom and
builtin runs). The unit tests indicate that the correct commands are
being run, but I have not actually tried this with a real CLI.
Also, convert `instanceof Array` to `Array.isArray` since that is
sightly better in some situations. See:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray#instanceof_vs_isarray
This commit is contained in:
Andrew Eisenberg
parent
cbdf0df97b
commit
86a804f9a7
22 changed files with 940 additions and 45 deletions
3
lib/analysis-paths.test.js
generated
3
lib/analysis-paths.test.js
generated
|
|
@ -29,6 +29,7 @@ ava_1.default("emptyPaths", async (t) => {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
@ -49,6 +50,7 @@ ava_1.default("nonEmptyPaths", async (t) => {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
|
||||
|
|
@ -70,6 +72,7 @@ ava_1.default("exclude temp dir", async (t) => {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
|
||||
dbLocation: path.resolve(tempDir, "codeql_databases"),
|
||||
packs: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;SACrD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;SACrD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO;YACP,YAAY;YACZ,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;SACtD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
||||
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAElD,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAW;SACnB,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAW;SACnB,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO;YACP,YAAY;YACZ,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;YACrD,KAAK,EAAE,EAAW;SACnB,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
||||
30
lib/analyze.js
generated
30
lib/analyze.js
generated
|
|
@ -87,6 +87,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
|
|||
for (const language of config.languages) {
|
||||
logger.startGroup(`Analyzing ${language}`);
|
||||
const queries = config.queries[language];
|
||||
const packsWithVersion = config.packs[language] || [];
|
||||
if (queries === undefined ||
|
||||
(queries.builtin.length === 0 && queries.custom.length === 0)) {
|
||||
throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
|
||||
|
|
@ -96,7 +97,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
|
|||
const customAnalysisSummaries = [];
|
||||
if (queries["builtin"].length > 0) {
|
||||
const startTimeBuiltIn = new Date().getTime();
|
||||
const { sarifFile, stdout } = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
|
||||
const { sarifFile, stdout } = await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"]), sarifFolder, undefined);
|
||||
analysisSummaryBuiltIn = stdout;
|
||||
await injectLinesOfCode(sarifFile, language, locPromise);
|
||||
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
|
||||
|
|
@ -107,11 +108,16 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
|
|||
const temporarySarifFiles = [];
|
||||
for (let i = 0; i < queries["custom"].length; ++i) {
|
||||
if (queries["custom"][i].queries.length > 0) {
|
||||
const { sarifFile, stdout } = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
|
||||
const { sarifFile, stdout } = await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries), temporarySarifDir, queries["custom"][i].searchPath);
|
||||
customAnalysisSummaries.push(stdout);
|
||||
temporarySarifFiles.push(sarifFile);
|
||||
}
|
||||
}
|
||||
if (packsWithVersion.length > 0) {
|
||||
const { sarifFile, stdout } = await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), temporarySarifDir, undefined);
|
||||
customAnalysisSummaries.push(stdout);
|
||||
temporarySarifFiles.push(sarifFile);
|
||||
}
|
||||
if (temporarySarifFiles.length > 0) {
|
||||
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
|
||||
fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles));
|
||||
|
|
@ -146,16 +152,13 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
|
|||
}
|
||||
}
|
||||
return statusReport;
|
||||
async function runQueryGroup(language, type, queries, destinationFolder, searchPath) {
|
||||
async function runQueryGroup(language, type, querySuiteContents, destinationFolder, searchPath) {
|
||||
const databasePath = util.getCodeQLDatabasePath(config, language);
|
||||
// Pass the queries to codeql using a file instead of using the command
|
||||
// line to avoid command line length restrictions, particularly on windows.
|
||||
const querySuitePath = `${databasePath}-queries-${type}.qls`;
|
||||
const querySuiteContents = queries
|
||||
.map((q) => `- query: ${q}`)
|
||||
.join("\n");
|
||||
fs.writeFileSync(querySuitePath, querySuiteContents);
|
||||
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
|
||||
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
|
||||
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
|
||||
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
|
||||
const databaseAnalyzeStdout = await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
|
||||
|
|
@ -164,6 +167,19 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
|
|||
}
|
||||
}
|
||||
exports.runQueries = runQueries;
|
||||
function createQuerySuiteContents(queries) {
|
||||
return queries.map((q) => `- query: ${q}`).join("\n");
|
||||
}
|
||||
function createPackSuiteContents(packsWithVersion) {
|
||||
return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
|
||||
}
|
||||
function packWithVersionToQuerySuiteEntry(pack) {
|
||||
let text = `- qlpack: ${pack.packName}`;
|
||||
if (pack.version) {
|
||||
text += `${"\n"} version: ${pack.version}`;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
|
||||
// Delete the tracer config env var to avoid tracing ourselves
|
||||
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
72
lib/analyze.test.js
generated
72
lib/analyze.test.js
generated
|
|
@ -13,6 +13,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const ava_1 = __importDefault(require("ava"));
|
||||
const yaml = __importStar(require("js-yaml"));
|
||||
const semver_1 = require("semver");
|
||||
const sinon_1 = __importDefault(require("sinon"));
|
||||
const analyze_1 = require("./analyze");
|
||||
const codeql_1 = require("./codeql");
|
||||
|
|
@ -39,6 +41,20 @@ ava_1.default("status report fields and search path setting", async (t) => {
|
|||
const memoryFlag = "";
|
||||
const addSnippetsFlag = "";
|
||||
const threadsFlag = "";
|
||||
const packs = {
|
||||
[languages_1.Language.cpp]: [
|
||||
{
|
||||
packName: "a/b",
|
||||
version: semver_1.parse("1.0.0"),
|
||||
},
|
||||
],
|
||||
[languages_1.Language.java]: [
|
||||
{
|
||||
packName: "c/d",
|
||||
version: semver_1.parse("2.0.0"),
|
||||
},
|
||||
],
|
||||
};
|
||||
for (const language of Object.values(languages_1.Language)) {
|
||||
codeql_1.setCodeQL({
|
||||
databaseAnalyze: async (_, sarifFile, searchPath) => {
|
||||
|
|
@ -89,6 +105,7 @@ ava_1.default("status report fields and search path setting", async (t) => {
|
|||
type: util.GitHubVariant.DOTCOM,
|
||||
},
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs,
|
||||
};
|
||||
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
|
||||
recursive: true,
|
||||
|
|
@ -98,7 +115,8 @@ ava_1.default("status report fields and search path setting", async (t) => {
|
|||
custom: [],
|
||||
};
|
||||
const builtinStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
|
||||
t.deepEqual(Object.keys(builtinStatusReport).length, 1);
|
||||
const hasPacks = language in packs;
|
||||
t.deepEqual(Object.keys(builtinStatusReport).length, hasPacks ? 2 : 1);
|
||||
t.true(`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport);
|
||||
config.queries[language] = {
|
||||
builtin: [],
|
||||
|
|
@ -116,9 +134,13 @@ ava_1.default("status report fields and search path setting", async (t) => {
|
|||
const customStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
|
||||
t.deepEqual(Object.keys(customStatusReport).length, 1);
|
||||
t.true(`analyze_custom_queries_${language}_duration_ms` in customStatusReport);
|
||||
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
|
||||
const expectedSearchPathsUsed = hasPacks
|
||||
? [undefined, undefined, "/1", "/2", undefined]
|
||||
: [undefined, "/1", "/2"];
|
||||
t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
|
||||
}
|
||||
verifyLineCounts(tmpDir);
|
||||
verifyQuerySuites(tmpDir);
|
||||
});
|
||||
function verifyLineCounts(tmpDir) {
|
||||
// eslint-disable-next-line github/array-foreach
|
||||
|
|
@ -146,8 +168,52 @@ ava_1.default("status report fields and search path setting", async (t) => {
|
|||
baseline: lineCount,
|
||||
},
|
||||
]);
|
||||
// when the rule doesn't exists, it should not be added
|
||||
// when the rule doesn't exist, it should not be added
|
||||
t.deepEqual(sarif.runs[2].properties.metricResults, []);
|
||||
}
|
||||
function verifyQuerySuites(tmpDir) {
|
||||
const qlsContent = [
|
||||
{
|
||||
query: "foo.ql",
|
||||
},
|
||||
];
|
||||
const qlsContent2 = [
|
||||
{
|
||||
query: "bar.ql",
|
||||
},
|
||||
];
|
||||
const qlsPackContentCpp = [
|
||||
{
|
||||
qlpack: "a/b",
|
||||
version: "1.0.0",
|
||||
},
|
||||
];
|
||||
const qlsPackContentJava = [
|
||||
{
|
||||
qlpack: "c/d",
|
||||
version: "2.0.0",
|
||||
},
|
||||
];
|
||||
for (const lang of Object.values(languages_1.Language)) {
|
||||
t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
|
||||
t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
|
||||
t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
|
||||
const packSuiteName = `${lang}-queries-packs.qls`;
|
||||
if (lang === languages_1.Language.cpp) {
|
||||
t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
|
||||
}
|
||||
else if (lang === languages_1.Language.java) {
|
||||
t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
|
||||
}
|
||||
else {
|
||||
t.false(fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName)));
|
||||
}
|
||||
}
|
||||
function readContents(name) {
|
||||
const x = fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8");
|
||||
console.log(x);
|
||||
return yaml.safeLoad(fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8"));
|
||||
}
|
||||
}
|
||||
});
|
||||
//# sourceMappingURL=analyze.test.js.map
|
||||
File diff suppressed because one or more lines are too long
31
lib/codeql.js
generated
31
lib/codeql.js
generated
|
|
@ -286,6 +286,7 @@ function setCodeQL(partialCodeql) {
|
|||
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
|
||||
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
|
||||
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
|
||||
packDownload: resolveFunction(partialCodeql, "packDownload"),
|
||||
};
|
||||
return cachedCodeQL;
|
||||
}
|
||||
|
|
@ -498,8 +499,38 @@ function getCodeQLForCmd(cmd) {
|
|||
}).exec();
|
||||
return output;
|
||||
},
|
||||
// Download specified packs into the package cache. If the specified
|
||||
// package and version already exists (e.g., from a previous analysis run),
|
||||
// then it is not downloaded again (unless the extra option `--force` is
|
||||
// specified).
|
||||
async packDownload(packs) {
|
||||
const args = [
|
||||
"pack",
|
||||
"download",
|
||||
"-v",
|
||||
...getExtraOptionsFromEnv(["pack", "download"]),
|
||||
...packs.map(packWithVersionToString),
|
||||
];
|
||||
let output = "";
|
||||
await new toolrunner.ToolRunner(cmd, args, {
|
||||
listeners: {
|
||||
stdout: (data) => {
|
||||
output += data.toString("utf8");
|
||||
},
|
||||
},
|
||||
}).exec();
|
||||
try {
|
||||
return JSON.parse(output);
|
||||
}
|
||||
catch (e) {
|
||||
throw new Error(`Attempted to download specified packs but got error ${e}.`);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
function packWithVersionToString(pack) {
|
||||
return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
|
||||
}
|
||||
/**
|
||||
* Gets the options for `path` of `options` as an array of extra option strings.
|
||||
*/
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
97
lib/config-utils.js
generated
97
lib/config-utils.js
generated
|
|
@ -10,6 +10,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const yaml = __importStar(require("js-yaml"));
|
||||
const semver = __importStar(require("semver"));
|
||||
const api = __importStar(require("./api-client"));
|
||||
const externalQueries = __importStar(require("./external-queries"));
|
||||
const languages_1 = require("./languages");
|
||||
|
|
@ -20,6 +21,7 @@ const QUERIES_PROPERTY = "queries";
|
|||
const QUERIES_USES_PROPERTY = "uses";
|
||||
const PATHS_IGNORE_PROPERTY = "paths-ignore";
|
||||
const PATHS_PROPERTY = "paths";
|
||||
const PACKS_PROPERTY = "packs";
|
||||
/**
|
||||
* A list of queries from https://github.com/github/codeql that
|
||||
* we don't want to run. Disabling them here is a quicker alternative to
|
||||
|
|
@ -254,6 +256,22 @@ function getPathsInvalid(configFile) {
|
|||
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, "must be an array of non-empty strings");
|
||||
}
|
||||
exports.getPathsInvalid = getPathsInvalid;
|
||||
function getPacksRequireLanguage(lang, configFile) {
|
||||
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `has "${lang}", but it is not one of the languages to analyze`);
|
||||
}
|
||||
exports.getPacksRequireLanguage = getPacksRequireLanguage;
|
||||
function getPacksInvalidSplit(configFile) {
|
||||
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must split packages by language");
|
||||
}
|
||||
exports.getPacksInvalidSplit = getPacksInvalidSplit;
|
||||
function getPacksInvalid(configFile) {
|
||||
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must be an array of non-empty strings");
|
||||
}
|
||||
exports.getPacksInvalid = getPacksInvalid;
|
||||
function getPacksStrInvalid(packStr, configFile) {
|
||||
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `"${packStr}" is not a valid pack`);
|
||||
}
|
||||
exports.getPacksStrInvalid = getPacksStrInvalid;
|
||||
function getLocalPathOutsideOfRepository(configFile, localPath) {
|
||||
return getConfigFilePropertyError(configFile, `${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`, `is invalid as the local path "${localPath}" is outside of the repository`);
|
||||
}
|
||||
|
|
@ -409,6 +427,7 @@ async function getDefaultConfig(languagesInput, queriesInput, dbLocation, reposi
|
|||
queries,
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
packs: {},
|
||||
originalUserInput: {},
|
||||
tempDir,
|
||||
toolCacheDir,
|
||||
|
|
@ -470,7 +489,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
|
|||
}
|
||||
if (shouldAddConfigFileQueries(queriesInput) &&
|
||||
QUERIES_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
|
||||
if (!Array.isArray(parsedYAML[QUERIES_PROPERTY])) {
|
||||
throw new Error(getQueriesInvalid(configFile));
|
||||
}
|
||||
for (const query of parsedYAML[QUERIES_PROPERTY]) {
|
||||
|
|
@ -482,7 +501,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
|
|||
}
|
||||
}
|
||||
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
|
||||
if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
|
||||
throw new Error(getPathsIgnoreInvalid(configFile));
|
||||
}
|
||||
for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]) {
|
||||
|
|
@ -493,7 +512,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
|
|||
}
|
||||
}
|
||||
if (PATHS_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
|
||||
if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
|
||||
throw new Error(getPathsInvalid(configFile));
|
||||
}
|
||||
for (const includePath of parsedYAML[PATHS_PROPERTY]) {
|
||||
|
|
@ -503,11 +522,13 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
|
|||
paths.push(validateAndSanitisePath(includePath, PATHS_PROPERTY, configFile, logger));
|
||||
}
|
||||
}
|
||||
const packs = parsePacks(parsedYAML[PACKS_PROPERTY], languages, configFile);
|
||||
return {
|
||||
languages,
|
||||
queries,
|
||||
pathsIgnore,
|
||||
paths,
|
||||
packs,
|
||||
originalUserInput: parsedYAML,
|
||||
tempDir,
|
||||
toolCacheDir,
|
||||
|
|
@ -516,6 +537,68 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
|
|||
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
|
||||
};
|
||||
}
|
||||
// Only alpha-numeric characters, with `-` allowed as long as not the first or last char
|
||||
const PACK_IDENTIFIER_PATTERN = (function () {
|
||||
const alphaNumeric = "[a-z0-9]";
|
||||
const alphaNumericDash = "[a-z0-9-]";
|
||||
const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
|
||||
return new RegExp(`^${component}/${component}$`);
|
||||
})();
|
||||
// Exported for testing
|
||||
function parsePacks(packsByLanguage, languages, configFile) {
|
||||
const packs = {};
|
||||
if (!packsByLanguage) {
|
||||
return packs;
|
||||
}
|
||||
if (Array.isArray(packsByLanguage)) {
|
||||
if (languages.length === 1) {
|
||||
// single language analysis, so language is implicit
|
||||
packsByLanguage = {
|
||||
[languages[0]]: packsByLanguage,
|
||||
};
|
||||
}
|
||||
else {
|
||||
// this is an error since multi-language analysis requires
|
||||
// packs split by language
|
||||
throw new Error(getPacksInvalidSplit(configFile));
|
||||
}
|
||||
}
|
||||
for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
|
||||
if (!Array.isArray(packsArr)) {
|
||||
throw new Error(getPacksInvalid(configFile));
|
||||
}
|
||||
if (!languages.includes(lang)) {
|
||||
throw new Error(getPacksRequireLanguage(lang, configFile));
|
||||
}
|
||||
packs[lang] = [];
|
||||
for (const packStr of packsArr) {
|
||||
packs[lang].push(toPackWithVersion(packStr, configFile));
|
||||
}
|
||||
}
|
||||
return packs;
|
||||
}
|
||||
exports.parsePacks = parsePacks;
|
||||
function toPackWithVersion(packStr, configFile) {
|
||||
if (typeof packStr !== "string") {
|
||||
throw new Error(getPacksStrInvalid(packStr, configFile));
|
||||
}
|
||||
const nameWithVersion = packStr.split("@");
|
||||
let version;
|
||||
if (nameWithVersion.length > 2 ||
|
||||
!PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])) {
|
||||
throw new Error(getPacksStrInvalid(packStr, configFile));
|
||||
}
|
||||
else if (nameWithVersion.length === 2) {
|
||||
version = semver.parse(nameWithVersion[1]) || undefined;
|
||||
if (!version) {
|
||||
throw new Error(getPacksStrInvalid(packStr, configFile));
|
||||
}
|
||||
}
|
||||
return {
|
||||
packName: nameWithVersion[0],
|
||||
version,
|
||||
};
|
||||
}
|
||||
function dbLocationOrDefault(dbLocation, tempDir) {
|
||||
return dbLocation || path.resolve(tempDir, "codeql_databases");
|
||||
}
|
||||
|
|
@ -526,6 +609,7 @@ function dbLocationOrDefault(dbLocation, tempDir) {
|
|||
* a default config. The parsed config is then stored to a known location.
|
||||
*/
|
||||
async function initConfig(languagesInput, queriesInput, configFile, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
|
||||
var _a, _b, _c;
|
||||
let config;
|
||||
// If no config file was provided create an empty one
|
||||
if (!configFile) {
|
||||
|
|
@ -538,9 +622,10 @@ async function initConfig(languagesInput, queriesInput, configFile, dbLocation,
|
|||
// The list of queries should not be empty for any language. If it is then
|
||||
// it is a user configuration error.
|
||||
for (const language of config.languages) {
|
||||
if (config.queries[language] === undefined ||
|
||||
(config.queries[language].builtin.length === 0 &&
|
||||
config.queries[language].custom.length === 0)) {
|
||||
const hasPacks = ((_a = config.packs[language]) === null || _a === void 0 ? void 0 : _a.length) > 0;
|
||||
const hasQueries = ((_b = config.queries[language]) === null || _b === void 0 ? void 0 : _b.builtin.length) > 0 ||
|
||||
((_c = config.queries[language]) === null || _c === void 0 ? void 0 : _c.custom.length) > 0;
|
||||
if (!hasPacks && !hasQueries) {
|
||||
throw new Error(`Did not detect any queries to run for ${language}. ` +
|
||||
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
147
lib/config-utils.test.js
generated
147
lib/config-utils.test.js
generated
|
|
@ -14,6 +14,7 @@ const fs = __importStar(require("fs"));
|
|||
const path = __importStar(require("path"));
|
||||
const github = __importStar(require("@actions/github"));
|
||||
const ava_1 = __importDefault(require("ava"));
|
||||
const semver_1 = require("semver");
|
||||
const sinon_1 = __importDefault(require("sinon"));
|
||||
const api = __importStar(require("./api-client"));
|
||||
const codeql_1 = require("./codeql");
|
||||
|
|
@ -200,6 +201,7 @@ ava_1.default("load non-empty input", async (t) => {
|
|||
codeQLCmd: codeQL.getPath(),
|
||||
gitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {},
|
||||
};
|
||||
const languages = "javascript";
|
||||
const configFilePath = createConfigFile(inputFileContents, tmpDir);
|
||||
|
|
@ -557,6 +559,101 @@ ava_1.default("Unknown languages", async (t) => {
|
|||
}
|
||||
});
|
||||
});
|
||||
ava_1.default("Config specifies packages", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
async resolveQueries() {
|
||||
return {
|
||||
byLanguage: {},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
disable-default-queries: true
|
||||
packs:
|
||||
- a/b@1.2.3
|
||||
`;
|
||||
const configFile = path.join(tmpDir, "codeql-config.yaml");
|
||||
fs.writeFileSync(configFile, inputFileContents);
|
||||
const languages = "javascript";
|
||||
const { packs } = await configUtils.initConfig(languages, undefined, configFile, undefined, { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
|
||||
t.deepEqual(packs, {
|
||||
[languages_1.Language.javascript]: [
|
||||
{
|
||||
packName: "a/b",
|
||||
version: semver_1.parse("1.2.3"),
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
ava_1.default("Config specifies packages for multiple languages", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
async resolveQueries() {
|
||||
return {
|
||||
byLanguage: {
|
||||
cpp: { "/foo/a.ql": {} },
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
disable-default-queries: true
|
||||
queries:
|
||||
- uses: ./foo
|
||||
packs:
|
||||
javascript:
|
||||
- a/b@1.2.3
|
||||
python:
|
||||
- c/d@1.2.3
|
||||
`;
|
||||
const configFile = path.join(tmpDir, "codeql-config.yaml");
|
||||
fs.writeFileSync(configFile, inputFileContents);
|
||||
fs.mkdirSync(path.join(tmpDir, "foo"));
|
||||
const languages = "javascript,python,cpp";
|
||||
const { packs, queries } = await configUtils.initConfig(languages, undefined, configFile, undefined, { owner: "github", repo: "example" }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
|
||||
t.deepEqual(packs, {
|
||||
[languages_1.Language.javascript]: [
|
||||
{
|
||||
packName: "a/b",
|
||||
version: semver_1.parse("1.2.3"),
|
||||
},
|
||||
],
|
||||
[languages_1.Language.python]: [
|
||||
{
|
||||
packName: "c/d",
|
||||
version: semver_1.parse("1.2.3"),
|
||||
},
|
||||
],
|
||||
});
|
||||
t.deepEqual(queries, {
|
||||
cpp: {
|
||||
builtin: [],
|
||||
custom: [
|
||||
{
|
||||
queries: ["/foo/a.ql"],
|
||||
searchPath: tmpDir,
|
||||
},
|
||||
],
|
||||
},
|
||||
javascript: {
|
||||
builtin: [],
|
||||
custom: [],
|
||||
},
|
||||
python: {
|
||||
builtin: [],
|
||||
custom: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
function doInvalidInputTest(testName, inputFileContents, expectedErrorMessageGenerator) {
|
||||
ava_1.default(`load invalid input - ${testName}`, async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
|
|
@ -644,4 +741,54 @@ ava_1.default("path sanitisation", (t) => {
|
|||
// Trailing stars are stripped
|
||||
t.deepEqual(configUtils.validateAndSanitisePath("foo/**", propertyName, configFile, logging_1.getRunnerLogger(true)), "foo/");
|
||||
});
|
||||
/**
|
||||
* Test macro for ensuring the packs block is valid
|
||||
*/
|
||||
function parsePacksMacro(t, packsByLanguage, languages, expected) {
|
||||
t.deepEqual(configUtils.parsePacks(packsByLanguage, languages, "/a/b"), expected);
|
||||
}
|
||||
parsePacksMacro.title = (providedTitle) => `Parse Packs: ${providedTitle}`;
|
||||
/**
|
||||
* Test macro for testing when the packs block is invalid
|
||||
*/
|
||||
function parsePacksErrorMacro(t, packsByLanguage, languages, expected) {
|
||||
t.throws(() => {
|
||||
configUtils.parsePacks(packsByLanguage, languages, "/a/b");
|
||||
}, {
|
||||
message: expected,
|
||||
});
|
||||
}
|
||||
parsePacksErrorMacro.title = (providedTitle) => `Parse Packs Error: ${providedTitle}`;
|
||||
function invalidPackNameMacro(t, name) {
|
||||
parsePacksErrorMacro(t, { [languages_1.Language.cpp]: [name] }, [languages_1.Language.cpp], new RegExp(`The configuration file "/a/b" is invalid: property "packs" "${name}" is not a valid pack`));
|
||||
}
|
||||
invalidPackNameMacro.title = (_, arg) => `Invalid pack string: ${arg}`;
|
||||
ava_1.default("no packs", parsePacksMacro, undefined, [], {});
|
||||
ava_1.default("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [languages_1.Language.cpp], {
|
||||
[languages_1.Language.cpp]: [
|
||||
{ packName: "a/b", version: undefined },
|
||||
{ packName: "c/d", version: semver_1.parse("1.2.3") },
|
||||
],
|
||||
});
|
||||
ava_1.default("two packs with language", parsePacksMacro, {
|
||||
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
|
||||
[languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
|
||||
}, [languages_1.Language.cpp, languages_1.Language.java, languages_1.Language.csharp], {
|
||||
[languages_1.Language.cpp]: [
|
||||
{ packName: "a/b", version: undefined },
|
||||
{ packName: "c/d", version: semver_1.parse("1.2.3") },
|
||||
],
|
||||
[languages_1.Language.java]: [
|
||||
{ packName: "d/e", version: undefined },
|
||||
{ packName: "f/g", version: semver_1.parse("1.2.3") },
|
||||
],
|
||||
});
|
||||
ava_1.default("no language", parsePacksErrorMacro, ["a/b@1.2.3"], [languages_1.Language.java, languages_1.Language.python], /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/);
|
||||
ava_1.default("invalid language", parsePacksErrorMacro, { [languages_1.Language.java]: ["c/d"] }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/);
|
||||
ava_1.default("not an array", parsePacksErrorMacro, { [languages_1.Language.cpp]: "c/d" }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/);
|
||||
ava_1.default(invalidPackNameMacro, "c");
|
||||
ava_1.default(invalidPackNameMacro, "c-/d");
|
||||
ava_1.default(invalidPackNameMacro, "-c/d");
|
||||
ava_1.default(invalidPackNameMacro, "c/d_d");
|
||||
ava_1.default(invalidPackNameMacro, "c/d@x");
|
||||
//# sourceMappingURL=config-utils.test.js.map
|
||||
File diff suppressed because one or more lines are too long
1
lib/tracer-config.test.js
generated
1
lib/tracer-config.test.js
generated
|
|
@ -31,6 +31,7 @@ function getTestConfig(tmpDir) {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {},
|
||||
};
|
||||
}
|
||||
// A very minimal setup
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
10
node_modules/.package-lock.json
generated
vendored
10
node_modules/.package-lock.json
generated
vendored
|
|
@ -405,6 +405,12 @@
|
|||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/js-yaml": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.1.tgz",
|
||||
"integrity": "sha512-xdOvNmXmrZqqPy3kuCQ+fz6wA0xU5pji9cd1nDrflWaAWtYLLGk5ykW0H6yg5TVyehHP1pfmuuSaZkhP+kspVA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/json-schema": {
|
||||
"version": "7.0.6",
|
||||
"dev": true,
|
||||
|
|
@ -1253,6 +1259,7 @@
|
|||
"dependencies": {
|
||||
"anymatch": "~3.1.1",
|
||||
"braces": "~3.0.2",
|
||||
"fsevents": "~2.1.2",
|
||||
"glob-parent": "~5.1.0",
|
||||
"is-binary-path": "~2.1.0",
|
||||
"is-glob": "~4.0.1",
|
||||
|
|
@ -3238,6 +3245,9 @@
|
|||
"node_modules/jsonfile": {
|
||||
"version": "4.0.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"graceful-fs": "^4.1.6"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"graceful-fs": "^4.1.6"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import * as path from "path";
|
|||
import test from "ava";
|
||||
|
||||
import * as analysisPaths from "./analysis-paths";
|
||||
import { Packs } from "./config-utils";
|
||||
import { setupTests } from "./testing-utils";
|
||||
import * as util from "./util";
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ test("emptyPaths", async (t) => {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {} as Packs,
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
@ -42,6 +44,7 @@ test("nonEmptyPaths", async (t) => {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {} as Packs,
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
|
||||
|
|
@ -67,6 +70,7 @@ test("exclude temp dir", async (t) => {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
dbLocation: path.resolve(tempDir, "codeql_databases"),
|
||||
packs: {} as Packs,
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@ import * as fs from "fs";
|
|||
import * as path from "path";
|
||||
|
||||
import test from "ava";
|
||||
import * as yaml from "js-yaml";
|
||||
import { parse } from "semver";
|
||||
import sinon from "sinon";
|
||||
|
||||
import { runQueries } from "./analyze";
|
||||
import { setCodeQL } from "./codeql";
|
||||
import { Config } from "./config-utils";
|
||||
import { Config, Packs } from "./config-utils";
|
||||
import { getIdPrefix } from "./count-loc";
|
||||
import * as count from "./count-loc";
|
||||
import { Language } from "./languages";
|
||||
|
|
@ -26,13 +28,27 @@ test("status report fields and search path setting", async (t) => {
|
|||
return obj;
|
||||
}, {});
|
||||
sinon.stub(count, "countLoc").resolves(mockLinesOfCode);
|
||||
let searchPathsUsed: string[] = [];
|
||||
let searchPathsUsed: Array<string | undefined> = [];
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
const memoryFlag = "";
|
||||
const addSnippetsFlag = "";
|
||||
const threadsFlag = "";
|
||||
const packs = {
|
||||
[Language.cpp]: [
|
||||
{
|
||||
packName: "a/b",
|
||||
version: parse("1.0.0"),
|
||||
},
|
||||
],
|
||||
[Language.java]: [
|
||||
{
|
||||
packName: "c/d",
|
||||
version: parse("2.0.0"),
|
||||
},
|
||||
],
|
||||
} as Packs;
|
||||
|
||||
for (const language of Object.values(Language)) {
|
||||
setCodeQL({
|
||||
|
|
@ -75,7 +91,7 @@ test("status report fields and search path setting", async (t) => {
|
|||
],
|
||||
})
|
||||
);
|
||||
searchPathsUsed.push(searchPath!);
|
||||
searchPathsUsed.push(searchPath);
|
||||
return "";
|
||||
},
|
||||
});
|
||||
|
|
@ -94,6 +110,7 @@ test("status report fields and search path setting", async (t) => {
|
|||
type: util.GitHubVariant.DOTCOM,
|
||||
} as util.GitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs,
|
||||
};
|
||||
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
|
||||
recursive: true,
|
||||
|
|
@ -112,7 +129,8 @@ test("status report fields and search path setting", async (t) => {
|
|||
config,
|
||||
getRunnerLogger(true)
|
||||
);
|
||||
t.deepEqual(Object.keys(builtinStatusReport).length, 1);
|
||||
const hasPacks = language in packs;
|
||||
t.deepEqual(Object.keys(builtinStatusReport).length, hasPacks ? 2 : 1);
|
||||
t.true(
|
||||
`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport
|
||||
);
|
||||
|
|
@ -143,10 +161,14 @@ test("status report fields and search path setting", async (t) => {
|
|||
t.true(
|
||||
`analyze_custom_queries_${language}_duration_ms` in customStatusReport
|
||||
);
|
||||
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
|
||||
const expectedSearchPathsUsed = hasPacks
|
||||
? [undefined, undefined, "/1", "/2", undefined]
|
||||
: [undefined, "/1", "/2"];
|
||||
t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
|
||||
}
|
||||
|
||||
verifyLineCounts(tmpDir);
|
||||
verifyQuerySuites(tmpDir);
|
||||
});
|
||||
|
||||
function verifyLineCounts(tmpDir: string) {
|
||||
|
|
@ -188,7 +210,59 @@ test("status report fields and search path setting", async (t) => {
|
|||
baseline: lineCount,
|
||||
},
|
||||
]);
|
||||
// when the rule doesn't exists, it should not be added
|
||||
// when the rule doesn't exist, it should not be added
|
||||
t.deepEqual(sarif.runs[2].properties.metricResults, []);
|
||||
}
|
||||
|
||||
function verifyQuerySuites(tmpDir: string) {
|
||||
const qlsContent = [
|
||||
{
|
||||
query: "foo.ql",
|
||||
},
|
||||
];
|
||||
const qlsContent2 = [
|
||||
{
|
||||
query: "bar.ql",
|
||||
},
|
||||
];
|
||||
const qlsPackContentCpp = [
|
||||
{
|
||||
qlpack: "a/b",
|
||||
version: "1.0.0",
|
||||
},
|
||||
];
|
||||
const qlsPackContentJava = [
|
||||
{
|
||||
qlpack: "c/d",
|
||||
version: "2.0.0",
|
||||
},
|
||||
];
|
||||
for (const lang of Object.values(Language)) {
|
||||
t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
|
||||
t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
|
||||
t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
|
||||
const packSuiteName = `${lang}-queries-packs.qls`;
|
||||
if (lang === Language.cpp) {
|
||||
t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
|
||||
} else if (lang === Language.java) {
|
||||
t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
|
||||
} else {
|
||||
t.false(
|
||||
fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function readContents(name: string) {
|
||||
const x = fs.readFileSync(
|
||||
path.join(tmpDir, "codeql_databases", name),
|
||||
"utf8"
|
||||
);
|
||||
console.log(x);
|
||||
|
||||
return yaml.safeLoad(
|
||||
fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8")
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ export async function runQueries(
|
|||
logger.startGroup(`Analyzing ${language}`);
|
||||
|
||||
const queries = config.queries[language];
|
||||
const packsWithVersion = config.packs[language] || [];
|
||||
if (
|
||||
queries === undefined ||
|
||||
(queries.builtin.length === 0 && queries.custom.length === 0)
|
||||
|
|
@ -183,7 +184,7 @@ export async function runQueries(
|
|||
const { sarifFile, stdout } = await runQueryGroup(
|
||||
language,
|
||||
"builtin",
|
||||
queries["builtin"],
|
||||
createQuerySuiteContents(queries["builtin"]),
|
||||
sarifFolder,
|
||||
undefined
|
||||
);
|
||||
|
|
@ -201,7 +202,7 @@ export async function runQueries(
|
|||
const { sarifFile, stdout } = await runQueryGroup(
|
||||
language,
|
||||
`custom-${i}`,
|
||||
queries["custom"][i].queries,
|
||||
createQuerySuiteContents(queries["custom"][i].queries),
|
||||
temporarySarifDir,
|
||||
queries["custom"][i].searchPath
|
||||
);
|
||||
|
|
@ -209,6 +210,17 @@ export async function runQueries(
|
|||
temporarySarifFiles.push(sarifFile);
|
||||
}
|
||||
}
|
||||
if (packsWithVersion.length > 0) {
|
||||
const { sarifFile, stdout } = await runQueryGroup(
|
||||
language,
|
||||
"packs",
|
||||
createPackSuiteContents(packsWithVersion),
|
||||
temporarySarifDir,
|
||||
undefined
|
||||
);
|
||||
customAnalysisSummaries.push(stdout);
|
||||
temporarySarifFiles.push(sarifFile);
|
||||
}
|
||||
if (temporarySarifFiles.length > 0) {
|
||||
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
|
||||
fs.writeFileSync(sarifFile, combineSarifFiles(temporarySarifFiles));
|
||||
|
|
@ -256,7 +268,7 @@ export async function runQueries(
|
|||
async function runQueryGroup(
|
||||
language: Language,
|
||||
type: string,
|
||||
queries: string[],
|
||||
querySuiteContents: string,
|
||||
destinationFolder: string,
|
||||
searchPath: string | undefined
|
||||
): Promise<{ sarifFile: string; stdout: string }> {
|
||||
|
|
@ -264,11 +276,10 @@ export async function runQueries(
|
|||
// Pass the queries to codeql using a file instead of using the command
|
||||
// line to avoid command line length restrictions, particularly on windows.
|
||||
const querySuitePath = `${databasePath}-queries-${type}.qls`;
|
||||
const querySuiteContents = queries
|
||||
.map((q: string) => `- query: ${q}`)
|
||||
.join("\n");
|
||||
fs.writeFileSync(querySuitePath, querySuiteContents);
|
||||
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
|
||||
logger.debug(
|
||||
`Query suite file for ${language}-${type}...\n${querySuiteContents}`
|
||||
);
|
||||
|
||||
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
|
||||
|
||||
|
|
@ -291,6 +302,26 @@ export async function runQueries(
|
|||
}
|
||||
}
|
||||
|
||||
function createQuerySuiteContents(queries: string[]) {
|
||||
return queries.map((q: string) => `- query: ${q}`).join("\n");
|
||||
}
|
||||
|
||||
function createPackSuiteContents(
|
||||
packsWithVersion: configUtils.PackWithVersion[]
|
||||
) {
|
||||
return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
|
||||
}
|
||||
|
||||
function packWithVersionToQuerySuiteEntry(
|
||||
pack: configUtils.PackWithVersion
|
||||
): string {
|
||||
let text = `- qlpack: ${pack.packName}`;
|
||||
if (pack.version) {
|
||||
text += `${"\n"} version: ${pack.version}`;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
export async function runAnalyze(
|
||||
outputDir: string,
|
||||
memoryFlag: string,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import { v4 as uuidV4 } from "uuid";
|
|||
|
||||
import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
|
||||
import * as api from "./api-client";
|
||||
import { PackWithVersion } from "./config-utils";
|
||||
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
|
||||
import { errorMatchers } from "./error-matcher";
|
||||
import { Language } from "./languages";
|
||||
|
|
@ -88,6 +89,12 @@ export interface CodeQL {
|
|||
queries: string[],
|
||||
extraSearchPath: string | undefined
|
||||
): Promise<ResolveQueriesOutput>;
|
||||
|
||||
/**
|
||||
* Run 'codeql pack download'.
|
||||
*/
|
||||
packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput>;
|
||||
|
||||
/**
|
||||
* Run 'codeql database analyze'.
|
||||
*/
|
||||
|
|
@ -121,6 +128,17 @@ export interface ResolveQueriesOutput {
|
|||
};
|
||||
}
|
||||
|
||||
export interface PackDownloadOutput {
|
||||
packs: PackDownloadItem[];
|
||||
}
|
||||
|
||||
interface PackDownloadItem {
|
||||
name: string;
|
||||
version: string;
|
||||
packDir: string;
|
||||
installResult: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
|
||||
* Can be overridden in tests using `setCodeQL`.
|
||||
|
|
@ -481,6 +499,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
|
|||
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
|
||||
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
|
||||
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
|
||||
packDownload: resolveFunction(partialCodeql, "packDownload"),
|
||||
};
|
||||
return cachedCodeQL;
|
||||
}
|
||||
|
|
@ -747,9 +766,43 @@ function getCodeQLForCmd(cmd: string): CodeQL {
|
|||
}).exec();
|
||||
return output;
|
||||
},
|
||||
|
||||
// Download specified packs into the package cache. If the specified
|
||||
// package and version already exists (e.g., from a previous analysis run),
|
||||
// then it is not downloaded again (unless the extra option `--force` is
|
||||
// specified).
|
||||
async packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput> {
|
||||
const args = [
|
||||
"pack",
|
||||
"download",
|
||||
"-v",
|
||||
...getExtraOptionsFromEnv(["pack", "download"]),
|
||||
...packs.map(packWithVersionToString),
|
||||
];
|
||||
|
||||
let output = "";
|
||||
await new toolrunner.ToolRunner(cmd, args, {
|
||||
listeners: {
|
||||
stdout: (data: Buffer) => {
|
||||
output += data.toString("utf8");
|
||||
},
|
||||
},
|
||||
}).exec();
|
||||
|
||||
try {
|
||||
return JSON.parse(output) as PackDownloadOutput;
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Attempted to download specified packs but got error ${e}.`
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function packWithVersionToString(pack: PackWithVersion): string {
|
||||
return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
|
||||
}
|
||||
/**
|
||||
* Gets the options for `path` of `options` as an array of extra option strings.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import * as fs from "fs";
|
|||
import * as path from "path";
|
||||
|
||||
import * as github from "@actions/github";
|
||||
import test from "ava";
|
||||
import test, { ExecutionContext } from "ava";
|
||||
import { parse } from "semver";
|
||||
import sinon from "sinon";
|
||||
|
||||
import * as api from "./api-client";
|
||||
|
|
@ -318,6 +319,7 @@ test("load non-empty input", async (t) => {
|
|||
codeQLCmd: codeQL.getPath(),
|
||||
gitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {} as configUtils.Packs,
|
||||
};
|
||||
|
||||
const languages = "javascript";
|
||||
|
|
@ -983,6 +985,137 @@ test("Unknown languages", async (t) => {
|
|||
});
|
||||
});
|
||||
|
||||
test("Config specifies packages", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
const codeQL = setCodeQL({
|
||||
async resolveQueries() {
|
||||
return {
|
||||
byLanguage: {},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
disable-default-queries: true
|
||||
packs:
|
||||
- a/b@1.2.3
|
||||
`;
|
||||
|
||||
const configFile = path.join(tmpDir, "codeql-config.yaml");
|
||||
fs.writeFileSync(configFile, inputFileContents);
|
||||
|
||||
const languages = "javascript";
|
||||
|
||||
const { packs } = await configUtils.initConfig(
|
||||
languages,
|
||||
undefined,
|
||||
configFile,
|
||||
undefined,
|
||||
{ owner: "github", repo: "example " },
|
||||
tmpDir,
|
||||
tmpDir,
|
||||
codeQL,
|
||||
tmpDir,
|
||||
gitHubVersion,
|
||||
sampleApiDetails,
|
||||
getRunnerLogger(true)
|
||||
);
|
||||
t.deepEqual(packs as unknown, {
|
||||
[Language.javascript]: [
|
||||
{
|
||||
packName: "a/b",
|
||||
version: parse("1.2.3"),
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("Config specifies packages for multiple languages", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
const codeQL = setCodeQL({
|
||||
async resolveQueries() {
|
||||
return {
|
||||
byLanguage: {
|
||||
cpp: { "/foo/a.ql": {} },
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
disable-default-queries: true
|
||||
queries:
|
||||
- uses: ./foo
|
||||
packs:
|
||||
javascript:
|
||||
- a/b@1.2.3
|
||||
python:
|
||||
- c/d@1.2.3
|
||||
`;
|
||||
|
||||
const configFile = path.join(tmpDir, "codeql-config.yaml");
|
||||
fs.writeFileSync(configFile, inputFileContents);
|
||||
fs.mkdirSync(path.join(tmpDir, "foo"));
|
||||
|
||||
const languages = "javascript,python,cpp";
|
||||
|
||||
const { packs, queries } = await configUtils.initConfig(
|
||||
languages,
|
||||
undefined,
|
||||
configFile,
|
||||
undefined,
|
||||
{ owner: "github", repo: "example" },
|
||||
tmpDir,
|
||||
tmpDir,
|
||||
codeQL,
|
||||
tmpDir,
|
||||
gitHubVersion,
|
||||
sampleApiDetails,
|
||||
getRunnerLogger(true)
|
||||
);
|
||||
t.deepEqual(packs as unknown, {
|
||||
[Language.javascript]: [
|
||||
{
|
||||
packName: "a/b",
|
||||
version: parse("1.2.3"),
|
||||
},
|
||||
],
|
||||
[Language.python]: [
|
||||
{
|
||||
packName: "c/d",
|
||||
version: parse("1.2.3"),
|
||||
},
|
||||
],
|
||||
});
|
||||
t.deepEqual(queries, {
|
||||
cpp: {
|
||||
builtin: [],
|
||||
custom: [
|
||||
{
|
||||
queries: ["/foo/a.ql"],
|
||||
searchPath: tmpDir,
|
||||
},
|
||||
],
|
||||
},
|
||||
javascript: {
|
||||
builtin: [],
|
||||
custom: [],
|
||||
},
|
||||
python: {
|
||||
builtin: [],
|
||||
custom: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function doInvalidInputTest(
|
||||
testName: string,
|
||||
inputFileContents: string,
|
||||
|
|
@ -1177,3 +1310,109 @@ test("path sanitisation", (t) => {
|
|||
"foo/"
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* Test macro for ensuring the packs block is valid
|
||||
*/
|
||||
function parsePacksMacro(
|
||||
t: ExecutionContext<unknown>,
|
||||
packsByLanguage,
|
||||
languages: Language[],
|
||||
expected
|
||||
) {
|
||||
t.deepEqual(
|
||||
configUtils.parsePacks(packsByLanguage, languages, "/a/b"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
parsePacksMacro.title = (providedTitle: string) =>
|
||||
`Parse Packs: ${providedTitle}`;
|
||||
|
||||
/**
|
||||
* Test macro for testing when the packs block is invalid
|
||||
*/
|
||||
function parsePacksErrorMacro(
|
||||
t: ExecutionContext<unknown>,
|
||||
packsByLanguage,
|
||||
languages: Language[],
|
||||
expected: RegExp
|
||||
) {
|
||||
t.throws(
|
||||
() => {
|
||||
configUtils.parsePacks(packsByLanguage, languages, "/a/b");
|
||||
},
|
||||
{
|
||||
message: expected,
|
||||
}
|
||||
);
|
||||
}
|
||||
parsePacksErrorMacro.title = (providedTitle: string) =>
|
||||
`Parse Packs Error: ${providedTitle}`;
|
||||
|
||||
function invalidPackNameMacro(t: ExecutionContext<unknown>, name: string) {
|
||||
parsePacksErrorMacro(
|
||||
t,
|
||||
{ [Language.cpp]: [name] },
|
||||
[Language.cpp],
|
||||
new RegExp(
|
||||
`The configuration file "/a/b" is invalid: property "packs" "${name}" is not a valid pack`
|
||||
)
|
||||
);
|
||||
}
|
||||
invalidPackNameMacro.title = (_: string, arg: string) =>
|
||||
`Invalid pack string: ${arg}`;
|
||||
|
||||
test("no packs", parsePacksMacro, undefined, [], {});
|
||||
test("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [Language.cpp], {
|
||||
[Language.cpp]: [
|
||||
{ packName: "a/b", version: undefined },
|
||||
{ packName: "c/d", version: parse("1.2.3") },
|
||||
],
|
||||
});
|
||||
test(
|
||||
"two packs with language",
|
||||
parsePacksMacro,
|
||||
{
|
||||
[Language.cpp]: ["a/b", "c/d@1.2.3"],
|
||||
[Language.java]: ["d/e", "f/g@1.2.3"],
|
||||
},
|
||||
[Language.cpp, Language.java, Language.csharp],
|
||||
{
|
||||
[Language.cpp]: [
|
||||
{ packName: "a/b", version: undefined },
|
||||
{ packName: "c/d", version: parse("1.2.3") },
|
||||
],
|
||||
[Language.java]: [
|
||||
{ packName: "d/e", version: undefined },
|
||||
{ packName: "f/g", version: parse("1.2.3") },
|
||||
],
|
||||
}
|
||||
);
|
||||
|
||||
test(
|
||||
"no language",
|
||||
parsePacksErrorMacro,
|
||||
["a/b@1.2.3"],
|
||||
[Language.java, Language.python],
|
||||
/The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/
|
||||
);
|
||||
test(
|
||||
"invalid language",
|
||||
parsePacksErrorMacro,
|
||||
{ [Language.java]: ["c/d"] },
|
||||
[Language.cpp],
|
||||
/The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/
|
||||
);
|
||||
test(
|
||||
"not an array",
|
||||
parsePacksErrorMacro,
|
||||
{ [Language.cpp]: "c/d" },
|
||||
[Language.cpp],
|
||||
/The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/
|
||||
);
|
||||
|
||||
test(invalidPackNameMacro, "c");
|
||||
test(invalidPackNameMacro, "c-/d");
|
||||
test(invalidPackNameMacro, "-c/d");
|
||||
test(invalidPackNameMacro, "c/d_d");
|
||||
test(invalidPackNameMacro, "c/d@x");
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import * as fs from "fs";
|
|||
import * as path from "path";
|
||||
|
||||
import * as yaml from "js-yaml";
|
||||
import * as semver from "semver";
|
||||
|
||||
import * as api from "./api-client";
|
||||
import { CodeQL, ResolveQueriesOutput } from "./codeql";
|
||||
|
|
@ -18,6 +19,7 @@ const QUERIES_PROPERTY = "queries";
|
|||
const QUERIES_USES_PROPERTY = "uses";
|
||||
const PATHS_IGNORE_PROPERTY = "paths-ignore";
|
||||
const PATHS_PROPERTY = "paths";
|
||||
const PACKS_PROPERTY = "packs";
|
||||
|
||||
/**
|
||||
* Format of the config file supplied by the user.
|
||||
|
|
@ -31,6 +33,11 @@ export interface UserConfig {
|
|||
}>;
|
||||
"paths-ignore"?: string[];
|
||||
paths?: string[];
|
||||
|
||||
// If this is a multi-language analysis, then the packages must be split by
|
||||
// language. If this is a single language analysis, then no split by
|
||||
// language is necessary.
|
||||
packs?: Record<string, string[]> | string[];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -114,6 +121,19 @@ export interface Config {
|
|||
* The location where CodeQL databases should be stored.
|
||||
*/
|
||||
dbLocation: string;
|
||||
/**
|
||||
* List of packages, separated by language to download before any analysis.
|
||||
*/
|
||||
packs: Packs;
|
||||
}
|
||||
|
||||
export type Packs = Record<Partial<Language>, PackWithVersion[]>;
|
||||
|
||||
export interface PackWithVersion {
|
||||
/** qualified name of a package reference */
|
||||
packName: string;
|
||||
/** version of the package, or undefined, which means latest version */
|
||||
version?: semver.SemVer;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -536,6 +556,44 @@ export function getPathsInvalid(configFile: string): string {
|
|||
);
|
||||
}
|
||||
|
||||
export function getPacksRequireLanguage(
|
||||
lang: string,
|
||||
configFile: string
|
||||
): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
PACKS_PROPERTY,
|
||||
`has "${lang}", but it is not one of the languages to analyze`
|
||||
);
|
||||
}
|
||||
|
||||
export function getPacksInvalidSplit(configFile: string): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
PACKS_PROPERTY,
|
||||
"must split packages by language"
|
||||
);
|
||||
}
|
||||
|
||||
export function getPacksInvalid(configFile: string): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
PACKS_PROPERTY,
|
||||
"must be an array of non-empty strings"
|
||||
);
|
||||
}
|
||||
|
||||
export function getPacksStrInvalid(
|
||||
packStr: string,
|
||||
configFile: string
|
||||
): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
PACKS_PROPERTY,
|
||||
`"${packStr}" is not a valid pack`
|
||||
);
|
||||
}
|
||||
|
||||
export function getLocalPathOutsideOfRepository(
|
||||
configFile: string | undefined,
|
||||
localPath: string
|
||||
|
|
@ -787,6 +845,7 @@ export async function getDefaultConfig(
|
|||
queries,
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
packs: {} as Record<Language, PackWithVersion[]>,
|
||||
originalUserInput: {},
|
||||
tempDir,
|
||||
toolCacheDir,
|
||||
|
|
@ -883,7 +942,7 @@ async function loadConfig(
|
|||
shouldAddConfigFileQueries(queriesInput) &&
|
||||
QUERIES_PROPERTY in parsedYAML
|
||||
) {
|
||||
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
|
||||
if (!Array.isArray(parsedYAML[QUERIES_PROPERTY])) {
|
||||
throw new Error(getQueriesInvalid(configFile));
|
||||
}
|
||||
for (const query of parsedYAML[QUERIES_PROPERTY]!) {
|
||||
|
|
@ -908,7 +967,7 @@ async function loadConfig(
|
|||
}
|
||||
|
||||
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
|
||||
if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
|
||||
throw new Error(getPathsIgnoreInvalid(configFile));
|
||||
}
|
||||
for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]!) {
|
||||
|
|
@ -927,7 +986,7 @@ async function loadConfig(
|
|||
}
|
||||
|
||||
if (PATHS_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
|
||||
if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
|
||||
throw new Error(getPathsInvalid(configFile));
|
||||
}
|
||||
for (const includePath of parsedYAML[PATHS_PROPERTY]!) {
|
||||
|
|
@ -940,11 +999,14 @@ async function loadConfig(
|
|||
}
|
||||
}
|
||||
|
||||
const packs = parsePacks(parsedYAML[PACKS_PROPERTY], languages, configFile);
|
||||
|
||||
return {
|
||||
languages,
|
||||
queries,
|
||||
pathsIgnore,
|
||||
paths,
|
||||
packs,
|
||||
originalUserInput: parsedYAML,
|
||||
tempDir,
|
||||
toolCacheDir,
|
||||
|
|
@ -954,6 +1016,78 @@ async function loadConfig(
|
|||
};
|
||||
}
|
||||
|
||||
// Only alpha-numeric characters, with `-` allowed as long as not the first or last char
|
||||
const PACK_IDENTIFIER_PATTERN = (function () {
|
||||
const alphaNumeric = "[a-z0-9]";
|
||||
const alphaNumericDash = "[a-z0-9-]";
|
||||
const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
|
||||
return new RegExp(`^${component}/${component}$`);
|
||||
})();
|
||||
|
||||
// Exported for testing
|
||||
export function parsePacks(
|
||||
packsByLanguage: string[] | Record<string, string[]> | undefined,
|
||||
languages: Language[],
|
||||
configFile: string
|
||||
) {
|
||||
const packs = {} as Packs;
|
||||
|
||||
if (!packsByLanguage) {
|
||||
return packs;
|
||||
}
|
||||
|
||||
if (Array.isArray(packsByLanguage)) {
|
||||
if (languages.length === 1) {
|
||||
// single language analysis, so language is implicit
|
||||
packsByLanguage = {
|
||||
[languages[0]]: packsByLanguage,
|
||||
};
|
||||
} else {
|
||||
// this is an error since multi-language analysis requires
|
||||
// packs split by language
|
||||
throw new Error(getPacksInvalidSplit(configFile));
|
||||
}
|
||||
}
|
||||
|
||||
for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
|
||||
if (!Array.isArray(packsArr)) {
|
||||
throw new Error(getPacksInvalid(configFile));
|
||||
}
|
||||
if (!languages.includes(lang as Language)) {
|
||||
throw new Error(getPacksRequireLanguage(lang, configFile));
|
||||
}
|
||||
packs[lang] = [];
|
||||
for (const packStr of packsArr) {
|
||||
packs[lang].push(toPackWithVersion(packStr, configFile));
|
||||
}
|
||||
}
|
||||
return packs;
|
||||
}
|
||||
|
||||
function toPackWithVersion(packStr, configFile: string): PackWithVersion {
|
||||
if (typeof packStr !== "string") {
|
||||
throw new Error(getPacksStrInvalid(packStr, configFile));
|
||||
}
|
||||
const nameWithVersion = packStr.split("@");
|
||||
let version: semver.SemVer | undefined;
|
||||
if (
|
||||
nameWithVersion.length > 2 ||
|
||||
!PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])
|
||||
) {
|
||||
throw new Error(getPacksStrInvalid(packStr, configFile));
|
||||
} else if (nameWithVersion.length === 2) {
|
||||
version = semver.parse(nameWithVersion[1]) || undefined;
|
||||
if (!version) {
|
||||
throw new Error(getPacksStrInvalid(packStr, configFile));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
packName: nameWithVersion[0],
|
||||
version,
|
||||
};
|
||||
}
|
||||
|
||||
function dbLocationOrDefault(
|
||||
dbLocation: string | undefined,
|
||||
tempDir: string
|
||||
|
|
@ -1019,11 +1153,11 @@ export async function initConfig(
|
|||
// The list of queries should not be empty for any language. If it is then
|
||||
// it is a user configuration error.
|
||||
for (const language of config.languages) {
|
||||
if (
|
||||
config.queries[language] === undefined ||
|
||||
(config.queries[language].builtin.length === 0 &&
|
||||
config.queries[language].custom.length === 0)
|
||||
) {
|
||||
const hasPacks = config.packs[language]?.length > 0;
|
||||
const hasQueries =
|
||||
config.queries[language]?.builtin.length > 0 ||
|
||||
config.queries[language]?.custom.length > 0;
|
||||
if (!hasPacks && !hasQueries) {
|
||||
throw new Error(
|
||||
`Did not detect any queries to run for ${language}. ` +
|
||||
"Please make sure that the default queries are enabled, or you are specifying queries to run."
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
|
|||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {} as configUtils.Packs,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue