robojerk/codeql-action

Merge pull request #1218 from github/aeisenberg/move-pack-download-to-init

Browse source 
Move calls to pack download to the init action
This commit is contained in:
Andrew Eisenberg 2022-08-30 09:58:46 -07:00 • committed by GitHub
commit d92a91c5c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 220 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -2,7 +2,7 @@
## [UNRELEASED]
No user facing changes.
- Downloading CodeQL packs has been moved to the `init` step. Previously, CodeQL packs were downloaded during the `analyze` step. [#1218](https://github.com/github/codeql-action/pull/1218)
## 2.1.21 - 25 Aug 2022

9
lib/analyze.js generated
View file

@ -170,15 +170,6 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
logger.info(analysisSummary);
}
else {
if (hasPackWithCustomQueries) {
logger.info("Performing analysis with custom CodeQL Packs.");
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths = [];
if (queries["builtin"].length > 0) {

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

31
lib/config-utils.js generated
View file

@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
// We need to import `performance` on Node 12
@ -905,6 +905,12 @@ async function initConfig(languagesInput, queriesInput, packsInput, configFile,
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
}
}
// When using the codescanning config in the CLI, pack downloads
// happen in the CLI during the `database init` command, so no need
// to download them here.
if (!(await (0, util_1.useCodeScanningConfigInCli)(codeQL))) {
await downloadPacks(codeQL, config.languages, config.packs, logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config, logger);
return config;
@ -989,4 +995,27 @@ async function getConfig(tempDir, logger) {
return JSON.parse(configString);
}
exports.getConfig = getConfig;
async function downloadPacks(codeQL, languages, packs, logger) {
let numPacksDownloaded = 0;
logger.startGroup("Downloading packs");
for (const language of languages) {
const packsWithVersion = packs[language];
if (packsWithVersion === null || packsWithVersion === void 0 ? void 0 : packsWithVersion.length) {
logger.info(`Downloading custom packs for ${language}`);
const results = await codeQL.packDownload(packsWithVersion);
numPacksDownloaded += results.packs.length;
logger.info(`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`);
}
}
if (numPacksDownloaded > 0) {
logger.info(`Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`);
}
else {
logger.info("No packs to download");
}
logger.endGroup();
}
exports.downloadPacks = downloadPacks;
//# sourceMappingURL=config-utils.js.map

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

68
lib/config-utils.test.js generated
View file

@ -88,6 +88,9 @@ function mockListLanguages(languages) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const config = await configUtils.initConfig(languages, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
t.deepEqual(config, await configUtils.getDefaultConfig(languages, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger));
@ -107,6 +110,9 @@ function mockListLanguages(languages) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
// Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
@ -178,6 +184,9 @@ function mockListLanguages(languages) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
// Just create a generic config object with non-default values for all fields
const inputFileContents = `
@ -254,6 +263,9 @@ function mockListLanguages(languages) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
// The important point of this config is that it doesn't specify
// the disable-default-queries field.
@ -305,6 +317,9 @@ function queriesToResolvedQueryForm(queries) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload() {
return { packs: [] };
},
});
const languages = "javascript";
const config = await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -338,6 +353,9 @@ function queriesToResolvedQueryForm(queries) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload() {
return { packs: [] };
},
});
const languages = "javascript";
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -370,6 +388,9 @@ function queriesToResolvedQueryForm(queries) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload() {
return { packs: [] };
},
});
const languages = "javascript";
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -396,6 +417,9 @@ function queriesToResolvedQueryForm(queries) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload() {
return { packs: [] };
},
});
const languages = "javascript";
const config = await configUtils.initConfig(languages, testQueries, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -435,6 +459,9 @@ function queriesToResolvedQueryForm(queries) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload() {
return { packs: [] };
},
});
const languages = "javascript";
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -474,6 +501,9 @@ function queriesToResolvedQueryForm(queries) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
try {
await configUtils.initConfig(languages, queries, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -498,6 +528,9 @@ function queriesToResolvedQueryForm(queries) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const inputFileContents = `
name: my config
@ -560,6 +593,9 @@ function queriesToResolvedQueryForm(queries) {
async resolveLanguages() {
return {};
},
async packDownload() {
return { packs: [] };
},
});
try {
await configUtils.initConfig(undefined, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@ -592,6 +628,9 @@ function queriesToResolvedQueryForm(queries) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const inputFileContents = `
name: my config
@ -620,6 +659,9 @@ function queriesToResolvedQueryForm(queries) {
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const inputFileContents = `
name: my config
@ -673,6 +715,9 @@ function doInvalidInputTest(testName, inputFileContents, expectedErrorMessageGen
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const languages = "javascript";
const configFile = "input";
@ -936,6 +981,9 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const { packs } = await configUtils.initConfig("javascript", queriesInput, packsInput, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)(isMlPoweredQueriesFlagEnabled
? [feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled]
@ -1039,4 +1087,24 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
(0, ava_1.default)(calculateAugmentationErrorMacro, "Packs input with multiple languages", " + a/b, c/d ", undefined, [languages_1.Language.javascript, languages_1.Language.java], /Cannot specify a 'packs' input in a multi-language analysis/);
(0, ava_1.default)(calculateAugmentationErrorMacro, "Packs input with no languages", " + a/b, c/d ", undefined, [], /No languages specified/);
(0, ava_1.default)(calculateAugmentationErrorMacro, "Invalid packs", " a-pack-without-a-scope ", undefined, [languages_1.Language.javascript], /"a-pack-without-a-scope" is not a valid pack/);
(0, ava_1.default)("downloadPacks", async (t) => {
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs) => ({
packs,
}));
const codeQL = (0, codeql_1.setCodeQL)({
packDownload: packDownloadStub,
});
const logger = (0, logging_1.getRunnerLogger)(true);
// packs are supplied for go, java, and python
// analyzed languages are java, javascript, and python
await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
}, logger);
t.deepEqual(packDownloadStub.callCount, 2);
t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"]]);
t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"]]);
});
//# sourceMappingURL=config-utils.test.js.map

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

15
src/analyze.ts
View file

@ -280,21 +280,6 @@ export async function runQueries(
logger.endGroup();
logger.info(analysisSummary);
} else {
if (hasPackWithCustomQueries) {
logger.info("Performing analysis with custom CodeQL Packs.");
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {

78
src/config-utils.test.ts
View file

@ -6,7 +6,7 @@ import test, { ExecutionContext } from "ava";
import * as sinon from "sinon";
import * as api from "./api-client";
import { getCachedCodeQL, setCodeQL } from "./codeql";
import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { createFeatureFlags, FeatureFlag } from "./feature-flags";
import { Language } from "./languages";
@ -78,6 +78,9 @@ test("load empty config", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const config = await configUtils.initConfig(
@ -139,6 +142,9 @@ test("loading config saves config", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
// Sanity check the saved config file does not already exist
@ -311,6 +317,9 @@ test("load non-empty input", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
// Just create a generic config object with non-default values for all fields
@ -419,6 +428,9 @@ test("Default queries are used", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
// The important point of this config is that it doesn't specify
@ -504,6 +516,9 @@ test("Queries can be specified in config file", async (t) => {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const languages = "javascript";
@ -578,6 +593,9 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const languages = "javascript";
@ -650,6 +668,9 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const languages = "javascript";
@ -713,6 +734,9 @@ test("Multiple queries can be specified in workflow file, no config file require
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const languages = "javascript";
@ -797,6 +821,9 @@ test("Queries in workflow file can be added to the set of queries without overri
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const languages = "javascript";
@ -876,6 +903,9 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
try {
@ -922,6 +952,9 @@ test("API client used when reading remote config", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const inputFileContents = `
@ -1051,6 +1084,9 @@ test("No detected languages", async (t) => {
async resolveLanguages() {
return {};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
try {
@ -1124,6 +1160,9 @@ test("Config specifies packages", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const inputFileContents = `
@ -1175,6 +1214,9 @@ test("Config specifies packages for multiple languages", async (t) => {
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const inputFileContents = `
@ -1255,6 +1297,9 @@ function doInvalidInputTest(
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const languages = "javascript";
@ -1845,6 +1890,9 @@ const mlPoweredQueriesMacro = test.macro({
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const { packs } = await configUtils.initConfig(
@ -2159,3 +2207,31 @@ test(
[Language.javascript],
/"a-pack-without-a-scope" is not a valid pack/
);
test("downloadPacks", async (t) => {
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs) => ({
packs,
}));
const codeQL = setCodeQL({
packDownload: packDownloadStub,
});
const logger = getRunnerLogger(true);
// packs are supplied for go, java, and python
// analyzed languages are java, javascript, and python
await configUtils.downloadPacks(
codeQL,
[Language.javascript, Language.java, Language.python],
{
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
},
logger
);
t.deepEqual(packDownloadStub.callCount, 2);
t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"]]);
t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"]]);
});

41
src/config-utils.ts
View file

@ -24,6 +24,7 @@ import {
getMlPoweredJsQueriesPack,
GitHubVersion,
ML_POWERED_JS_QUERIES_PACK_NAME,
useCodeScanningConfigInCli,
} from "./util";
// Property names from the user-supplied config file.
@ -953,6 +954,7 @@ async function addQueriesAndPacksFromWorkflow(
);
injectedMlQueries = injectedMlQueries || didInject;
}
return injectedMlQueries;
}
@ -1680,6 +1682,13 @@ export async function initConfig(
}
}
// When using the codescanning config in the CLI, pack downloads
// happen in the CLI during the `database init` command, so no need
// to download them here.
if (!(await useCodeScanningConfigInCli(codeQL))) {
await downloadPacks(codeQL, config.languages, config.packs, logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config, logger);
return config;
@ -1781,3 +1790,35 @@ export async function getConfig(
logger.debug(configString);
return JSON.parse(configString);
}
export async function downloadPacks(
codeQL: CodeQL,
languages: Language[],
packs: Packs,
logger: Logger
) {
let numPacksDownloaded = 0;
logger.startGroup("Downloading packs");
for (const language of languages) {
const packsWithVersion = packs[language];
if (packsWithVersion?.length) {
logger.info(`Downloading custom packs for ${language}`);
const results = await codeQL.packDownload(packsWithVersion);
numPacksDownloaded += results.packs.length;
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
);
}
}
if (numPacksDownloaded > 0) {
logger.info(
`Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`
);
} else {
logger.info("No packs to download");
}
logger.endGroup();
}