robojerk/codeql-action

Ensure qlconfig file is created when config parsing in cli is on

Browse source 
Previously, with the config parsing in the cli feature flag turned on,
the CLI was not able to download packs from other registries. This PR
adds the codeql-action changes required for this. The CLI changes will
be in a separate, internal PR.
This commit is contained in:
Andrew Eisenberg 2023-02-07 10:40:49 -08:00
parent 4369dda4ae
commit bbe8d375fd
20 changed files with 480 additions and 138 deletions
Download patch file Download diff file Expand all files Collapse all files

78
src/codeql.test.ts
View file

@ -681,6 +681,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
"",
undefined,
createFeatures([]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
@ -689,7 +690,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should have injected a codescanning config");
t.falsy(configArg, "Should NOT have injected a codescanning config");
});
});
@ -720,6 +721,7 @@ const injectedConfigMacro = test.macro({
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
undefined,
getRunnerLogger(true)
);
@ -1011,33 +1013,59 @@ test(
);
test("does not use injected config", async (t: ExecutionContext<unknown>) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([]),
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
// should not have used a config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should NOT have injected a codescanning config");
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should NOT have injected a codescanning config");
} finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
// should not have passed a qlconfig file
const qlconfigArg = args.find((arg: string) => arg.startsWith("--qlconfig="));
t.falsy(qlconfigArg, "Should NOT have injected a codescanning config");
});
test("uses injected config AND qlconfig", async (t: ExecutionContext<unknown>) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
// should have used a config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.truthy(configArg, "Should have injected a qlconfig");
// should have passed a qlconfig file
const qlconfigArg = args.find((arg: string) => arg.startsWith("--qlconfig="));
t.truthy(qlconfigArg, "Should have injected a codescanning config");
});
test("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {

14
src/codeql.ts
View file

@ -91,6 +91,7 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
qlconfigFile: string | undefined,
logger: Logger
): Promise<void>;
/**
@ -283,6 +284,11 @@ export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
*/
export const CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/**
* Versions 2.12.2+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
*/
export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.3";
/**
* Set up CodeQL CLI access.
*
@ -562,6 +568,7 @@ export async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
qlconfigFile: string | undefined,
logger: Logger
) {
const extraArgs = config.languages.map(
@ -601,13 +608,18 @@ export async function getCodeQLForCmd(
// Only pass external repository token if a config file is going to be parsed by the CLI.
let externalRepositoryToken: string | undefined;
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
externalRepositoryToken = getOptionalInput("external-repository-token");
extraArgs.push(`--codescanning-config=${configLocation}`);
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
if (
await util.codeQlVersionAbove(this, CODEQL_VERSION_INIT_WITH_QLCONFIG)
) {
extraArgs.push(`--qlconfig=${qlconfigFile}`);
}
await runTool(
cmd,
[

93
src/config-utils.test.ts
View file

@ -7,9 +7,13 @@ import * as yaml from "js-yaml";
import * as sinon from "sinon";
import * as api from "./api-client";
import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
import {
CODEQL_VERSION_GHES_PACK_DOWNLOAD,
getCachedCodeQL,
PackDownloadOutput,
setCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { RegistryConfigWithCredentials } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger, Logger } from "./logging";
@ -2277,9 +2281,9 @@ test("downloadPacks-no-registries", async (t) => {
go: ["c", "d"],
python: ["e", "f"],
},
undefined, // registries
sampleApiDetails,
tmpDir,
undefined, // registriesAuthTokens
tmpDir, // qlconfig file path
logger
);
@ -2299,7 +2303,7 @@ test("downloadPacks-with-registries", async (t) => {
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
@ -2312,9 +2316,12 @@ test("downloadPacks-with-registries", async (t) => {
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
// append a slash to the first url
const registries = yaml.load(
registriesInput
) as configUtils.RegistryConfigWithCredentials[];
const expectedRegistries = registries.map((r, i) => ({
packages: r.packages,
url: i === 0 ? `${r.url}/` : r.url,
@ -2356,8 +2363,8 @@ test("downloadPacks-with-registries", async (t) => {
go: ["c", "d"],
python: ["e", "f"],
},
registries,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2387,7 +2394,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
@ -2398,7 +2405,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.3"),
@ -2409,8 +2416,8 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2429,7 +2436,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
// missing url property
packages: ["codeql/*", "dsp-testing/*"],
@ -2440,7 +2447,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.4"),
@ -2451,8 +2458,8 @@ test("downloadPacks-with-registries fails with invalid registries block", async
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries as RegistryConfigWithCredentials[] | undefined,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2463,6 +2470,66 @@ test("downloadPacks-with-registries fails with invalid registries block", async
});
});
// the happy path for generateRegistries is already tested in downloadPacks.
// these following tests are for the error cases and when nothing is generated.
test("no generateRegistries when CLI is too old", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
{
// with slash
url: "https://containers.GHEHOSTNAME1/v2/",
packages: "semmle/*",
token: "still-not-a-token",
},
]);
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve("2.10.3"),
});
const logger = getRunnerLogger(true);
await t.throwsAsync(
async () =>
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
),
undefined,
"'registries' input is not supported on CodeQL versions less than 2.10.4."
);
// t.is(registriesAuthTokens, undefined);
// t.is(qlconfigFile, undefined);
});
});
test("no generateRegistries when registries is undefined", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = undefined;
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = getRunnerLogger(true);
const { registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
);
t.is(registriesAuthTokens, undefined);
t.is(qlconfigFile, undefined);
});
});
// getLanguages
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");

85
src/config-utils.ts
View file

@ -1403,7 +1403,7 @@ function parseQueriesFromInput(
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim();
: rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(
getConfigFilePropertyError(
@ -1769,13 +1769,12 @@ export async function initConfig(
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(
codeQL,
config.languages,
config.packs,
registries,
apiDetails,
registriesInput,
config.tempDir,
logger
);
@ -1899,32 +1898,19 @@ export async function downloadPacks(
codeQL: CodeQL,
languages: Language[],
packs: Packs,
registries: RegistryConfigWithCredentials[] | undefined,
apiDetails: api.GitHubApiDetails,
tmpDir: string,
registriesInput: string | undefined,
tempDir: string,
logger: Logger
) {
let qlconfigFile: string | undefined;
let registriesAuthTokens: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
// When config parsing in the cli is used, the registries will be generated
// immediately before the call to database init.
const { registriesAuthTokens, qlconfigFile } = await generateRegistries(
registriesInput,
codeQL,
tempDir,
logger
);
await wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
@ -1963,6 +1949,51 @@ export async function downloadPacks(
);
}
/**
* Generate a `qlconfig.yml` file from the `registries` input.
* This file is used by the CodeQL CLI to list the registries to use for each
* pack.
*
* @param registriesInput The value of the `registries` input.
* @param codeQL a codeQL object, used only for checking the version of CodeQL.
* @param tmpDir a temporary directory to store the generated qlconfig.yml file.
* @param logger a logger object.
* @returns The path to the generated `qlconfig.yml` file and the auth tokens to
* use for each registry.
*/
export async function generateRegistries(
registriesInput: string | undefined,
codeQL: CodeQL,
tmpDir: string,
logger: Logger
) {
const registries = parseRegistries(registriesInput);
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
const qlconfigContents = yaml.dump(qlconfig);
fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
logger.debug("Generated qlconfig.yml:");
logger.debug(qlconfigContents);
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
return { registriesAuthTokens, qlconfigFile };
}
function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
registries: RegistryConfigNoCredentials[];
} {
@ -1999,7 +2030,7 @@ function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
* @param env
* @param operation
*/
async function wrapEnvironment(
export async function wrapEnvironment(
env: Record<string, string | undefined>,
operation: Function
) {

6
src/init-action.ts
View file

@ -203,6 +203,8 @@ async function run() {
getRequiredEnvParam("GITHUB_REPOSITORY")
);
const registriesInput = getOptionalInput("registries");
const features = new Features(
gitHubVersion,
repositoryNwo,
@ -257,7 +259,7 @@ async function run() {
getOptionalInput("languages"),
getOptionalInput("queries"),
getOptionalInput("packs"),
getOptionalInput("registries"),
registriesInput,
getOptionalInput("config-file"),
getOptionalInput("db-location"),
await getTrapCachingEnabled(features),
@ -341,7 +343,9 @@ async function run() {
config,
sourceRoot,
"Runner.Worker.exe",
registriesInput,
features,
apiDetails,
logger
);
if (tracerConfig !== undefined) {

39
src/init.ts
View file

@ -104,20 +104,45 @@ export async function runInit(
config: configUtils.Config,
sourceRoot: string,
processName: string | undefined,
registriesInput: string | undefined,
featureEnablement: FeatureEnablement,
apiDetails: GitHubApiCombinedDetails,
logger: Logger
): Promise<TracerConfig | undefined> {
fs.mkdirSync(config.dbLocation, { recursive: true });
try {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Init a database cluster
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
featureEnablement,
logger
// Only create the qlconfig file if we haven't already created it.
// If we are not parsing the config file in the cli, then the qlconfig
// file has already been created.
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeql,
config.tempDir,
logger
));
}
await configUtils.wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
async () =>
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
featureEnablement,
qlconfigFile,
logger
)
);
} else {
for (const language of config.languages) {