robojerk/codeql-action

Ensure qlconfig file is created when config parsing in cli is on

Browse source 
Previously, with the config parsing in the cli feature flag turned on,
the CLI was not able to download packs from other registries. This PR
adds the codeql-action changes required for this. The CLI changes will
be in a separate, internal PR.
This commit is contained in:
Andrew Eisenberg 2023-02-07 10:40:49 -08:00
parent 4369dda4ae
commit bbe8d375fd
20 changed files with 480 additions and 138 deletions
Download patch file Download diff file Expand all files Collapse all files

42
.github/workflows/__init-with-registries.yml generated vendored
View file

@ -25,6 +25,18 @@ jobs:
strategy:
matrix:
include:
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-latest
version: cached
- os: ubuntu-latest
version: latest
- os: macos-latest
version: latest
- os: windows-latest
version: latest
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
@ -75,5 +87,35 @@ jobs:
echo "::error $CODEQL_PACK1 pack was not installed."
exit 1
fi
- name: Verify qlconfig.yml file was created
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
echo "Expected qlconfig.yml file to be created at $QLCONFIG_PATH"
if [[ -f $QLCONFIG_PATH ]]
then
echo "qlconfig.yml file was created."
else
echo "::error qlconfig.yml file was not created."
exit 1
fi
- name: Verify contents of qlconfig.yml
# yq is not available on windows
if: runner.os != 'Windows'
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
cat $QLCONFIG_PATH | yq -e '.registries[] | select(.url == "https://ghcr.io/v2/") | select(.packages == "*/*")'
if [[ $? -eq 0 ]]
then
echo "Registry was added to qlconfig.yml file."
else
echo "::error Registry was not added to qlconfig.yml file."
echo "Contents of qlconfig.yml file:"
cat $QLCONFIG_PATH
exit 1
fi
env:
CODEQL_ACTION_TEST_MODE: true

13
lib/codeql.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@ -98,6 +98,10 @@ exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
* Versions 2.11.1+ of the CodeQL Bundle include a `security-experimental` built-in query suite for each language.
*/
exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/**
* Versions 2.12.2+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
*/
exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.3";
/**
* Set up CodeQL CLI access.
*
@ -303,7 +307,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
...getExtraOptionsFromEnv(["database", "init"]),
]);
},
async databaseInitCluster(config, sourceRoot, processName, featureEnablement, logger) {
async databaseInitCluster(config, sourceRoot, processName, featureEnablement, qlconfigFile, logger) {
const extraArgs = config.languages.map((language) => `--language=${language}`);
if (config.languages.filter((l) => (0, languages_1.isTracedLanguage)(l)).length > 0) {
extraArgs.push("--begin-tracing");
@ -326,12 +330,15 @@ async function getCodeQLForCmd(cmd, checkVersion) {
// Only pass external repository token if a config file is going to be parsed by the CLI.
let externalRepositoryToken;
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
externalRepositoryToken = (0, actions_util_1.getOptionalInput)("external-repository-token");
extraArgs.push(`--codescanning-config=${configLocation}`);
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
if (await util.codeQlVersionAbove(this, exports.CODEQL_VERSION_INIT_WITH_QLCONFIG)) {
extraArgs.push(`--qlconfig=${qlconfigFile}`);
}
await runTool(cmd, [
"database",
"init",

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

51
lib/codeql.test.js generated
View file

@ -452,11 +452,11 @@ for (const isBundleVersionInUrl of [true, false]) {
packsInputCombines: false,
},
};
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should NOT have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should have injected a codescanning config");
t.falsy(configArg, "Should NOT have injected a codescanning config");
});
});
// Test macro for ensuring different variants of injected augmented configurations
@ -474,7 +474,7 @@ const injectedConfigMacro = ava_1.default.macro({
tempDir,
augmentationProperties,
};
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), (0, logging_1.getRunnerLogger)(true));
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), undefined, (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
@ -666,23 +666,34 @@ const injectedConfigMacro = ava_1.default.macro({
},
}, {});
(0, ava_1.default)("does not use injected config", async (t) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(feature_flags_1.featureConfig[feature_flags_1.Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should NOT have injected a codescanning config");
}
finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(feature_flags_1.featureConfig[feature_flags_1.Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should not have used a config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should NOT have injected a codescanning config");
// should not have passed a qlconfig file
const qlconfigArg = args.find((arg) => arg.startsWith("--qlconfig="));
t.falsy(qlconfigArg, "Should NOT have injected a codescanning config");
});
(0, ava_1.default)("uses injected config AND qlconfig", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used a config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.truthy(configArg, "Should have injected a qlconfig");
// should have passed a qlconfig file
const qlconfigArg = args.find((arg) => arg.startsWith("--qlconfig="));
t.truthy(qlconfigArg, "Should have injected a codescanning config");
});
(0, ava_1.default)("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();

2
lib/codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

61
lib/config-utils.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
exports.wrapEnvironment = exports.generateRegistries = exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
@ -725,7 +725,7 @@ function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) {
}
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim();
: rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(getConfigFilePropertyError(undefined, "queries", "A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."));
}
@ -959,8 +959,7 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(codeQL, config.languages, config.packs, registries, apiDetails, config.tempDir, logger);
await downloadPacks(codeQL, config.languages, config.packs, apiDetails, registriesInput, config.tempDir, logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config, logger);
@ -1056,21 +1055,10 @@ async function getConfig(tempDir, logger) {
return JSON.parse(configString);
}
exports.getConfig = getConfig;
async function downloadPacks(codeQL, languages, packs, registries, apiDetails, tmpDir, logger) {
let qlconfigFile;
let registriesAuthTokens;
if (registries) {
if (!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD))) {
throw new Error(`'registries' input is not supported on CodeQL versions less than ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
async function downloadPacks(codeQL, languages, packs, apiDetails, registriesInput, tempDir, logger) {
// When config parsing in the cli is used, the registries will be generated
// immediately before the call to database init.
const { registriesAuthTokens, qlconfigFile } = await generateRegistries(registriesInput, codeQL, tempDir, logger);
await wrapEnvironment({
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
@ -1098,6 +1086,40 @@ async function downloadPacks(codeQL, languages, packs, registries, apiDetails, t
});
}
exports.downloadPacks = downloadPacks;
/**
* Generate a `qlconfig.yml` file from the `registries` input.
* This file is used by the CodeQL CLI to list the registries to use for each
* pack.
*
* @param registriesInput The value of the `registries` input.
* @param codeQL a codeQL object, used only for checking the version of CodeQL.
* @param tmpDir a temporary directory to store the generated qlconfig.yml file.
* @param logger a logger object.
* @returns The path to the generated `qlconfig.yml` file and the auth tokens to
* use for each registry.
*/
async function generateRegistries(registriesInput, codeQL, tmpDir, logger) {
const registries = parseRegistries(registriesInput);
let registriesAuthTokens;
let qlconfigFile;
if (registries) {
if (!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD))) {
throw new Error(`'registries' input is not supported on CodeQL versions less than ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
const qlconfigContents = yaml.dump(qlconfig);
fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
logger.debug("Generated qlconfig.yml:");
logger.debug(qlconfigContents);
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
return { registriesAuthTokens, qlconfigFile };
}
exports.generateRegistries = generateRegistries;
function createRegistriesBlock(registries) {
if (!Array.isArray(registries) ||
registries.some((r) => !r.url || !r.packages)) {
@ -1147,4 +1169,5 @@ async function wrapEnvironment(env, operation) {
}
}
}
exports.wrapEnvironment = wrapEnvironment;
//# sourceMappingURL=config-utils.js.map

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

65
lib/config-utils.test.js generated
View file

@ -1114,8 +1114,9 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
}, undefined, // registries
sampleApiDetails, tmpDir, logger);
}, sampleApiDetails, undefined, // registriesAuthTokens
tmpDir, // qlconfig file path
logger);
// Expecting packs to be downloaded once for java and once for python
t.deepEqual(packDownloadStub.callCount, 2);
// no config file was created, so pass `undefined` as the config file path
@ -1130,7 +1131,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = (0, logging_1.getRunnerLogger)(true);
const registries = [
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
@ -1143,8 +1144,9 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
// append a slash to the first url
const registries = yaml.load(registriesInput);
const expectedRegistries = registries.map((r, i) => ({
packages: r.packages,
url: i === 0 ? `${r.url}/` : r.url,
@ -1173,7 +1175,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
}, registries, sampleApiDetails, tmpDir, logger);
}, sampleApiDetails, registriesInput, tmpDir, logger);
// Same packs are downloaded as in previous test
t.deepEqual(packDownloadStub.callCount, 2);
t.deepEqual(packDownloadStub.firstCall.args, [
@ -1196,7 +1198,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = (0, logging_1.getRunnerLogger)(true);
const registries = [
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
@ -1207,12 +1209,12 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = (0, codeql_1.setCodeQL)({
getVersion: () => Promise.resolve("2.10.3"),
});
await t.throwsAsync(async () => {
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, registries, sampleApiDetails, tmpDir, logger);
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, sampleApiDetails, registriesInput, tmpDir, logger);
}, { instanceOf: Error }, "'registries' input is not supported on CodeQL versions less than 2.10.4.");
});
});
@ -1223,7 +1225,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = (0, logging_1.getRunnerLogger)(true);
const registries = [
const registriesInput = yaml.dump([
{
// missing url property
packages: ["codeql/*", "dsp-testing/*"],
@ -1234,15 +1236,56 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = (0, codeql_1.setCodeQL)({
getVersion: () => Promise.resolve("2.10.4"),
});
await t.throwsAsync(async () => {
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, registries, sampleApiDetails, tmpDir, logger);
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, sampleApiDetails, registriesInput, tmpDir, logger);
}, { instanceOf: Error }, "Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties.");
});
});
// the happy path for generateRegistries is already tested in downloadPacks.
// these following tests are for the error cases and when nothing is generated.
(0, ava_1.default)("no generateRegistries when CLI is too old", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
{
// with slash
url: "https://containers.GHEHOSTNAME1/v2/",
packages: "semmle/*",
token: "still-not-a-token",
},
]);
const codeQL = (0, codeql_1.setCodeQL)({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve("2.10.3"),
});
const logger = (0, logging_1.getRunnerLogger)(true);
await t.throwsAsync(async () => await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger), undefined, "'registries' input is not supported on CodeQL versions less than 2.10.4.");
// t.is(registriesAuthTokens, undefined);
// t.is(qlconfigFile, undefined);
});
});
(0, ava_1.default)("no generateRegistries when registries is undefined", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = undefined;
const codeQL = (0, codeql_1.setCodeQL)({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = (0, logging_1.getRunnerLogger)(true);
const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger);
t.is(registriesAuthTokens, undefined);
t.is(qlconfigFile, undefined);
});
});
// getLanguages
const mockRepositoryNwo = (0, repository_1.parseRepositoryNwo)("owner/repo");
// eslint-disable-next-line github/array-foreach

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

5
lib/init-action.js generated
View file

@ -112,6 +112,7 @@ async function run() {
const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
(0, util_1.checkGitHubVersionInRange)(gitHubVersion, logger);
const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
const registriesInput = (0, actions_util_1.getOptionalInput)("registries");
const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, (0, actions_util_1.getTemporaryDirectory)(), logger);
try {
const workflowErrors = await (0, workflow_1.validateWorkflow)();
@ -128,7 +129,7 @@ async function run() {
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
await (0, util_1.enrichEnvironment)(codeql);
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), await getTrapCachingEnabled(features),
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), registriesInput, (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), await getTrapCachingEnabled(features),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@ -172,7 +173,7 @@ async function run() {
core.exportVariable("CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN", "true");
}
const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", features, logger);
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", registriesInput, features, apiDetails, logger);
if (tracerConfig !== undefined) {
for (const [key, value] of Object.entries(tracerConfig.env)) {
core.exportVariable(key, value);

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

17
lib/init.js generated
View file

@ -57,12 +57,25 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
return config;
}
exports.initConfig = initConfig;
async function runInit(codeql, config, sourceRoot, processName, featureEnablement, logger) {
async function runInit(codeql, config, sourceRoot, processName, registriesInput, featureEnablement, apiDetails, logger) {
fs.mkdirSync(config.dbLocation, { recursive: true });
try {
if (await (0, util_1.codeQlVersionAbove)(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {
// Only create the qlconfig file if we haven't already created it.
// If we are not parsing the config file in the cli, then the qlconfig
// file has already been created.
let registriesAuthTokens;
let qlconfigFile;
if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(registriesInput, codeql, config.tempDir, logger));
}
await configUtils.wrapEnvironment({
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
await codeql.databaseInitCluster(config, sourceRoot, processName, featureEnablement, logger);
async () => await codeql.databaseInitCluster(config, sourceRoot, processName, featureEnablement, qlconfigFile, logger));
}
else {
for (const language of config.languages) {

2
lib/init.js.map
View file

File diff suppressed because one or more lines are too long

37
pr-checks/checks/init-with-registries.yml
View file

@ -4,7 +4,12 @@
# basic mechanics of multi-registry auth is working.
name: "Packaging: Download using registries"
description: "Checks that specifying a registries block and associated auth works as expected"
versions: ["nightly-latest"] # This feature is not compatible with old CLIs
versions: [
# This feature is not compatible with older CLIs
"cached",
"latest",
"nightly-latest",
]
steps:
- name: Init with registries
@ -40,3 +45,33 @@ steps:
echo "::error $CODEQL_PACK1 pack was not installed."
exit 1
fi
- name: Verify qlconfig.yml file was created
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
echo "Expected qlconfig.yml file to be created at $QLCONFIG_PATH"
if [[ -f $QLCONFIG_PATH ]]
then
echo "qlconfig.yml file was created."
else
echo "::error qlconfig.yml file was not created."
exit 1
fi
- name: Verify contents of qlconfig.yml
# yq is not available on windows
if: runner.os != 'Windows'
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
cat $QLCONFIG_PATH | yq -e '.registries[] | select(.url == "https://ghcr.io/v2/") | select(.packages == "*/*")'
if [[ $? -eq 0 ]]
then
echo "Registry was added to qlconfig.yml file."
else
echo "::error Registry was not added to qlconfig.yml file."
echo "Contents of qlconfig.yml file:"
cat $QLCONFIG_PATH
exit 1
fi

78
src/codeql.test.ts
View file

@ -681,6 +681,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
"",
undefined,
createFeatures([]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
@ -689,7 +690,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should have injected a codescanning config");
t.falsy(configArg, "Should NOT have injected a codescanning config");
});
});
@ -720,6 +721,7 @@ const injectedConfigMacro = test.macro({
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
undefined,
getRunnerLogger(true)
);
@ -1011,33 +1013,59 @@ test(
);
test("does not use injected config", async (t: ExecutionContext<unknown>) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([]),
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
// should not have used a config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should NOT have injected a codescanning config");
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should NOT have injected a codescanning config");
} finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
// should not have passed a qlconfig file
const qlconfigArg = args.find((arg: string) => arg.startsWith("--qlconfig="));
t.falsy(qlconfigArg, "Should NOT have injected a codescanning config");
});
test("uses injected config AND qlconfig", async (t: ExecutionContext<unknown>) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
// should have used a config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.truthy(configArg, "Should have injected a qlconfig");
// should have passed a qlconfig file
const qlconfigArg = args.find((arg: string) => arg.startsWith("--qlconfig="));
t.truthy(qlconfigArg, "Should have injected a codescanning config");
});
test("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {

14
src/codeql.ts
View file

@ -91,6 +91,7 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
qlconfigFile: string | undefined,
logger: Logger
): Promise<void>;
/**
@ -283,6 +284,11 @@ export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
*/
export const CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/**
* Versions 2.12.2+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
*/
export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.3";
/**
* Set up CodeQL CLI access.
*
@ -562,6 +568,7 @@ export async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
qlconfigFile: string | undefined,
logger: Logger
) {
const extraArgs = config.languages.map(
@ -601,13 +608,18 @@ export async function getCodeQLForCmd(
// Only pass external repository token if a config file is going to be parsed by the CLI.
let externalRepositoryToken: string | undefined;
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
externalRepositoryToken = getOptionalInput("external-repository-token");
extraArgs.push(`--codescanning-config=${configLocation}`);
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
if (
await util.codeQlVersionAbove(this, CODEQL_VERSION_INIT_WITH_QLCONFIG)
) {
extraArgs.push(`--qlconfig=${qlconfigFile}`);
}
await runTool(
cmd,
[

93
src/config-utils.test.ts
View file

@ -7,9 +7,13 @@ import * as yaml from "js-yaml";
import * as sinon from "sinon";
import * as api from "./api-client";
import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
import {
CODEQL_VERSION_GHES_PACK_DOWNLOAD,
getCachedCodeQL,
PackDownloadOutput,
setCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { RegistryConfigWithCredentials } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger, Logger } from "./logging";
@ -2277,9 +2281,9 @@ test("downloadPacks-no-registries", async (t) => {
go: ["c", "d"],
python: ["e", "f"],
},
undefined, // registries
sampleApiDetails,
tmpDir,
undefined, // registriesAuthTokens
tmpDir, // qlconfig file path
logger
);
@ -2299,7 +2303,7 @@ test("downloadPacks-with-registries", async (t) => {
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
@ -2312,9 +2316,12 @@ test("downloadPacks-with-registries", async (t) => {
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
// append a slash to the first url
const registries = yaml.load(
registriesInput
) as configUtils.RegistryConfigWithCredentials[];
const expectedRegistries = registries.map((r, i) => ({
packages: r.packages,
url: i === 0 ? `${r.url}/` : r.url,
@ -2356,8 +2363,8 @@ test("downloadPacks-with-registries", async (t) => {
go: ["c", "d"],
python: ["e", "f"],
},
registries,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2387,7 +2394,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
@ -2398,7 +2405,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.3"),
@ -2409,8 +2416,8 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2429,7 +2436,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
// missing url property
packages: ["codeql/*", "dsp-testing/*"],
@ -2440,7 +2447,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.4"),
@ -2451,8 +2458,8 @@ test("downloadPacks-with-registries fails with invalid registries block", async
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries as RegistryConfigWithCredentials[] | undefined,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2463,6 +2470,66 @@ test("downloadPacks-with-registries fails with invalid registries block", async
});
});
// the happy path for generateRegistries is already tested in downloadPacks.
// these following tests are for the error cases and when nothing is generated.
test("no generateRegistries when CLI is too old", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
{
// with slash
url: "https://containers.GHEHOSTNAME1/v2/",
packages: "semmle/*",
token: "still-not-a-token",
},
]);
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve("2.10.3"),
});
const logger = getRunnerLogger(true);
await t.throwsAsync(
async () =>
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
),
undefined,
"'registries' input is not supported on CodeQL versions less than 2.10.4."
);
// t.is(registriesAuthTokens, undefined);
// t.is(qlconfigFile, undefined);
});
});
test("no generateRegistries when registries is undefined", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = undefined;
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = getRunnerLogger(true);
const { registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
);
t.is(registriesAuthTokens, undefined);
t.is(qlconfigFile, undefined);
});
});
// getLanguages
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");

85
src/config-utils.ts
View file

@ -1403,7 +1403,7 @@ function parseQueriesFromInput(
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim();
: rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(
getConfigFilePropertyError(
@ -1769,13 +1769,12 @@ export async function initConfig(
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(
codeQL,
config.languages,
config.packs,
registries,
apiDetails,
registriesInput,
config.tempDir,
logger
);
@ -1899,32 +1898,19 @@ export async function downloadPacks(
codeQL: CodeQL,
languages: Language[],
packs: Packs,
registries: RegistryConfigWithCredentials[] | undefined,
apiDetails: api.GitHubApiDetails,
tmpDir: string,
registriesInput: string | undefined,
tempDir: string,
logger: Logger
) {
let qlconfigFile: string | undefined;
let registriesAuthTokens: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
// When config parsing in the cli is used, the registries will be generated
// immediately before the call to database init.
const { registriesAuthTokens, qlconfigFile } = await generateRegistries(
registriesInput,
codeQL,
tempDir,
logger
);
await wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
@ -1963,6 +1949,51 @@ export async function downloadPacks(
);
}
/**
* Generate a `qlconfig.yml` file from the `registries` input.
* This file is used by the CodeQL CLI to list the registries to use for each
* pack.
*
* @param registriesInput The value of the `registries` input.
* @param codeQL a codeQL object, used only for checking the version of CodeQL.
* @param tmpDir a temporary directory to store the generated qlconfig.yml file.
* @param logger a logger object.
* @returns The path to the generated `qlconfig.yml` file and the auth tokens to
* use for each registry.
*/
export async function generateRegistries(
registriesInput: string | undefined,
codeQL: CodeQL,
tmpDir: string,
logger: Logger
) {
const registries = parseRegistries(registriesInput);
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
const qlconfigContents = yaml.dump(qlconfig);
fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
logger.debug("Generated qlconfig.yml:");
logger.debug(qlconfigContents);
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
return { registriesAuthTokens, qlconfigFile };
}
function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
registries: RegistryConfigNoCredentials[];
} {
@ -1999,7 +2030,7 @@ function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
* @param env
* @param operation
*/
async function wrapEnvironment(
export async function wrapEnvironment(
env: Record<string, string | undefined>,
operation: Function
) {

6
src/init-action.ts
View file

@ -203,6 +203,8 @@ async function run() {
getRequiredEnvParam("GITHUB_REPOSITORY")
);
const registriesInput = getOptionalInput("registries");
const features = new Features(
gitHubVersion,
repositoryNwo,
@ -257,7 +259,7 @@ async function run() {
getOptionalInput("languages"),
getOptionalInput("queries"),
getOptionalInput("packs"),
getOptionalInput("registries"),
registriesInput,
getOptionalInput("config-file"),
getOptionalInput("db-location"),
await getTrapCachingEnabled(features),
@ -341,7 +343,9 @@ async function run() {
config,
sourceRoot,
"Runner.Worker.exe",
registriesInput,
features,
apiDetails,
logger
);
if (tracerConfig !== undefined) {

39
src/init.ts
View file

@ -104,20 +104,45 @@ export async function runInit(
config: configUtils.Config,
sourceRoot: string,
processName: string | undefined,
registriesInput: string | undefined,
featureEnablement: FeatureEnablement,
apiDetails: GitHubApiCombinedDetails,
logger: Logger
): Promise<TracerConfig | undefined> {
fs.mkdirSync(config.dbLocation, { recursive: true });
try {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Init a database cluster
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
featureEnablement,
logger
// Only create the qlconfig file if we haven't already created it.
// If we are not parsing the config file in the cli, then the qlconfig
// file has already been created.
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeql,
config.tempDir,
logger
));
}
await configUtils.wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
async () =>
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
featureEnablement,
qlconfigFile,
logger
)
);
} else {
for (const language of config.languages) {