robojerk/codeql-action

Merge pull request #1527 from github/aeisenberg/qlconfig-in-cli

Browse source 
Ensure qlconfig file is created when config parsing in cli is on
This commit is contained in:
Andrew Eisenberg 2023-02-27 10:26:08 -08:00 • committed by GitHub
commit a589d4087e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 598 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

42
.github/workflows/__init-with-registries.yml generated vendored
View file

@ -25,6 +25,18 @@ jobs:
strategy:
matrix:
include:
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-latest
version: cached
- os: ubuntu-latest
version: latest
- os: macos-latest
version: latest
- os: windows-latest
version: latest
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
@ -75,5 +87,35 @@ jobs:
echo "::error $CODEQL_PACK1 pack was not installed."
exit 1
fi
- name: Verify qlconfig.yml file was created
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
echo "Expected qlconfig.yml file to be created at $QLCONFIG_PATH"
if [[ -f $QLCONFIG_PATH ]]
then
echo "qlconfig.yml file was created."
else
echo "::error qlconfig.yml file was not created."
exit 1
fi
- name: Verify contents of qlconfig.yml
# yq is not available on windows
if: runner.os != 'Windows'
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
cat $QLCONFIG_PATH | yq -e '.registries[] | select(.url == "https://ghcr.io/v2/") | select(.packages == "*/*")'
if [[ $? -eq 0 ]]
then
echo "Registry was added to qlconfig.yml file."
else
echo "::error Registry was not added to qlconfig.yml file."
echo "Contents of qlconfig.yml file:"
cat $QLCONFIG_PATH
exit 1
fi
env:
CODEQL_ACTION_TEST_MODE: true

27
lib/codeql.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@ -98,6 +98,10 @@ exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
* Versions 2.11.1+ of the CodeQL Bundle include a `security-experimental` built-in query suite for each language.
*/
exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/**
* Versions 2.12.4+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
*/
exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
/**
* Set up CodeQL CLI access.
*
@ -302,7 +306,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
...getExtraOptionsFromEnv(["database", "init"]),
]);
},
async databaseInitCluster(config, sourceRoot, processName, featureEnablement, logger) {
async databaseInitCluster(config, sourceRoot, processName, featureEnablement, qlconfigFile, logger) {
const extraArgs = config.languages.map((language) => `--language=${language}`);
if (config.languages.filter((l) => (0, languages_1.isTracedLanguage)(l)).length > 0) {
extraArgs.push("--begin-tracing");
@ -320,17 +324,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
extraArgs.push("--no-internal-use-lua-tracing");
}
}
// A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
const configLocation = await generateCodeScanningConfig(codeql, config, featureEnablement, logger);
// A code scanning config file is only generated if the CliConfigFileEnabled feature flag is enabled.
const codeScanningConfigFile = await generateCodeScanningConfig(codeql, config, featureEnablement, logger);
// Only pass external repository token if a config file is going to be parsed by the CLI.
let externalRepositoryToken;
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
if (codeScanningConfigFile) {
externalRepositoryToken = (0, actions_util_1.getOptionalInput)("external-repository-token");
extraArgs.push(`--codescanning-config=${codeScanningConfigFile}`);
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
if (await util.codeQlVersionAbove(this, exports.CODEQL_VERSION_INIT_WITH_QLCONFIG)) {
extraArgs.push(`--qlconfig=${qlconfigFile}`);
}
await runTool(cmd, [
"database",
"init",
@ -718,7 +725,7 @@ async function generateCodeScanningConfig(codeql, config, featureEnablement, log
if (!(await util.useCodeScanningConfigInCli(codeql, featureEnablement))) {
return;
}
const configLocation = path.resolve(config.tempDir, "user-config.yaml");
const codeScanningConfigFile = path.resolve(config.tempDir, "user-config.yaml");
// make a copy so we can modify it
const augmentedConfig = cloneObject(config.originalUserInput);
// Inject the queries from the input
@ -772,12 +779,12 @@ async function generateCodeScanningConfig(codeql, config, featureEnablement, log
augmentedConfig.packs["javascript"].push(packString);
}
}
logger.info(`Writing augmented user configuration file to ${configLocation}`);
logger.info(`Writing augmented user configuration file to ${codeScanningConfigFile}`);
logger.startGroup("Augmented user configuration file contents");
logger.info(yaml.dump(augmentedConfig));
logger.endGroup();
fs.writeFileSync(configLocation, yaml.dump(augmentedConfig));
return configLocation;
fs.writeFileSync(codeScanningConfigFile, yaml.dump(augmentedConfig));
return codeScanningConfigFile;
}
function cloneObject(obj) {
return JSON.parse(JSON.stringify(obj));

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

65
lib/codeql.test.js generated
View file

@ -452,11 +452,11 @@ for (const isBundleVersionInUrl of [true, false]) {
packsInputCombines: false,
},
};
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should NOT have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should have injected a codescanning config");
t.falsy(configArg, "Should NOT have injected a codescanning config");
});
});
// Test macro for ensuring different variants of injected augmented configurations
@ -474,7 +474,7 @@ const injectedConfigMacro = ava_1.default.macro({
tempDir,
augmentationProperties,
};
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), (0, logging_1.getRunnerLogger)(true));
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), undefined, (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
@ -665,24 +665,47 @@ const injectedConfigMacro = ava_1.default.macro({
queries: [],
},
}, {});
(0, ava_1.default)("does not use injected config", async (t) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(feature_flags_1.featureConfig[feature_flags_1.Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should NOT have injected a codescanning config");
}
finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
(0, ava_1.default)("does not pass a code scanning config or qlconfig file to the CLI when CLI config passing is disabled", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
// stubbed version doesn't matter. It just needs to be valid semver.
sinon.stub(codeqlObject, "getVersion").resolves("0.0.0");
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should not have used a config file
const hasConfigArg = args.some((arg) => arg.startsWith("--codescanning-config="));
t.false(hasConfigArg, "Should NOT have injected a codescanning config");
// should not have passed a qlconfig file
const hasQlconfigArg = args.some((arg) => arg.startsWith("--qlconfig="));
t.false(hasQlconfigArg, "Should NOT have passed a qlconfig file");
});
(0, ava_1.default)("passes a code scanning config AND qlconfig to the CLI when CLI config passing is enabled", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used a config file
const hasCodeScanningConfigArg = args.some((arg) => arg.startsWith("--codescanning-config="));
t.true(hasCodeScanningConfigArg, "Should have injected a qlconfig");
// should have passed a qlconfig file
const hasQlconfigArg = args.some((arg) => arg.startsWith("--qlconfig="));
t.truthy(hasQlconfigArg, "Should have injected a codescanning config");
});
(0, ava_1.default)("passes a code scanning config BUT NOT a qlconfig to the CLI when CLI config passing is enabled", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.12.2");
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used a config file
const hasCodeScanningConfigArg = args.some((arg) => arg.startsWith("--codescanning-config="));
t.true(hasCodeScanningConfigArg, "Should NOT have injected a qlconfig");
// should have passed a qlconfig file
const hasQlconfigArg = args.some((arg) => arg.startsWith("--qlconfig="));
t.false(hasQlconfigArg, "Should have injected a codescanning config");
});
(0, ava_1.default)("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();

2
lib/codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

70
lib/config-utils.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
exports.wrapEnvironment = exports.generateRegistries = exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
@ -149,7 +149,7 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
}
if (suiteName === "security-experimental" &&
!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE))) {
throw new Error(`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
throw new Error(`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`);
}
@ -725,7 +725,7 @@ function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) {
}
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim();
: rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(getConfigFilePropertyError(undefined, "queries", "A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."));
}
@ -959,8 +959,7 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(codeQL, config.languages, config.packs, registries, apiDetails, config.tempDir, logger);
await downloadPacks(codeQL, config.languages, config.packs, apiDetails, registriesInput, config.tempDir, logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config, logger);
@ -1056,21 +1055,9 @@ async function getConfig(tempDir, logger) {
return JSON.parse(configString);
}
exports.getConfig = getConfig;
async function downloadPacks(codeQL, languages, packs, registries, apiDetails, tmpDir, logger) {
let qlconfigFile;
let registriesAuthTokens;
if (registries) {
if (!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD))) {
throw new Error(`'registries' input is not supported on CodeQL versions less than ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
async function downloadPacks(codeQL, languages, packs, apiDetails, registriesInput, tempDir, logger) {
// This code path is only used when config parsing occurs in the Action.
const { registriesAuthTokens, qlconfigFile } = await generateRegistries(registriesInput, codeQL, tempDir, logger);
await wrapEnvironment({
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
@ -1098,6 +1085,48 @@ async function downloadPacks(codeQL, languages, packs, registries, apiDetails, t
});
}
exports.downloadPacks = downloadPacks;
/**
* Generate a `qlconfig.yml` file from the `registries` input.
* This file is used by the CodeQL CLI to list the registries to use for each
* pack.
*
* @param registriesInput The value of the `registries` input.
* @param codeQL a codeQL object, used only for checking the version of CodeQL.
* @param tempDir a temporary directory to store the generated qlconfig.yml file.
* @param logger a logger object.
* @returns The path to the generated `qlconfig.yml` file and the auth tokens to
* use for each registry.
*/
async function generateRegistries(registriesInput, codeQL, tempDir, logger) {
const registries = parseRegistries(registriesInput);
let registriesAuthTokens;
let qlconfigFile;
if (registries) {
if (!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD))) {
throw new Error(`The 'registries' input is not supported on CodeQL CLI versions earlier than ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD}. Please upgrade to CodeQL CLI version ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD} or later.`);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tempDir, "qlconfig.yml");
const qlconfigContents = yaml.dump(qlconfig);
fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
logger.debug("Generated qlconfig.yml:");
logger.debug(qlconfigContents);
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
if (typeof process.env.CODEQL_REGISTRIES_AUTH === "string") {
logger.debug("Using CODEQL_REGISTRIES_AUTH environment variable to authenticate with registries.");
}
return {
registriesAuthTokens:
// if the user has explicitly set the CODEQL_REGISTRIES_AUTH env var then use that
process.env.CODEQL_REGISTRIES_AUTH ?? registriesAuthTokens,
qlconfigFile,
};
}
exports.generateRegistries = generateRegistries;
function createRegistriesBlock(registries) {
if (!Array.isArray(registries) ||
registries.some((r) => !r.url || !r.packages)) {
@ -1147,4 +1176,5 @@ async function wrapEnvironment(env, operation) {
}
}
}
exports.wrapEnvironment = wrapEnvironment;
//# sourceMappingURL=config-utils.js.map

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

80
lib/config-utils.test.js generated
View file

@ -1114,8 +1114,8 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
}, undefined, // registries
sampleApiDetails, tmpDir, logger);
}, sampleApiDetails, undefined, // registriesAuthTokens
tmpDir, logger);
// Expecting packs to be downloaded once for java and once for python
t.deepEqual(packDownloadStub.callCount, 2);
// no config file was created, so pass `undefined` as the config file path
@ -1128,9 +1128,9 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
process.env.CODEQL_REGISTRIES_AUTH = undefined;
const logger = (0, logging_1.getRunnerLogger)(true);
const registries = [
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
@ -1143,8 +1143,9 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
// append a slash to the first url
const registries = yaml.load(registriesInput);
const expectedRegistries = registries.map((r, i) => ({
packages: r.packages,
url: i === 0 ? `${r.url}/` : r.url,
@ -1173,7 +1174,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
}, registries, sampleApiDetails, tmpDir, logger);
}, sampleApiDetails, registriesInput, tmpDir, logger);
// Same packs are downloaded as in previous test
t.deepEqual(packDownloadStub.callCount, 2);
t.deepEqual(packDownloadStub.firstCall.args, [
@ -1186,7 +1187,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
]);
// Verify that the env vars were unset.
t.deepEqual(process.env.GITHUB_TOKEN, "not-a-token");
t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "not-a-registries-auth");
t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, undefined);
});
});
(0, ava_1.default)("downloadPacks-with-registries fails on 2.10.3", async (t) => {
@ -1196,7 +1197,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = (0, logging_1.getRunnerLogger)(true);
const registries = [
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
@ -1207,12 +1208,12 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = (0, codeql_1.setCodeQL)({
getVersion: () => Promise.resolve("2.10.3"),
});
await t.throwsAsync(async () => {
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, registries, sampleApiDetails, tmpDir, logger);
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, sampleApiDetails, registriesInput, tmpDir, logger);
}, { instanceOf: Error }, "'registries' input is not supported on CodeQL versions less than 2.10.4.");
});
});
@ -1223,7 +1224,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = (0, logging_1.getRunnerLogger)(true);
const registries = [
const registriesInput = yaml.dump([
{
// missing url property
packages: ["codeql/*", "dsp-testing/*"],
@ -1234,15 +1235,68 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = (0, codeql_1.setCodeQL)({
getVersion: () => Promise.resolve("2.10.4"),
});
await t.throwsAsync(async () => {
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, registries, sampleApiDetails, tmpDir, logger);
return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, sampleApiDetails, registriesInput, tmpDir, logger);
}, { instanceOf: Error }, "Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties.");
});
});
// the happy path for generateRegistries is already tested in downloadPacks.
// these following tests are for the error cases and when nothing is generated.
(0, ava_1.default)("no generateRegistries when CLI is too old", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
]);
const codeQL = (0, codeql_1.setCodeQL)({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve("2.10.3"),
});
const logger = (0, logging_1.getRunnerLogger)(true);
await t.throwsAsync(async () => await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger), undefined, "'registries' input is not supported on CodeQL versions less than 2.10.4.");
});
});
(0, ava_1.default)("no generateRegistries when registries is undefined", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = undefined;
const codeQL = (0, codeql_1.setCodeQL)({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = (0, logging_1.getRunnerLogger)(true);
const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger);
t.is(registriesAuthTokens, undefined);
t.is(qlconfigFile, undefined);
});
});
(0, ava_1.default)("generateRegistries prefers original CODEQL_REGISTRIES_AUTH", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env.CODEQL_REGISTRIES_AUTH = "original";
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
]);
const codeQL = (0, codeql_1.setCodeQL)({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = (0, logging_1.getRunnerLogger)(true);
const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger);
t.is(registriesAuthTokens, "original");
t.is(qlconfigFile, path.join(tmpDir, "qlconfig.yml"));
});
});
// getLanguages
const mockRepositoryNwo = (0, repository_1.parseRepositoryNwo)("owner/repo");
// eslint-disable-next-line github/array-foreach

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

5
lib/init-action.js generated
View file

@ -113,6 +113,7 @@ async function run() {
const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
(0, util_1.checkGitHubVersionInRange)(gitHubVersion, logger);
const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
const registriesInput = (0, actions_util_1.getOptionalInput)("registries");
const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, (0, actions_util_1.getTemporaryDirectory)(), logger);
try {
const workflowErrors = await (0, workflow_1.validateWorkflow)();
@ -129,7 +130,7 @@ async function run() {
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
await (0, util_1.enrichEnvironment)(codeql);
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), getTrapCachingEnabled(),
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), registriesInput, (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), getTrapCachingEnabled(),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@ -173,7 +174,7 @@ async function run() {
core.exportVariable("CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN", "true");
}
const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", features, logger);
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", registriesInput, features, apiDetails, logger);
if (tracerConfig !== undefined) {
for (const [key, value] of Object.entries(tracerConfig.env)) {
core.exportVariable(key, value);

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

18
lib/init.js generated
View file

@ -57,12 +57,26 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
return config;
}
exports.initConfig = initConfig;
async function runInit(codeql, config, sourceRoot, processName, featureEnablement, logger) {
async function runInit(codeql, config, sourceRoot, processName, registriesInput, featureEnablement, apiDetails, logger) {
fs.mkdirSync(config.dbLocation, { recursive: true });
try {
if (await (0, util_1.codeQlVersionAbove)(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {
// When parsing the codeql config in the CLI, we have not yet created the qlconfig file.
// So, create it now.
// If we are parsing the config file in the Action, then the qlconfig file was already created
// before the `pack download` command was invoked. It is not required for the init command.
let registriesAuthTokens;
let qlconfigFile;
if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(registriesInput, codeql, config.tempDir, logger));
}
await configUtils.wrapEnvironment({
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
await codeql.databaseInitCluster(config, sourceRoot, processName, featureEnablement, logger);
async () => await codeql.databaseInitCluster(config, sourceRoot, processName, featureEnablement, qlconfigFile, logger));
}
else {
for (const language of config.languages) {

2
lib/init.js.map
View file

File diff suppressed because one or more lines are too long

37
pr-checks/checks/init-with-registries.yml
View file

@ -4,7 +4,12 @@
# basic mechanics of multi-registry auth is working.
name: "Packaging: Download using registries"
description: "Checks that specifying a registries block and associated auth works as expected"
versions: ["nightly-latest"] # This feature is not compatible with old CLIs
versions: [
# This feature is not compatible with older CLIs
"cached",
"latest",
"nightly-latest",
]
steps:
- name: Init with registries
@ -40,3 +45,33 @@ steps:
echo "::error $CODEQL_PACK1 pack was not installed."
exit 1
fi
- name: Verify qlconfig.yml file was created
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
echo "Expected qlconfig.yml file to be created at $QLCONFIG_PATH"
if [[ -f $QLCONFIG_PATH ]]
then
echo "qlconfig.yml file was created."
else
echo "::error qlconfig.yml file was not created."
exit 1
fi
- name: Verify contents of qlconfig.yml
# yq is not available on windows
if: runner.os != 'Windows'
shell: bash
run: |
QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
cat $QLCONFIG_PATH | yq -e '.registries[] | select(.url == "https://ghcr.io/v2/") | select(.packages == "*/*")'
if [[ $? -eq 0 ]]
then
echo "Registry was added to qlconfig.yml file."
else
echo "::error Registry was not added to qlconfig.yml file."
echo "Contents of qlconfig.yml file:"
cat $QLCONFIG_PATH
exit 1
fi

110
src/codeql.test.ts
View file

@ -671,6 +671,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
"",
undefined,
createFeatures([]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
@ -679,7 +680,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should have injected a codescanning config");
t.falsy(configArg, "Should NOT have injected a codescanning config");
});
});
@ -710,6 +711,7 @@ const injectedConfigMacro = test.macro({
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
undefined,
getRunnerLogger(true)
);
@ -1000,34 +1002,90 @@ test(
{}
);
test("does not use injected config", async (t: ExecutionContext<unknown>) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
test("does not pass a code scanning config or qlconfig file to the CLI when CLI config passing is disabled", async (t: ExecutionContext<unknown>) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
// stubbed version doesn't matter. It just needs to be valid semver.
sinon.stub(codeqlObject, "getVersion").resolves("0.0.0");
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([]),
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
// should not have used a config file
const hasConfigArg = args.some((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.false(hasConfigArg, "Should NOT have injected a codescanning config");
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should NOT have injected a codescanning config");
} finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
// should not have passed a qlconfig file
const hasQlconfigArg = args.some((arg: string) =>
arg.startsWith("--qlconfig=")
);
t.false(hasQlconfigArg, "Should NOT have passed a qlconfig file");
});
test("passes a code scanning config AND qlconfig to the CLI when CLI config passing is enabled", async (t: ExecutionContext<unknown>) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
// should have used a config file
const hasCodeScanningConfigArg = args.some((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.true(hasCodeScanningConfigArg, "Should have injected a qlconfig");
// should have passed a qlconfig file
const hasQlconfigArg = args.some((arg: string) =>
arg.startsWith("--qlconfig=")
);
t.truthy(hasQlconfigArg, "Should have injected a codescanning config");
});
test("passes a code scanning config BUT NOT a qlconfig to the CLI when CLI config passing is enabled", async (t: ExecutionContext<unknown>) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.12.2");
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
"/path/to/qlconfig.yml",
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1] as any[];
// should have used a config file
const hasCodeScanningConfigArg = args.some((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.true(hasCodeScanningConfigArg, "Should NOT have injected a qlconfig");
// should have passed a qlconfig file
const hasQlconfigArg = args.some((arg: string) =>
arg.startsWith("--qlconfig=")
);
t.false(hasQlconfigArg, "Should have injected a codescanning config");
});
test("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {

33
src/codeql.ts
View file

@ -91,6 +91,7 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
qlconfigFile: string | undefined,
logger: Logger
): Promise<void>;
/**
@ -289,6 +290,11 @@ export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
*/
export const CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/**
* Versions 2.12.4+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
*/
export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
/**
* Set up CodeQL CLI access.
*
@ -565,6 +571,7 @@ export async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
qlconfigFile: string | undefined,
logger: Logger
) {
const extraArgs = config.languages.map(
@ -594,8 +601,8 @@ export async function getCodeQLForCmd(
}
}
// A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
const configLocation = await generateCodeScanningConfig(
// A code scanning config file is only generated if the CliConfigFileEnabled feature flag is enabled.
const codeScanningConfigFile = await generateCodeScanningConfig(
codeql,
config,
featureEnablement,
@ -603,14 +610,19 @@ export async function getCodeQLForCmd(
);
// Only pass external repository token if a config file is going to be parsed by the CLI.
let externalRepositoryToken: string | undefined;
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
if (codeScanningConfigFile) {
externalRepositoryToken = getOptionalInput("external-repository-token");
extraArgs.push(`--codescanning-config=${codeScanningConfigFile}`);
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
if (
await util.codeQlVersionAbove(this, CODEQL_VERSION_INIT_WITH_QLCONFIG)
) {
extraArgs.push(`--qlconfig=${qlconfigFile}`);
}
await runTool(
cmd,
[
@ -1111,7 +1123,10 @@ async function generateCodeScanningConfig(
if (!(await util.useCodeScanningConfigInCli(codeql, featureEnablement))) {
return;
}
const configLocation = path.resolve(config.tempDir, "user-config.yaml");
const codeScanningConfigFile = path.resolve(
config.tempDir,
"user-config.yaml"
);
// make a copy so we can modify it
const augmentedConfig = cloneObject(config.originalUserInput);
@ -1168,13 +1183,15 @@ async function generateCodeScanningConfig(
augmentedConfig.packs["javascript"].push(packString);
}
}
logger.info(`Writing augmented user configuration file to ${configLocation}`);
logger.info(
`Writing augmented user configuration file to ${codeScanningConfigFile}`
);
logger.startGroup("Augmented user configuration file contents");
logger.info(yaml.dump(augmentedConfig));
logger.endGroup();
fs.writeFileSync(configLocation, yaml.dump(augmentedConfig));
return configLocation;
fs.writeFileSync(codeScanningConfigFile, yaml.dump(augmentedConfig));
return codeScanningConfigFile;
}
function cloneObject<T>(obj: T): T {

114
src/config-utils.test.ts
View file

@ -7,9 +7,13 @@ import * as yaml from "js-yaml";
import * as sinon from "sinon";
import * as api from "./api-client";
import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
import {
CODEQL_VERSION_GHES_PACK_DOWNLOAD,
getCachedCodeQL,
PackDownloadOutput,
setCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { RegistryConfigWithCredentials } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger, Logger } from "./logging";
@ -2277,8 +2281,8 @@ test("downloadPacks-no-registries", async (t) => {
go: ["c", "d"],
python: ["e", "f"],
},
undefined, // registries
sampleApiDetails,
undefined, // registriesAuthTokens
tmpDir,
logger
);
@ -2296,10 +2300,10 @@ test("downloadPacks-with-registries", async (t) => {
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
process.env.CODEQL_REGISTRIES_AUTH = undefined;
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
@ -2312,9 +2316,12 @@ test("downloadPacks-with-registries", async (t) => {
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
// append a slash to the first url
const registries = yaml.load(
registriesInput
) as configUtils.RegistryConfigWithCredentials[];
const expectedRegistries = registries.map((r, i) => ({
packages: r.packages,
url: i === 0 ? `${r.url}/` : r.url,
@ -2356,8 +2363,8 @@ test("downloadPacks-with-registries", async (t) => {
go: ["c", "d"],
python: ["e", "f"],
},
registries,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2375,7 +2382,7 @@ test("downloadPacks-with-registries", async (t) => {
// Verify that the env vars were unset.
t.deepEqual(process.env.GITHUB_TOKEN, "not-a-token");
t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "not-a-registries-auth");
t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, undefined);
});
});
@ -2387,7 +2394,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
@ -2398,7 +2405,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.3"),
@ -2409,8 +2416,8 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2429,7 +2436,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
const registriesInput = yaml.dump([
{
// missing url property
packages: ["codeql/*", "dsp-testing/*"],
@ -2440,7 +2447,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
packages: "semmle/*",
token: "still-not-a-token",
},
];
]);
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.4"),
@ -2451,8 +2458,8 @@ test("downloadPacks-with-registries fails with invalid registries block", async
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries as RegistryConfigWithCredentials[] | undefined,
sampleApiDetails,
registriesInput,
tmpDir,
logger
);
@ -2463,6 +2470,85 @@ test("downloadPacks-with-registries fails with invalid registries block", async
});
});
// the happy path for generateRegistries is already tested in downloadPacks.
// these following tests are for the error cases and when nothing is generated.
test("no generateRegistries when CLI is too old", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = yaml.dump([
{
// no slash
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
]);
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve("2.10.3"),
});
const logger = getRunnerLogger(true);
await t.throwsAsync(
async () =>
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
),
undefined,
"'registries' input is not supported on CodeQL versions less than 2.10.4."
);
});
});
test("no generateRegistries when registries is undefined", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const registriesInput = undefined;
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = getRunnerLogger(true);
const { registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
);
t.is(registriesAuthTokens, undefined);
t.is(qlconfigFile, undefined);
});
});
test("generateRegistries prefers original CODEQL_REGISTRIES_AUTH", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env.CODEQL_REGISTRIES_AUTH = "original";
const registriesInput = yaml.dump([
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
]);
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
getVersion: () => Promise.resolve(CODEQL_VERSION_GHES_PACK_DOWNLOAD),
});
const logger = getRunnerLogger(true);
const { registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeQL,
tmpDir,
logger
);
t.is(registriesAuthTokens, "original");
t.is(qlconfigFile, path.join(tmpDir, "qlconfig.yml"));
});
});
// getLanguages
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");

98
src/config-utils.ts
View file

@ -414,7 +414,7 @@ async function addBuiltinSuiteQueries(
))
) {
throw new Error(
`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`
);
@ -1403,7 +1403,7 @@ function parseQueriesFromInput(
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim();
: rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(
getConfigFilePropertyError(
@ -1769,13 +1769,12 @@ export async function initConfig(
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(
codeQL,
config.languages,
config.packs,
registries,
apiDetails,
registriesInput,
config.tempDir,
logger
);
@ -1899,32 +1898,18 @@ export async function downloadPacks(
codeQL: CodeQL,
languages: Language[],
packs: Packs,
registries: RegistryConfigWithCredentials[] | undefined,
apiDetails: api.GitHubApiDetails,
tmpDir: string,
registriesInput: string | undefined,
tempDir: string,
logger: Logger
) {
let qlconfigFile: string | undefined;
let registriesAuthTokens: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
// This code path is only used when config parsing occurs in the Action.
const { registriesAuthTokens, qlconfigFile } = await generateRegistries(
registriesInput,
codeQL,
tempDir,
logger
);
await wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
@ -1963,6 +1948,63 @@ export async function downloadPacks(
);
}
/**
* Generate a `qlconfig.yml` file from the `registries` input.
* This file is used by the CodeQL CLI to list the registries to use for each
* pack.
*
* @param registriesInput The value of the `registries` input.
* @param codeQL a codeQL object, used only for checking the version of CodeQL.
* @param tempDir a temporary directory to store the generated qlconfig.yml file.
* @param logger a logger object.
* @returns The path to the generated `qlconfig.yml` file and the auth tokens to
* use for each registry.
*/
export async function generateRegistries(
registriesInput: string | undefined,
codeQL: CodeQL,
tempDir: string,
logger: Logger
) {
const registries = parseRegistries(registriesInput);
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`The 'registries' input is not supported on CodeQL CLI versions earlier than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}. Please upgrade to CodeQL CLI version ${CODEQL_VERSION_GHES_PACK_DOWNLOAD} or later.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tempDir, "qlconfig.yml");
const qlconfigContents = yaml.dump(qlconfig);
fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
logger.debug("Generated qlconfig.yml:");
logger.debug(qlconfigContents);
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
if (typeof process.env.CODEQL_REGISTRIES_AUTH === "string") {
logger.debug(
"Using CODEQL_REGISTRIES_AUTH environment variable to authenticate with registries."
);
}
return {
registriesAuthTokens:
// if the user has explicitly set the CODEQL_REGISTRIES_AUTH env var then use that
process.env.CODEQL_REGISTRIES_AUTH ?? registriesAuthTokens,
qlconfigFile,
};
}
function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
registries: RegistryConfigNoCredentials[];
} {
@ -1999,7 +2041,7 @@ function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
* @param env
* @param operation
*/
async function wrapEnvironment(
export async function wrapEnvironment(
env: Record<string, string | undefined>,
operation: Function
) {

6
src/init-action.ts
View file

@ -204,6 +204,8 @@ async function run() {
getRequiredEnvParam("GITHUB_REPOSITORY")
);
const registriesInput = getOptionalInput("registries");
const features = new Features(
gitHubVersion,
repositoryNwo,
@ -251,7 +253,7 @@ async function run() {
getOptionalInput("languages"),
getOptionalInput("queries"),
getOptionalInput("packs"),
getOptionalInput("registries"),
registriesInput,
getOptionalInput("config-file"),
getOptionalInput("db-location"),
getTrapCachingEnabled(),
@ -335,7 +337,9 @@ async function run() {
config,
sourceRoot,
"Runner.Worker.exe",
registriesInput,
features,
apiDetails,
logger
);
if (tracerConfig !== undefined) {

40
src/init.ts
View file

@ -102,20 +102,46 @@ export async function runInit(
config: configUtils.Config,
sourceRoot: string,
processName: string | undefined,
registriesInput: string | undefined,
featureEnablement: FeatureEnablement,
apiDetails: GitHubApiCombinedDetails,
logger: Logger
): Promise<TracerConfig | undefined> {
fs.mkdirSync(config.dbLocation, { recursive: true });
try {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Init a database cluster
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
featureEnablement,
logger
// When parsing the codeql config in the CLI, we have not yet created the qlconfig file.
// So, create it now.
// If we are parsing the config file in the Action, then the qlconfig file was already created
// before the `pack download` command was invoked. It is not required for the init command.
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeql,
config.tempDir,
logger
));
}
await configUtils.wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
async () =>
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
featureEnablement,
qlconfigFile,
logger
)
);
} else {
for (const language of config.languages) {