robojerk/codeql-action

Send the external repository token to the CLI

Browse source 
This commit does a few related things:

1. Bumps the minimum version for cli config parsing to 2.10.6
2. Ensures that if cli config parsing is enabled, then remove repos
   are _not_ downloaded by the action. It happens in the CLI.
3. Passes the `--external-repository-token-stdin` option to the CLI
   and passes the appropriate token via stdin if cli config parsing is
   enabled.
This commit is contained in:
Andrew Eisenberg 2023-01-05 17:34:59 -08:00
parent cf1437a514
commit 4023575d64
14 changed files with 102 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

1
lib/analyze.js generated
View file

@ -159,6 +159,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
logger.info(analysisSummary);
}
else {
// config was generated by the action, so must be interpreted by the action.
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths = [];
if (queries["builtin"].length > 0) {

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

15
lib/codeql.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CODEQL_VERSION_CONFIG_FILES = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = exports.CommandInvocationError = void 0;
exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = exports.CommandInvocationError = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@ -72,7 +72,6 @@ const CODEQL_MINIMUM_VERSION = "2.6.3";
*/
const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
exports.CODEQL_VERSION_CONFIG_FILES = "2.10.1";
const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
const CODEQL_VERSION_FILE_BASELINE_INFORMATION = "2.11.3";
@ -537,9 +536,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
extraArgs.push("--no-internal-use-lua-tracing");
}
}
// A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
// Only pass external repository token if a config file is
let externalRepositoryToken;
const configLocation = await generateCodeScanningConfig(codeql, config, featureEnablement);
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
externalRepositoryToken = (0, actions_util_1.getOptionalInput)("external-repository-token");
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
await runTool(cmd, [
"database",
@ -549,7 +555,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
`--source-root=${sourceRoot}`,
...extraArgs,
...getExtraOptionsFromEnv(["database", "init"]),
]);
], externalRepositoryToken);
},
async runAutobuild(language) {
const cmdName = process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
@ -884,7 +890,7 @@ exports.getExtraOptions = getExtraOptions;
* status reports on GitHub.com.
*/
const maxErrorSize = 20000;
async function runTool(cmd, args = []) {
async function runTool(cmd, args = [], stdin) {
let output = "";
let error = "";
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
@ -903,6 +909,7 @@ async function runTool(cmd, args = []) {
},
},
ignoreReturnCode: true,
input: Buffer.from(stdin || ""),
}).exec();
if (exitCode !== 0)
throw new CommandInvocationError(cmd, args, exitCode, error, output);

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

12
lib/codeql.test.js generated
View file

@ -354,7 +354,7 @@ const injectedConfigMacro = ava_1.default.macro({
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
.resolves(feature_flags_1.featureConfig[feature_flags_1.Feature.CliConfigFileEnabled].minimumVersion);
const thisStubConfig = {
...stubConfig,
...configOverride,
@ -384,7 +384,7 @@ const injectedConfigMacro = ava_1.default.macro({
queriesInputCombines: false,
packsInputCombines: false,
}, {}, {
packs: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
packs: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
});
(0, ava_1.default)("injected ML queries with existing packs", injectedConfigMacro, {
injectedMlQueries: true,
@ -398,7 +398,7 @@ const injectedConfigMacro = ava_1.default.macro({
packs: {
javascript: [
"codeql/something-else",
"codeql/javascript-experimental-atm-queries@~0.3.0",
"codeql/javascript-experimental-atm-queries@~0.4.0",
],
},
});
@ -413,7 +413,7 @@ const injectedConfigMacro = ava_1.default.macro({
}, {
packs: {
cpp: ["codeql/something-else"],
javascript: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
javascript: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
},
});
(0, ava_1.default)("injected packs from input", injectedConfigMacro, {
@ -466,7 +466,7 @@ const injectedConfigMacro = ava_1.default.macro({
},
},
}, {
packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.3.0"],
packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.4.0"],
});
// similar, but with queries
(0, ava_1.default)("injected queries from input", injectedConfigMacro, {
@ -560,7 +560,7 @@ const injectedConfigMacro = ava_1.default.macro({
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
.resolves(feature_flags_1.featureConfig[feature_flags_1.Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file

2
lib/codeql.test.js.map
View file

File diff suppressed because one or more lines are too long

31
lib/config-utils.js generated
View file

@ -240,8 +240,12 @@ async function parseQueryUses(languages, codeQL, resultMap, packs, queryUses, te
if (queryUses.indexOf("/") === -1 && queryUses.indexOf("@") === -1) {
return await addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, queryUses, featureEnablement, configFile);
}
// Otherwise, must be a reference to another repo
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, apiDetails, logger, configFile);
// Otherwise, must be a reference to another repo.
// If config parsing is handled in CLI, then this repo will be downloaded
// later by the CLI.
if (!(await (0, util_1.useCodeScanningConfigInCli)(codeQL, featureEnablement))) {
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, apiDetails, logger, configFile);
}
return false;
}
// Regex validating stars in paths or paths-ignore entries.
@ -925,22 +929,23 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
else {
config = await loadConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureEnablement, logger);
}
// The list of queries should not be empty for any language. If it is then
// it is a user configuration error.
for (const language of config.languages) {
const hasBuiltinQueries = ((_a = config.queries[language]) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
const hasCustomQueries = ((_b = config.queries[language]) === null || _b === void 0 ? void 0 : _b.custom.length) > 0;
const hasPacks = (((_c = config.packs[language]) === null || _c === void 0 ? void 0 : _c.length) || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
}
}
// When using the codescanning config in the CLI, pack downloads
// happen in the CLI during the `database init` command, so no need
// to download them here.
await (0, util_1.logCodeScanningConfigInCli)(codeQL, featureEnablement, logger);
if (!(await (0, util_1.useCodeScanningConfigInCli)(codeQL, featureEnablement))) {
// The list of queries should not be empty for any language. If it is then
// it is a user configuration error.
// This check occurs in the CLI when it parses the config file.
for (const language of config.languages) {
const hasBuiltinQueries = ((_a = config.queries[language]) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
const hasCustomQueries = ((_b = config.queries[language]) === null || _b === void 0 ? void 0 : _b.custom.length) > 0;
const hasPacks = (((_c = config.packs[language]) === null || _c === void 0 ? void 0 : _c.length) || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(codeQL, config.languages, config.packs, registries, apiDetails, config.tempDir, logger);
}

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/feature-flags.js generated
View file

@ -53,7 +53,7 @@ exports.featureConfig = {
},
[Feature.CliConfigFileEnabled]: {
envVar: "CODEQL_PASS_CONFIG_TO_CLI",
minimumVersion: "2.11.1",
minimumVersion: "2.11.6",
},
[Feature.MlPoweredQueriesEnabled]: {
envVar: "CODEQL_ML_POWERED_QUERIES",

1
src/analyze.ts
View file

@ -262,6 +262,7 @@ export async function runQueries(
logger.endGroup();
logger.info(analysisSummary);
} else {
// config was generated by the action, so must be interpreted by the action.
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {

14
src/codeql.test.ts
View file

@ -15,7 +15,7 @@ import { GitHubApiDetails } from "./api-client";
import * as codeql from "./codeql";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
import { Feature } from "./feature-flags";
import { Feature, featureConfig } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
@ -513,7 +513,7 @@ const injectedConfigMacro = test.macro({
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
const thisStubConfig: Config = {
...stubConfig,
@ -570,7 +570,7 @@ test(
},
{},
{
packs: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
packs: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
}
);
@ -591,7 +591,7 @@ test(
packs: {
javascript: [
"codeql/something-else",
"codeql/javascript-experimental-atm-queries@~0.3.0",
"codeql/javascript-experimental-atm-queries@~0.4.0",
],
},
}
@ -613,7 +613,7 @@ test(
{
packs: {
cpp: ["codeql/something-else"],
javascript: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
javascript: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
},
}
);
@ -694,7 +694,7 @@ test(
},
},
{
packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.3.0"],
packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.4.0"],
}
);
@ -826,7 +826,7 @@ test("does not use injected config", async (t: ExecutionContext<unknown>) => {
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
.resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
await codeqlObject.databaseInitCluster(
stubConfig,

35
src/codeql.ts
View file

@ -9,7 +9,7 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { isRunningLocalAction } from "./actions-util";
import { getOptionalInput, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
@ -252,7 +252,6 @@ const CODEQL_MINIMUM_VERSION = "2.6.3";
*/
const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
export const CODEQL_VERSION_CONFIG_FILES = "2.10.1";
const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
const CODEQL_VERSION_FILE_BASELINE_INFORMATION = "2.11.3";
@ -885,6 +884,9 @@ async function getCodeQLForCmd(
}
}
// A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
// Only pass external repository token if a config file is
let externalRepositoryToken: string | undefined;
const configLocation = await generateCodeScanningConfig(
codeql,
config,
@ -892,17 +894,25 @@ async function getCodeQLForCmd(
);
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
externalRepositoryToken = getOptionalInput("external-repository-token");
if (externalRepositoryToken) {
extraArgs.push("--external-repository-token-stdin");
}
}
await runTool(cmd, [
"database",
"init",
"--db-cluster",
config.dbLocation,
`--source-root=${sourceRoot}`,
...extraArgs,
...getExtraOptionsFromEnv(["database", "init"]),
]);
await runTool(
cmd,
[
"database",
"init",
"--db-cluster",
config.dbLocation,
`--source-root=${sourceRoot}`,
...extraArgs,
...getExtraOptionsFromEnv(["database", "init"]),
],
externalRepositoryToken
);
},
async runAutobuild(language: Language) {
const cmdName =
@ -1335,7 +1345,7 @@ export function getExtraOptions(
*/
const maxErrorSize = 20_000;
async function runTool(cmd: string, args: string[] = []) {
async function runTool(cmd: string, args: string[] = [], stdin?: string) {
let output = "";
let error = "";
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
@ -1354,6 +1364,7 @@ async function runTool(cmd: string, args: string[] = []) {
},
},
ignoreReturnCode: true,
input: Buffer.from(stdin || ""),
}).exec();
if (exitCode !== 0)
throw new CommandInvocationError(cmd, args, exitCode, error, output);

53
src/config-utils.ts
View file

@ -582,16 +582,20 @@ async function parseQueryUses(
);
}
// Otherwise, must be a reference to another repo
await addRemoteQueries(
codeQL,
resultMap,
queryUses,
tempDir,
apiDetails,
logger,
configFile
);
// Otherwise, must be a reference to another repo.
// If config parsing is handled in CLI, then this repo will be downloaded
// later by the CLI.
if (!(await useCodeScanningConfigInCli(codeQL, featureEnablement))) {
await addRemoteQueries(
codeQL,
resultMap,
queryUses,
tempDir,
apiDetails,
logger,
configFile
);
}
return false;
}
@ -1724,26 +1728,27 @@ export async function initConfig(
);
}
// The list of queries should not be empty for any language. If it is then
// it is a user configuration error.
for (const language of config.languages) {
const hasBuiltinQueries = config.queries[language]?.builtin.length > 0;
const hasCustomQueries = config.queries[language]?.custom.length > 0;
const hasPacks = (config.packs[language]?.length || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(
`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run."
);
}
}
// When using the codescanning config in the CLI, pack downloads
// happen in the CLI during the `database init` command, so no need
// to download them here.
await logCodeScanningConfigInCli(codeQL, featureEnablement, logger);
if (!(await useCodeScanningConfigInCli(codeQL, featureEnablement))) {
// The list of queries should not be empty for any language. If it is then
// it is a user configuration error.
// This check occurs in the CLI when it parses the config file.
for (const language of config.languages) {
const hasBuiltinQueries = config.queries[language]?.builtin.length > 0;
const hasCustomQueries = config.queries[language]?.custom.length > 0;
const hasPacks = (config.packs[language]?.length || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(
`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run."
);
}
}
const registries = parseRegistries(registriesInput);
await downloadPacks(
codeQL,

2
src/feature-flags.ts
View file

@ -43,7 +43,7 @@ export const featureConfig: Record<
},
[Feature.CliConfigFileEnabled]: {
envVar: "CODEQL_PASS_CONFIG_TO_CLI",
minimumVersion: "2.11.1",
minimumVersion: "2.11.6",
},
[Feature.MlPoweredQueriesEnabled]: {
envVar: "CODEQL_ML_POWERED_QUERIES",