robojerk/codeql-action

Merge branch 'main' into robertbrignull/go_build_trace

Browse source
This commit is contained in:
Robert 2020-11-04 11:07:46 +00:00 • committed by GitHub
commit 82e3812a35
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 409 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/analyze.ts
View file

@ -122,6 +122,7 @@ async function createdDBForScannedLanguages(
async function finalizeDatabaseCreation(
config: configUtils.Config,
threadsFlag: string,
logger: Logger
) {
await createdDBForScannedLanguages(config, logger);
@ -130,7 +131,8 @@ async function finalizeDatabaseCreation(
for (const language of config.languages) {
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(
util.getCodeQLDatabasePath(config.tempDir, language)
util.getCodeQLDatabasePath(config.tempDir, language),
threadsFlag
);
logger.endGroup();
}
@ -239,7 +241,7 @@ export async function runAnalyze(
fs.mkdirSync(outputDir, { recursive: true });
logger.info("Finalizing database creation");
await finalizeDatabaseCreation(config, logger);
await finalizeDatabaseCreation(config, threadsFlag, logger);
logger.info("Analyzing database");
const queriesStats = await runQueries(

19
src/api-client.test.ts Normal file
View file

@ -0,0 +1,19 @@
import test from "ava";
import { apiVersionInRange, DisallowedAPIVersionReason } from "./api-client";
test("allowed API versions", async (t) => {
t.is(apiVersionInRange("1.33.0", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("1.33.1", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("1.34.0", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("2.0.0", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("2.0.1", "1.33", "2.0"), undefined);
t.is(
apiVersionInRange("1.32.0", "1.33", "2.0"),
DisallowedAPIVersionReason.ACTION_TOO_NEW
);
t.is(
apiVersionInRange("2.1.0", "1.33", "2.0"),
DisallowedAPIVersionReason.ACTION_TOO_OLD
);
});

85
src/api-client.ts
View file

@ -1,22 +1,83 @@
import * as path from "path";
import { exportVariable } from "@actions/core";
import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import { OctokitResponse } from "@octokit/types";
import consoleLogLevel from "console-log-level";
import * as semver from "semver";
import { getRequiredEnvParam, getRequiredInput } from "./actions-util";
import { isLocalRun } from "./util";
import * as apiCompatibility from "./api-compatibility.json";
import { Logger, getActionsLogger } from "./logging";
import { isLocalRun, Mode } from "./util";
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
}
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
const CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR =
"CODEQL_ACTION_WARNED_ABOUT_VERSION";
let hasBeenWarnedAboutVersion = false;
export const getApiClient = function (
githubAuth: string,
githubUrl: string,
allowLocalRun = false
mode: Mode,
logger: Logger,
allowLocalRun = false,
possibleFailureExpected = false
) {
if (isLocalRun() && !allowLocalRun) {
throw new Error("Invalid API call in local run");
}
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
return new retryingOctokit(
const customOctokit = githubUtils.GitHub.plugin(retry.retry, (octokit, _) => {
octokit.hook.after("request", (response: OctokitResponse<any>, _) => {
if (response.status < 400 && !possibleFailureExpected) {
if (hasBeenWarnedAboutVersion) {
return;
}
}
if (
response.headers[GITHUB_ENTERPRISE_VERSION_HEADER] === undefined ||
process.env[CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR] === undefined
) {
return;
}
const installedVersion = response.headers[
GITHUB_ENTERPRISE_VERSION_HEADER
] as string;
const disallowedAPIVersionReason = apiVersionInRange(
installedVersion,
apiCompatibility.minimumVersion,
apiCompatibility.maximumVersion
);
const toolName = mode === "actions" ? "Action" : "Runner";
if (
disallowedAPIVersionReason === DisallowedAPIVersionReason.ACTION_TOO_OLD
) {
logger.warning(
`The CodeQL ${toolName} version you are using is too old to be compatible with GitHub Enterprise ${installedVersion}. If you experience issues, please upgrade to a more recent version of the CodeQL ${toolName}.`
);
}
if (
disallowedAPIVersionReason === DisallowedAPIVersionReason.ACTION_TOO_NEW
) {
logger.warning(
`GitHub Enterprise ${installedVersion} is too old to be compatible with this version of the CodeQL ${toolName}. If you experience issues, please upgrade to a more recent version of GitHub Enterprise or use an older version of the CodeQL ${toolName}.`
);
}
hasBeenWarnedAboutVersion = true;
if (mode === "actions") {
exportVariable(CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR, true);
}
});
});
return new customOctokit(
githubUtils.getOctokitOptions(githubAuth, {
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
@ -46,6 +107,22 @@ export function getActionsApiClient(allowLocalRun = false) {
return getApiClient(
getRequiredInput("token"),
getRequiredEnvParam("GITHUB_SERVER_URL"),
"actions",
getActionsLogger(),
allowLocalRun
);
}
export function apiVersionInRange(
version: string,
minimumVersion: string,
maximumVersion: string
): DisallowedAPIVersionReason | undefined {
if (!semver.satisfies(version, `>=${minimumVersion}`)) {
return DisallowedAPIVersionReason.ACTION_TOO_NEW;
}
if (!semver.satisfies(version, `<=${maximumVersion}`)) {
return DisallowedAPIVersionReason.ACTION_TOO_OLD;
}
return undefined;
}

1
src/api-compatibility.json Normal file
View file

@ -0,0 +1 @@
{"maximumVersion": "3.0", "minimumVersion": "2.22"}

7
src/codeql.ts
View file

@ -74,7 +74,7 @@ export interface CodeQL {
/**
* Finalize a database using 'codeql database finalize'.
*/
finalizeDatabase(databasePath: string): Promise<void>;
finalizeDatabase(databasePath: string, threadsFlag: string): Promise<void>;
/**
* Run 'codeql resolve queries'.
*/
@ -188,7 +188,7 @@ async function getCodeQLBundleDownloadURL(
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api
.getApiClient(githubAuth, githubUrl)
.getApiClient(githubAuth, githubUrl, mode, logger, false, true)
.repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
@ -561,12 +561,13 @@ function getCodeQLForCmd(cmd: string): CodeQL {
errorMatchers
);
},
async finalizeDatabase(databasePath: string) {
async finalizeDatabase(databasePath: string, threadsFlag: string) {
await toolrunnerErrorCatcher(
cmd,
[
"database",
"finalize",
threadsFlag,
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
],

20
src/config-utils.test.ts
View file

@ -78,6 +78,7 @@ test("load empty config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
logger
);
@ -93,6 +94,7 @@ test("load empty config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
logger
)
);
@ -130,6 +132,7 @@ test("loading config saves config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
logger
);
@ -156,6 +159,7 @@ test("load input outside of workspace", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -189,6 +193,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -223,6 +228,7 @@ test("load non-existent input", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -307,6 +313,7 @@ test("load non-empty input", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -368,6 +375,7 @@ test("Default queries are used", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -437,6 +445,7 @@ test("Queries can be specified in config file", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -500,6 +509,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -561,6 +571,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -612,6 +623,7 @@ test("Multiple queries can be specified in workflow file, no config file require
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -681,6 +693,7 @@ test("Queries in workflow file can be added to the set of queries without overri
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -743,6 +756,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
t.fail("initConfig did not throw error");
@ -805,6 +819,7 @@ test("API client used when reading remote config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
t.assert(spyGetContents.called);
@ -829,6 +844,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -861,6 +877,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -889,6 +906,7 @@ test("No detected languages", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -914,6 +932,7 @@ test("Unknown languages", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -960,6 +979,7 @@ function doInvalidInputTest(
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");

27
src/config-utils.ts
View file

@ -9,6 +9,7 @@ import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { Mode } from "./util";
// Property names from the user-supplied config file.
const NAME_PROPERTY = "name";
@ -592,11 +593,12 @@ async function getLanguagesInRepo(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Language[]> {
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api
.getApiClient(githubAuth, githubUrl, true)
.getApiClient(githubAuth, githubUrl, mode, logger, true)
.repos.listLanguages({
owner: repository.owner,
repo: repository.repo,
@ -633,6 +635,7 @@ async function getLanguages(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Language[]> {
// Obtain from action input 'languages' if set
@ -648,6 +651,7 @@ async function getLanguages(
repository,
githubAuth,
githubUrl,
mode,
logger
);
logger.info(
@ -732,6 +736,7 @@ export async function getDefaultConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Config> {
const languages = await getLanguages(
@ -739,6 +744,7 @@ export async function getDefaultConfig(
repository,
githubAuth,
githubUrl,
mode,
logger
);
const queries: Queries = {};
@ -782,6 +788,7 @@ async function loadConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Config> {
let parsedYAML: UserConfig;
@ -791,7 +798,13 @@ async function loadConfig(
configFile = path.resolve(checkoutPath, configFile);
parsedYAML = getLocalConfig(configFile, checkoutPath);
} else {
parsedYAML = await getRemoteConfig(configFile, githubAuth, githubUrl);
parsedYAML = await getRemoteConfig(
configFile,
githubAuth,
githubUrl,
mode,
logger
);
}
// Validate that the 'name' property is syntactically correct,
@ -810,6 +823,7 @@ async function loadConfig(
repository,
githubAuth,
githubUrl,
mode,
logger
);
@ -944,6 +958,7 @@ export async function initConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Config> {
let config: Config;
@ -961,6 +976,7 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
mode,
logger
);
} else {
@ -975,6 +991,7 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
mode,
logger
);
}
@ -1010,7 +1027,9 @@ function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
async function getRemoteConfig(
configFile: string,
githubAuth: string,
githubUrl: string
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp(
@ -1023,7 +1042,7 @@ async function getRemoteConfig(
}
const response = await api
.getApiClient(githubAuth, githubUrl, true)
.getApiClient(githubAuth, githubUrl, mode, logger, true)
.repos.getContent({
owner: pieces.groups.owner,
repo: pieces.groups.repo,

1
src/external-queries.test.ts
View file

@ -59,6 +59,7 @@ test("checkoutExternalQueries", async (t) => {
await runGit(["init", repoPath]);
await runGit(["config", "user.email", "test@github.com"]);
await runGit(["config", "user.name", "Test Test"]);
await runGit(["config", "commit.gpgsign", "false"]);
fs.writeFileSync(path.join(repoPath, "a"), "a content");
await runGit(["add", "a"]);

1
src/init-action.ts
View file

@ -116,6 +116,7 @@ async function run() {
actionsUtil.getRequiredEnvParam("GITHUB_WORKSPACE"),
actionsUtil.getRequiredInput("token"),
actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"),
"actions",
logger
);

2
src/init.ts
View file

@ -46,6 +46,7 @@ export async function initConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger
): Promise<configUtils.Config> {
logger.startGroup("Load language configuration");
@ -60,6 +61,7 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
mode,
logger
);
analysisPaths.printPathFiltersWarning(config, logger);

1
src/runner.ts
View file

@ -172,6 +172,7 @@ program
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
"runner",
logger
);

2
src/upload-lib.ts
View file

@ -56,7 +56,7 @@ async function uploadPayload(
return;
}
const client = api.getApiClient(githubAuth, githubUrl);
const client = api.getApiClient(githubAuth, githubUrl, mode, logger);
const reqURL =
mode === "actions"