robojerk/codeql-action

Merge main into add-multi-cause-markdown-flag.

Browse source
This commit is contained in:
Chris Gavin 2020-11-18 08:26:39 +00:00
commit f94e06a382
No known key found for this signature in database
GPG key ID: 07F950B80C27E4DA
58 changed files with 530 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/actions-util.test.ts
View file

@ -33,16 +33,28 @@ test("getRef() returns head PR ref if GITHUB_SHA not currently checked out", asy
t.deepEqual(actualRef, "refs/pull/1/head");
});
test("prepareEnvironment() when a local run", (t) => {
const origLocalRun = process.env.CODEQL_LOCAL_RUN;
test("getAnalysisKey() when a local run", async (t) => {
process.env.CODEQL_LOCAL_RUN = "true";
process.env.CODEQL_ACTION_ANALYSIS_KEY = "";
process.env.GITHUB_JOB = "";
actionsutil.prepareLocalRunEnvironment();
const actualAnalysisKey = await actionsutil.getAnalysisKey();
t.deepEqual(actualAnalysisKey, "LOCAL-RUN:UNKNOWN-JOB");
});
test("prepareEnvironment() when a local run", (t) => {
process.env.CODEQL_LOCAL_RUN = "false";
process.env.GITHUB_JOB = "YYY";
process.env.CODEQL_ACTION_ANALYSIS_KEY = "TEST";
actionsutil.prepareLocalRunEnvironment();
// unchanged
t.deepEqual(process.env.GITHUB_JOB, "YYY");
t.deepEqual(process.env.CODEQL_ACTION_ANALYSIS_KEY, "TEST");
process.env.CODEQL_LOCAL_RUN = "true";
@ -50,13 +62,22 @@ test("prepareEnvironment() when a local run", (t) => {
// unchanged
t.deepEqual(process.env.GITHUB_JOB, "YYY");
t.deepEqual(process.env.CODEQL_ACTION_ANALYSIS_KEY, "TEST");
process.env.CODEQL_ACTION_ANALYSIS_KEY = "";
actionsutil.prepareLocalRunEnvironment();
// updated
t.deepEqual(process.env.GITHUB_JOB, "YYY");
t.deepEqual(process.env.CODEQL_ACTION_ANALYSIS_KEY, "LOCAL-RUN:YYY");
process.env.GITHUB_JOB = "";
process.env.CODEQL_ACTION_ANALYSIS_KEY = "";
actionsutil.prepareLocalRunEnvironment();
// updated
t.deepEqual(process.env.GITHUB_JOB, "UNKNOWN-JOB");
process.env.CODEQL_LOCAL_RUN = origLocalRun;
t.deepEqual(process.env.CODEQL_ACTION_ANALYSIS_KEY, "LOCAL-RUN:UNKNOWN-JOB");
});

6
src/actions-util.ts
View file

@ -52,6 +52,12 @@ export function prepareLocalRunEnvironment() {
if (!process.env.GITHUB_JOB) {
core.exportVariable("GITHUB_JOB", "UNKNOWN-JOB");
}
if (!process.env.CODEQL_ACTION_ANALYSIS_KEY) {
core.exportVariable(
"CODEQL_ACTION_ANALYSIS_KEY",
`LOCAL-RUN:${process.env.GITHUB_JOB}`
);
}
}
/**

6
src/analyze.ts
View file

@ -122,6 +122,7 @@ async function createdDBForScannedLanguages(
async function finalizeDatabaseCreation(
config: configUtils.Config,
threadsFlag: string,
logger: Logger
) {
await createdDBForScannedLanguages(config, logger);
@ -130,7 +131,8 @@ async function finalizeDatabaseCreation(
for (const language of config.languages) {
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(
util.getCodeQLDatabasePath(config.tempDir, language)
util.getCodeQLDatabasePath(config.tempDir, language),
threadsFlag
);
logger.endGroup();
}
@ -239,7 +241,7 @@ export async function runAnalyze(
fs.mkdirSync(outputDir, { recursive: true });
logger.info("Finalizing database creation");
await finalizeDatabaseCreation(config, logger);
await finalizeDatabaseCreation(config, threadsFlag, logger);
logger.info("Analyzing database");
const queriesStats = await runQueries(

19
src/api-client.test.ts Normal file
View file

@ -0,0 +1,19 @@
import test from "ava";
import { apiVersionInRange, DisallowedAPIVersionReason } from "./api-client";
test("allowed API versions", async (t) => {
t.is(apiVersionInRange("1.33.0", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("1.33.1", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("1.34.0", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("2.0.0", "1.33", "2.0"), undefined);
t.is(apiVersionInRange("2.0.1", "1.33", "2.0"), undefined);
t.is(
apiVersionInRange("1.32.0", "1.33", "2.0"),
DisallowedAPIVersionReason.ACTION_TOO_NEW
);
t.is(
apiVersionInRange("2.1.0", "1.33", "2.0"),
DisallowedAPIVersionReason.ACTION_TOO_OLD
);
});

85
src/api-client.ts
View file

@ -1,22 +1,83 @@
import * as path from "path";
import { exportVariable } from "@actions/core";
import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import { OctokitResponse } from "@octokit/types";
import consoleLogLevel from "console-log-level";
import * as semver from "semver";
import { getRequiredEnvParam, getRequiredInput } from "./actions-util";
import { isLocalRun } from "./util";
import * as apiCompatibility from "./api-compatibility.json";
import { Logger, getActionsLogger } from "./logging";
import { isLocalRun, Mode } from "./util";
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
}
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
const CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR =
"CODEQL_ACTION_WARNED_ABOUT_VERSION";
let hasBeenWarnedAboutVersion = false;
export const getApiClient = function (
githubAuth: string,
githubUrl: string,
allowLocalRun = false
mode: Mode,
logger: Logger,
allowLocalRun = false,
possibleFailureExpected = false
) {
if (isLocalRun() && !allowLocalRun) {
throw new Error("Invalid API call in local run");
}
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
return new retryingOctokit(
const customOctokit = githubUtils.GitHub.plugin(retry.retry, (octokit, _) => {
octokit.hook.after("request", (response: OctokitResponse<any>, _) => {
if (response.status < 400 && !possibleFailureExpected) {
if (hasBeenWarnedAboutVersion) {
return;
}
}
if (
response.headers[GITHUB_ENTERPRISE_VERSION_HEADER] === undefined ||
process.env[CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR] === undefined
) {
return;
}
const installedVersion = response.headers[
GITHUB_ENTERPRISE_VERSION_HEADER
] as string;
const disallowedAPIVersionReason = apiVersionInRange(
installedVersion,
apiCompatibility.minimumVersion,
apiCompatibility.maximumVersion
);
const toolName = mode === "actions" ? "Action" : "Runner";
if (
disallowedAPIVersionReason === DisallowedAPIVersionReason.ACTION_TOO_OLD
) {
logger.warning(
`The CodeQL ${toolName} version you are using is too old to be compatible with GitHub Enterprise ${installedVersion}. If you experience issues, please upgrade to a more recent version of the CodeQL ${toolName}.`
);
}
if (
disallowedAPIVersionReason === DisallowedAPIVersionReason.ACTION_TOO_NEW
) {
logger.warning(
`GitHub Enterprise ${installedVersion} is too old to be compatible with this version of the CodeQL ${toolName}. If you experience issues, please upgrade to a more recent version of GitHub Enterprise or use an older version of the CodeQL ${toolName}.`
);
}
hasBeenWarnedAboutVersion = true;
if (mode === "actions") {
exportVariable(CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR, true);
}
});
});
return new customOctokit(
githubUtils.getOctokitOptions(githubAuth, {
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
@ -46,6 +107,22 @@ export function getActionsApiClient(allowLocalRun = false) {
return getApiClient(
getRequiredInput("token"),
getRequiredEnvParam("GITHUB_SERVER_URL"),
"actions",
getActionsLogger(),
allowLocalRun
);
}
export function apiVersionInRange(
version: string,
minimumVersion: string,
maximumVersion: string
): DisallowedAPIVersionReason | undefined {
if (!semver.satisfies(version, `>=${minimumVersion}`)) {
return DisallowedAPIVersionReason.ACTION_TOO_NEW;
}
if (!semver.satisfies(version, `<=${maximumVersion}`)) {
return DisallowedAPIVersionReason.ACTION_TOO_OLD;
}
return undefined;
}

1
src/api-compatibility.json Normal file
View file

@ -0,0 +1 @@
{"maximumVersion": "3.0", "minimumVersion": "2.22"}

27
src/codeql.ts
View file

@ -74,7 +74,7 @@ export interface CodeQL {
/**
* Finalize a database using 'codeql database finalize'.
*/
finalizeDatabase(databasePath: string): Promise<void>;
finalizeDatabase(databasePath: string, threadsFlag: string): Promise<void>;
/**
* Run 'codeql resolve queries'.
*/
@ -132,14 +132,17 @@ function getCodeQLBundleName(): string {
return `codeql-bundle-${platform}.tar.gz`;
}
function getCodeQLActionRepository(mode: util.Mode): string {
function getCodeQLActionRepository(mode: util.Mode, logger: Logger): string {
if (mode !== "actions") {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
// Actions do not know their own repository name,
// so we currently use this hack to find the name based on where our files are.
// This can be removed once the change to the runner in https://github.com/actions/runner/pull/585 is deployed.
if (process.env["GITHUB_ACTION_REPOSITORY"] !== undefined) {
return process.env["GITHUB_ACTION_REPOSITORY"];
}
// The Actions Runner used with GitHub Enterprise Server 2.22 did not set the GITHUB_ACTION_REPOSITORY variable.
// This fallback logic can be removed after the end-of-support for 2.22 on 2021-09-23.
const runnerTemp = getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
const relativeScriptPath = path.relative(actionsDirectory, __filename);
@ -149,8 +152,14 @@ function getCodeQLActionRepository(mode: util.Mode): string {
relativeScriptPath.startsWith("..") ||
path.isAbsolute(relativeScriptPath)
) {
logger.info(
"The CodeQL Action is checked out locally. Using the default CodeQL Action repository."
);
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
logger.info(
"GITHUB_ACTION_REPOSITORY environment variable was not set. Falling back to legacy method of finding the GitHub Action."
);
const relativeScriptPathParts = relativeScriptPath.split(path.sep);
return `${relativeScriptPathParts[0]}/${relativeScriptPathParts[1]}`;
}
@ -161,7 +170,7 @@ async function getCodeQLBundleDownloadURL(
mode: util.Mode,
logger: Logger
): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(mode);
const codeQLActionRepository = getCodeQLActionRepository(mode, logger);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[githubUrl, codeQLActionRepository],
@ -188,7 +197,7 @@ async function getCodeQLBundleDownloadURL(
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api
.getApiClient(githubAuth, githubUrl)
.getApiClient(githubAuth, githubUrl, mode, logger, false, true)
.repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
@ -561,12 +570,13 @@ function getCodeQLForCmd(cmd: string): CodeQL {
errorMatchers
);
},
async finalizeDatabase(databasePath: string) {
async finalizeDatabase(databasePath: string, threadsFlag: string) {
await toolrunnerErrorCatcher(
cmd,
[
"database",
"finalize",
threadsFlag,
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
],
@ -612,6 +622,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
memoryFlag,
threadsFlag,
databasePath,
"--min-disk-free=1024", // Try to leave at least 1GB free
"--format=sarif-latest",
"--sarif-multicause-markdown",
`--output=${sarifFile}`,

20
src/config-utils.test.ts
View file

@ -78,6 +78,7 @@ test("load empty config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
logger
);
@ -93,6 +94,7 @@ test("load empty config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
logger
)
);
@ -130,6 +132,7 @@ test("loading config saves config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
logger
);
@ -156,6 +159,7 @@ test("load input outside of workspace", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -189,6 +193,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -223,6 +228,7 @@ test("load non-existent input", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -307,6 +313,7 @@ test("load non-empty input", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -368,6 +375,7 @@ test("Default queries are used", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -437,6 +445,7 @@ test("Queries can be specified in config file", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -500,6 +509,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -561,6 +571,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -612,6 +623,7 @@ test("Multiple queries can be specified in workflow file, no config file require
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -681,6 +693,7 @@ test("Queries in workflow file can be added to the set of queries without overri
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
@ -743,6 +756,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
t.fail("initConfig did not throw error");
@ -805,6 +819,7 @@ test("API client used when reading remote config", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
t.assert(spyGetContents.called);
@ -829,6 +844,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -861,6 +877,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -889,6 +906,7 @@ test("No detected languages", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -914,6 +932,7 @@ test("Unknown languages", async (t) => {
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");
@ -960,6 +979,7 @@ function doInvalidInputTest(
tmpDir,
"token",
"https://github.example.com",
"runner",
getRunnerLogger(true)
);
throw new Error("initConfig did not throw error");

27
src/config-utils.ts
View file

@ -9,6 +9,7 @@ import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { Mode } from "./util";
// Property names from the user-supplied config file.
const NAME_PROPERTY = "name";
@ -592,11 +593,12 @@ async function getLanguagesInRepo(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Language[]> {
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api
.getApiClient(githubAuth, githubUrl, true)
.getApiClient(githubAuth, githubUrl, mode, logger, true)
.repos.listLanguages({
owner: repository.owner,
repo: repository.repo,
@ -633,6 +635,7 @@ async function getLanguages(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Language[]> {
// Obtain from action input 'languages' if set
@ -648,6 +651,7 @@ async function getLanguages(
repository,
githubAuth,
githubUrl,
mode,
logger
);
logger.info(
@ -732,6 +736,7 @@ export async function getDefaultConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Config> {
const languages = await getLanguages(
@ -739,6 +744,7 @@ export async function getDefaultConfig(
repository,
githubAuth,
githubUrl,
mode,
logger
);
const queries: Queries = {};
@ -782,6 +788,7 @@ async function loadConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Config> {
let parsedYAML: UserConfig;
@ -791,7 +798,13 @@ async function loadConfig(
configFile = path.resolve(checkoutPath, configFile);
parsedYAML = getLocalConfig(configFile, checkoutPath);
} else {
parsedYAML = await getRemoteConfig(configFile, githubAuth, githubUrl);
parsedYAML = await getRemoteConfig(
configFile,
githubAuth,
githubUrl,
mode,
logger
);
}
// Validate that the 'name' property is syntactically correct,
@ -810,6 +823,7 @@ async function loadConfig(
repository,
githubAuth,
githubUrl,
mode,
logger
);
@ -944,6 +958,7 @@ export async function initConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<Config> {
let config: Config;
@ -961,6 +976,7 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
mode,
logger
);
} else {
@ -975,6 +991,7 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
mode,
logger
);
}
@ -1010,7 +1027,9 @@ function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
async function getRemoteConfig(
configFile: string,
githubAuth: string,
githubUrl: string
githubUrl: string,
mode: Mode,
logger: Logger
): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp(
@ -1023,7 +1042,7 @@ async function getRemoteConfig(
}
const response = await api
.getApiClient(githubAuth, githubUrl, true)
.getApiClient(githubAuth, githubUrl, mode, logger, true)
.repos.getContent({
owner: pieces.groups.owner,
repo: pieces.groups.repo,

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20201008"
"bundleVersion": "codeql-bundle-20201106"
}

1
src/external-queries.test.ts
View file

@ -59,6 +59,7 @@ test("checkoutExternalQueries", async (t) => {
await runGit(["init", repoPath]);
await runGit(["config", "user.email", "test@github.com"]);
await runGit(["config", "user.name", "Test Test"]);
await runGit(["config", "commit.gpgsign", "false"]);
fs.writeFileSync(path.join(repoPath, "a"), "a content");
await runGit(["add", "a"]);

3
src/init-action.ts
View file

@ -116,6 +116,7 @@ async function run() {
actionsUtil.getRequiredEnvParam("GITHUB_WORKSPACE"),
actionsUtil.getRequiredInput("token"),
actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"),
"actions",
logger
);
@ -175,6 +176,8 @@ async function run() {
);
}
}
core.setOutput("codeql-path", config.codeQLCmd);
} catch (error) {
core.setFailed(error.message);
console.log(error);

6
src/init.ts
View file

@ -46,6 +46,7 @@ export async function initConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger
): Promise<configUtils.Config> {
logger.startGroup("Load language configuration");
@ -60,6 +61,7 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
mode,
logger
);
analysisPaths.printPathFiltersWarning(config, logger);
@ -153,6 +155,10 @@ export async function injectWindowsTracer(
Write-Host "Found Runner.Worker.exe process which means we are running on GitHub Actions"
Write-Host "Aborting search early and using process: $p"
Break
} elseif ($p[0].Name -eq "Agent.Worker.exe") {
Write-Host "Found Agent.Worker.exe process which means we are running on Azure Pipelines"
Write-Host "Aborting search early and using process: $p"
Break
} else {
$id = $p[0].ParentProcessId
}

5
src/runner.ts
View file

@ -172,6 +172,7 @@ program
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
"runner",
logger
);
@ -209,7 +210,7 @@ program
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`Please export these variables to future processes so that CodeQL can monitor the build. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`
);
@ -226,7 +227,7 @@ program
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`Please export these variables to future processes so that CodeQL can monitor the build, ` +
`for example by running ". ${shEnvFile}".`
);
}

2
src/upload-lib.ts
View file

@ -56,7 +56,7 @@ async function uploadPayload(
return;
}
const client = api.getApiClient(githubAuth, githubUrl);
const client = api.getApiClient(githubAuth, githubUrl, mode, logger);
const reqURL =
mode === "actions"

4
src/util.test.ts
View file

@ -72,8 +72,6 @@ test("getThreadsFlag() throws if the threads input is not an integer", (t) => {
});
test("isLocalRun() runs correctly", (t) => {
const origLocalRun = process.env.CODEQL_LOCAL_RUN;
process.env.CODEQL_LOCAL_RUN = "";
t.assert(!util.isLocalRun());
@ -88,8 +86,6 @@ test("isLocalRun() runs correctly", (t) => {
process.env.CODEQL_LOCAL_RUN = "hucairz";
t.assert(util.isLocalRun());
process.env.CODEQL_LOCAL_RUN = origLocalRun;
});
test("getExtraOptionsEnvParam() succeeds on valid JSON with invalid options (for now)", (t) => {