robojerk/codeql-action

Merge branch 'henrymercer/delete-runner-part-2' into henrymercer/require-cli-2.6.3

Browse source
This commit is contained in:
Henry Mercer 2022-11-15 20:57:50 +00:00
commit d48707ce53
79 changed files with 373 additions and 11805 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/actions-util.test.ts
View file

@ -7,7 +7,7 @@ import * as sinon from "sinon";
import * as actionsutil from "./actions-util";
import { setupActionsVars, setupTests } from "./testing-utils";
import { getMode, initializeEnvironment, Mode, withTmpDir } from "./util";
import { initializeEnvironment, withTmpDir } from "./util";
function errorCodes(
actual: actionsutil.CodedError[],
@ -719,13 +719,8 @@ on: ["push"]
});
test("initializeEnvironment", (t) => {
initializeEnvironment(Mode.actions, "1.2.3");
t.deepEqual(getMode(), Mode.actions);
initializeEnvironment("1.2.3");
t.deepEqual(process.env.CODEQL_ACTION_VERSION, "1.2.3");
initializeEnvironment(Mode.runner, "4.5.6");
t.deepEqual(getMode(), Mode.runner);
t.deepEqual(process.env.CODEQL_ACTION_VERSION, "4.5.6");
});
test("isAnalyzingDefaultBranch()", async (t) => {

21
src/actions-util.ts
View file

@ -24,17 +24,11 @@ import {
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
/**
* The utils in this module are meant to be run inside of the action only.
* Code paths from the runner should not enter this module.
*/
/**
* Wrapper around core.getInput for inputs that always have a value.
* Also see getOptionalInput.
*
* This allows us to get stronger type checking of required/optional inputs
* and make behaviour more consistent between actions and the runner.
* This allows us to get stronger type checking of required/optional inputs.
*/
export function getRequiredInput(name: string): string {
return core.getInput(name, { required: true });
@ -44,8 +38,7 @@ export function getRequiredInput(name: string): string {
* Wrapper around core.getInput that converts empty inputs to undefined.
* Also see getRequiredInput.
*
* This allows us to get stronger type checking of required/optional inputs
* and make behaviour more consistent between actions and the runner.
* This allows us to get stronger type checking of required/optional inputs.
*/
export const getOptionalInput = function (name: string): string | undefined {
const value = core.getInput(name);
@ -418,7 +411,7 @@ async function getWorkflowPath(): Promise<string> {
const repo = repo_nwo[1];
const run_id = Number(getRequiredEnvParam("GITHUB_RUN_ID"));
const apiClient = api.getActionsApiClient();
const apiClient = api.getApiClient();
const runsResponse = await apiClient.request(
"GET /repos/:owner/:repo/actions/runs/:run_id?exclude_pull_requests=true",
{
@ -520,7 +513,11 @@ export async function getRef(): Promise<string> {
);
}
const ref = refInput || getRequiredEnvParam("GITHUB_REF");
// Workaround for a limitation of Actions dynamic workflows not setting
// the GITHUB_REF in some cases
const maybeCSRef = process.env["CODE_SCANNING_REF"];
const ref = refInput || maybeCSRef || getRequiredEnvParam("GITHUB_REF");
const sha = shaInput || getRequiredEnvParam("GITHUB_SHA");
// If the ref is a user-provided input, we have to skip logic
@ -775,7 +772,7 @@ export async function sendStatusReport<S extends StatusReportBase>(
const nwo = getRequiredEnvParam("GITHUB_REPOSITORY");
const [owner, repo] = nwo.split("/");
const client = api.getActionsApiClient();
const client = api.getApiClient();
try {
await client.request(

20
src/analyze-action.ts
View file

@ -14,7 +14,7 @@ import {
runFinalize,
runQueries,
} from "./analyze";
import { getApiDetails, getGitHubVersionActionsOnly } from "./api-client";
import { getApiDetails, getGitHubVersion } from "./api-client";
import { runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
@ -170,7 +170,7 @@ async function run() {
let trapCacheUploadTime: number | undefined = undefined;
let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
let didUploadTrapCaches = false;
util.initializeEnvironment(util.Mode.actions, pkg.version);
util.initializeEnvironment(pkg.version);
await util.checkActionVersion(pkg.version);
const logger = getActionsLogger();
@ -199,10 +199,7 @@ async function run() {
);
}
await util.enrichEnvironment(
util.Mode.actions,
await getCodeQL(config.codeQLCmd)
);
await util.enrichEnvironment(await getCodeQL(config.codeQLCmd));
const apiDetails = getApiDetails();
const outputDir = actionsUtil.getRequiredInput("output");
@ -218,14 +215,9 @@ async function run() {
util.getRequiredEnvParam("GITHUB_REPOSITORY")
);
const gitHubVersion = await getGitHubVersionActionsOnly();
const gitHubVersion = await getGitHubVersion();
const features = new Features(
gitHubVersion,
apiDetails,
repositoryNwo,
logger
);
const features = new Features(gitHubVersion, repositoryNwo, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
@ -268,7 +260,6 @@ async function run() {
uploadResult = await upload_lib.uploadFromActions(
outputDir,
config.gitHubVersion,
apiDetails,
logger
);
core.setOutput("sarif-id", uploadResult.sarifID);
@ -295,7 +286,6 @@ async function run() {
await upload_lib.waitForProcessing(
parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY")),
uploadResult.sarifID,
apiDetails,
getActionsLogger()
);
}

110
src/api-client.test.ts
View file

@ -1,10 +1,11 @@
import * as githubUtils from "@actions/github/lib/utils";
import test, { ExecutionContext } from "ava";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { getApiClient } from "./api-client";
import { setupTests } from "./testing-utils";
import { Mode, initializeEnvironment } from "./util";
import * as util from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
@ -18,102 +19,27 @@ test.beforeEach(() => {
pluginStub = sinon.stub(githubUtils.GitHub, "plugin");
githubStub = sinon.stub();
pluginStub.returns(githubStub);
initializeEnvironment(Mode.actions, pkg.version);
util.initializeEnvironment(pkg.version);
});
test("Get the client API", async (t) => {
doTest(
t,
{
auth: "xyz",
externalRepoAuth: "abc",
url: "http://hucairz",
},
undefined,
{
auth: "token xyz",
baseUrl: "http://hucairz/api/v3",
userAgent: `CodeQL-Action/${pkg.version}`,
}
);
});
test("getApiClient", async (t) => {
sinon.stub(actionsUtil, "getRequiredInput").withArgs("token").returns("xyz");
const requiredEnvParamStub = sinon.stub(util, "getRequiredEnvParam");
requiredEnvParamStub
.withArgs("GITHUB_SERVER_URL")
.returns("http://github.localhost");
requiredEnvParamStub
.withArgs("GITHUB_API_URL")
.returns("http://api.github.localhost");
test("Get the client API external", async (t) => {
doTest(
t,
{
auth: "xyz",
externalRepoAuth: "abc",
url: "http://hucairz",
},
{ allowExternal: true },
{
auth: "token abc",
baseUrl: "http://hucairz/api/v3",
userAgent: `CodeQL-Action/${pkg.version}`,
}
);
});
getApiClient();
test("Get the client API external not present", async (t) => {
doTest(
t,
{
auth: "xyz",
url: "http://hucairz",
},
{ allowExternal: true },
{
auth: "token xyz",
baseUrl: "http://hucairz/api/v3",
userAgent: `CodeQL-Action/${pkg.version}`,
}
);
});
test("Get the client API with github url", async (t) => {
doTest(
t,
{
auth: "xyz",
url: "https://github.com/some/invalid/url",
},
undefined,
{
auth: "token xyz",
baseUrl: "https://api.github.com",
userAgent: `CodeQL-Action/${pkg.version}`,
}
);
});
test("Get the API with an API URL directly", async (t) => {
doTest(
t,
{
auth: "xyz",
url: "http://github.localhost",
apiURL: "http://api.github.localhost",
},
undefined,
{
t.assert(
githubStub.calledOnceWithExactly({
auth: "token xyz",
baseUrl: "http://api.github.localhost",
log: sinon.match.any,
userAgent: `CodeQL-Action/${pkg.version}`,
}
})
);
});
function doTest(
t: ExecutionContext<unknown>,
clientArgs: any,
clientOptions: any,
expected: any
) {
getApiClient(clientArgs, clientOptions);
const firstCallArgs = githubStub.args[0];
// log is a function, so we don't need to test for equality of it
delete firstCallArgs[0].log;
t.deepEqual(firstCallArgs, [expected]);
}

47
src/api-client.ts
View file

@ -1,12 +1,10 @@
import * as path from "path";
import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getRequiredInput } from "./actions-util";
import * as util from "./util";
import { getMode, getRequiredEnvParam, GitHubVersion } from "./util";
import { getRequiredEnvParam, GitHubVersion } from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
@ -31,36 +29,20 @@ export interface GitHubApiExternalRepoDetails {
apiURL: string | undefined;
}
export const getApiClient = function (
function createApiClientWithDetails(
apiDetails: GitHubApiCombinedDetails,
{ allowExternal = false } = {}
) {
const auth =
(allowExternal && apiDetails.externalRepoAuth) || apiDetails.auth;
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
const apiURL = apiDetails.apiURL || deriveApiUrl(apiDetails.url);
return new retryingOctokit(
githubUtils.getOctokitOptions(auth, {
baseUrl: apiURL,
userAgent: `CodeQL-${getMode()}/${pkg.version}`,
baseUrl: apiDetails.apiURL,
userAgent: `CodeQL-Action/${pkg.version}`,
log: consoleLogLevel({ level: "debug" }),
})
);
};
// Once the runner is deleted, this can also be removed since the GitHub API URL is always available in an environment variable on Actions.
function deriveApiUrl(githubUrl: string): string {
const url = new URL(githubUrl);
// If we detect this is trying to connect to github.com
// then return with a fixed canonical URL.
if (url.hostname === "github.com" || url.hostname === "api.github.com") {
return "https://api.github.com";
}
// Add the /api/v3 API prefix
url.pathname = path.join(url.pathname, "api", "v3");
return url.toString();
}
export function getApiDetails() {
@ -71,11 +53,14 @@ export function getApiDetails() {
};
}
// Temporary function to aid in the transition to running on and off of github actions.
// Once all code has been converted this function should be removed or made canonical
// and called only from the action entrypoints.
export function getActionsApiClient() {
return getApiClient(getApiDetails());
export function getApiClient() {
return createApiClientWithDetails(getApiDetails());
}
export function getApiClientWithExternalAuth(
apiDetails: GitHubApiCombinedDetails
) {
return createApiClientWithDetails(apiDetails, { allowExternal: true });
}
let cachedGitHubVersion: GitHubVersion | undefined = undefined;
@ -83,15 +68,11 @@ let cachedGitHubVersion: GitHubVersion | undefined = undefined;
/**
* Report the GitHub server version. This is a wrapper around
* util.getGitHubVersion() that automatically supplies GitHub API details using
* GitHub Action inputs. If you need to get the GitHub server version from the
* Runner, please call util.getGitHubVersion() instead.
* GitHub Action inputs.
*
* @returns GitHub version
*/
export async function getGitHubVersionActionsOnly(): Promise<GitHubVersion> {
if (!util.isActions()) {
throw new Error("getGitHubVersionActionsOnly() works only in an action");
}
export async function getGitHubVersion(): Promise<GitHubVersion> {
if (cachedGitHubVersion === undefined) {
cachedGitHubVersion = await util.getGitHubVersion(getApiDetails());
}

9
src/autobuild-action.ts
View file

@ -8,7 +8,7 @@ import {
sendStatusReport,
StatusReportBase,
} from "./actions-util";
import { getGitHubVersionActionsOnly } from "./api-client";
import { getGitHubVersion } from "./api-client";
import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
@ -18,7 +18,6 @@ import {
checkActionVersion,
checkGitHubVersionInRange,
initializeEnvironment,
Mode,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -37,7 +36,7 @@ async function sendCompletedStatusReport(
failingLanguage?: string,
cause?: Error
) {
initializeEnvironment(Mode.actions, pkg.version);
initializeEnvironment(pkg.version);
const status = getActionsStatus(cause, failingLanguage);
const statusReportBase = await createStatusReportBase(
@ -70,8 +69,8 @@ async function run() {
return;
}
const gitHubVersion = await getGitHubVersionActionsOnly();
checkGitHubVersionInRange(gitHubVersion, logger, Mode.actions);
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const config = await configUtils.getConfig(getTemporaryDirectory(), logger);
if (config === undefined) {

13
src/codeql.test.ts
View file

@ -18,7 +18,7 @@ import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
import * as util from "./util";
import { Mode, initializeEnvironment } from "./util";
import { initializeEnvironment } from "./util";
setupTests(test);
@ -39,7 +39,7 @@ const sampleGHAEApiDetails = {
let stubConfig: Config;
test.beforeEach(() => {
initializeEnvironment(Mode.actions, "1.2.3");
initializeEnvironment("1.2.3");
stubConfig = {
languages: [Language.cpp],
@ -424,11 +424,7 @@ test("getExtraOptions throws for bad content", (t) => {
test("getCodeQLActionRepository", (t) => {
const logger = getRunnerLogger(true);
initializeEnvironment(Mode.runner, "1.2.3");
const repoActions = codeql.getCodeQLActionRepository(logger);
t.deepEqual(repoActions, "github/codeql-action");
initializeEnvironment(Mode.actions, "1.2.3");
initializeEnvironment("1.2.3");
// isRunningLocalAction() === true
delete process.env["GITHUB_ACTION_REPOSITORY"];
@ -501,7 +497,6 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
thisStubConfig,
"",
undefined,
undefined,
createFeatures([]),
getRunnerLogger(true)
);
@ -541,7 +536,6 @@ const injectedConfigMacro = test.macro({
thisStubConfig,
"",
undefined,
undefined,
createFeatures([Feature.CliConfigFileEnabled]),
getRunnerLogger(true)
);
@ -848,7 +842,6 @@ test("does not use injected config", async (t: ExecutionContext<unknown>) => {
stubConfig,
"",
undefined,
undefined,
createFeatures([]),
getRunnerLogger(true)
);

25
src/codeql.ts
View file

@ -96,7 +96,6 @@ export interface CodeQL {
config: Config,
sourceRoot: string,
processName: string | undefined,
processLevel: number | undefined,
featureEnablement: FeatureEnablement,
logger: Logger
): Promise<void>;
@ -297,14 +296,6 @@ function getCodeQLBundleName(): string {
}
export function getCodeQLActionRepository(logger: Logger): string {
if (!util.isActions()) {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
} else {
return getActionsCodeQLActionRepository(logger);
}
}
function getActionsCodeQLActionRepository(logger: Logger): string {
if (process.env["GITHUB_ACTION_REPOSITORY"] !== undefined) {
return process.env["GITHUB_ACTION_REPOSITORY"];
}
@ -352,14 +343,14 @@ async function getCodeQLBundleDownloadURL(
if (variant === util.GitHubVariant.GHAE) {
try {
const release = await api
.getApiClient(apiDetails)
.getApiClient()
.request("GET /enterprise/code-scanning/codeql-bundle/find/{tag}", {
tag: CODEQL_BUNDLE_VERSION,
});
const assetID = release.data.assets[codeQLBundleName];
if (assetID !== undefined) {
const download = await api
.getApiClient(apiDetails)
.getApiClient()
.request(
"GET /enterprise/code-scanning/codeql-bundle/download/{asset_id}",
{ asset_id: assetID }
@ -393,7 +384,7 @@ async function getCodeQLBundleDownloadURL(
}
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getApiClient(apiDetails).repos.getReleaseByTag({
const release = await api.getApiClient().repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION,
@ -801,7 +792,6 @@ async function getCodeQLForCmd(
config: Config,
sourceRoot: string,
processName: string | undefined,
processLevel: number | undefined,
featureEnablement: FeatureEnablement
) {
const extraArgs = config.languages.map(
@ -810,14 +800,7 @@ async function getCodeQLForCmd(
if (config.languages.filter((l) => isTracedLanguage(l)).length > 0) {
extraArgs.push("--begin-tracing");
extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
if (processName !== undefined) {
extraArgs.push(`--trace-process-name=${processName}`);
} else {
// We default to 3 if no other arguments are provided since this was the default
// behavior of the Runner. Note this path never happens in the CodeQL Action
// because that always passes in a process name.
extraArgs.push(`--trace-process-level=${processLevel || 3}`);
}
extraArgs.push(`--trace-process-name=${processName}`);
if (
// There's a bug in Lua tracing for Go on Windows in versions earlier than
// `CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED`, so don't use Lua tracing

1
src/config-utils.test.ts
View file

@ -48,6 +48,7 @@ function mockGetContents(
.stub(client.repos, "getContent")
.resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);
sinon.stub(api, "getApiClientWithExternalAuth").value(() => client);
return spyGetContents;
}

10
src/config-utils.ts
View file

@ -856,11 +856,10 @@ export function getUnknownLanguagesError(languages: string[]): string {
*/
async function getLanguagesInRepo(
repository: RepositoryNwo,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<Language[]> {
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api.getApiClient(apiDetails).repos.listLanguages({
const response = await api.getApiClient().repos.listLanguages({
owner: repository.owner,
repo: repository.repo,
});
@ -895,7 +894,6 @@ async function getLanguages(
codeQL: CodeQL,
languagesInput: string | undefined,
repository: RepositoryNwo,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<Language[]> {
// Obtain from action input 'languages' if set
@ -907,7 +905,7 @@ async function getLanguages(
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
languages = await getLanguagesInRepo(repository, apiDetails, logger);
languages = await getLanguagesInRepo(repository, logger);
const availableLanguages = await codeQL.resolveLanguages();
languages = languages.filter((value) => value in availableLanguages);
logger.info(
@ -1012,7 +1010,6 @@ export async function getDefaultConfig(
codeQL,
languagesInput,
repository,
apiDetails,
logger
);
const queries: Queries = {};
@ -1142,7 +1139,6 @@ async function loadConfig(
codeQL,
languagesInput,
repository,
apiDetails,
logger
);
@ -1773,7 +1769,7 @@ async function getRemoteConfig(
}
const response = await api
.getApiClient(apiDetails, { allowExternal: true })
.getApiClientWithExternalAuth(apiDetails)
.repos.getContent({
owner: pieces.groups.owner,
repo: pieces.groups.repo,

3
src/database-upload.test.ts
View file

@ -24,14 +24,13 @@ import {
GitHubVariant,
HTTPError,
initializeEnvironment,
Mode,
withTmpDir,
} from "./util";
setupTests(test);
test.beforeEach(() => {
initializeEnvironment(Mode.actions, "1.2.3");
initializeEnvironment("1.2.3");
});
const testRepoName: RepositoryNwo = { owner: "github", repo: "example" };

2
src/database-upload.ts
View file

@ -32,7 +32,7 @@ export async function uploadDatabases(
return;
}
const client = getApiClient(apiDetails);
const client = getApiClient();
const codeql = await getCodeQL(config.codeQLCmd);
for (const language of config.languages) {

13
src/feature-flags.test.ts
View file

@ -1,6 +1,5 @@
import test from "ava";
import { GitHubApiDetails } from "./api-client";
import {
Feature,
featureConfig,
@ -18,20 +17,14 @@ import {
setupTests,
} from "./testing-utils";
import * as util from "./util";
import { GitHubVariant, initializeEnvironment, Mode, withTmpDir } from "./util";
import { GitHubVariant, initializeEnvironment, withTmpDir } from "./util";
setupTests(test);
test.beforeEach(() => {
initializeEnvironment(Mode.actions, "1.2.3");
initializeEnvironment("1.2.3");
});
const testApiDetails: GitHubApiDetails = {
auth: "1234",
url: "https://github.com",
apiURL: undefined,
};
const testRepositoryNwo = parseRepositoryNwo("github/example");
const ALL_FEATURES_DISABLED_VARIANTS: Array<{
@ -319,7 +312,7 @@ function setUpTests(
): FeatureEnablement {
setupActionsVars(tmpDir, tmpDir);
return new Features(gitHubVersion, testApiDetails, testRepositoryNwo, logger);
return new Features(gitHubVersion, testRepositoryNwo, logger);
}
function includeCodeQlIfRequired(feature: string) {

8
src/feature-flags.ts
View file

@ -1,4 +1,4 @@
import { getApiClient, GitHubApiDetails } from "./api-client";
import { getApiClient } from "./api-client";
import { CodeQL } from "./codeql";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
@ -65,13 +65,11 @@ export class Features implements FeatureEnablement {
constructor(
gitHubVersion: util.GitHubVersion,
apiDetails: GitHubApiDetails,
repositoryNwo: RepositoryNwo,
logger: Logger
) {
this.gitHubFeatureFlags = new GitHubFeatureFlags(
gitHubVersion,
apiDetails,
repositoryNwo,
logger
);
@ -133,7 +131,6 @@ class GitHubFeatureFlags implements FeatureEnablement {
constructor(
private gitHubVersion: util.GitHubVersion,
private apiDetails: GitHubApiDetails,
private repositoryNwo: RepositoryNwo,
private logger: Logger
) {
@ -173,9 +170,8 @@ class GitHubFeatureFlags implements FeatureEnablement {
);
return {};
}
const client = getApiClient(this.apiDetails);
try {
const response = await client.request(
const response = await getApiClient().request(
"GET /repos/:owner/:repo/code-scanning/codeql-action/features",
{
owner: this.repositoryNwo.owner,

19
src/init-action.ts
View file

@ -12,7 +12,7 @@ import {
StatusReportBase,
validateWorkflow,
} from "./actions-util";
import { getGitHubVersionActionsOnly } from "./api-client";
import { getGitHubVersion } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { Feature, FeatureEnablement, Features } from "./feature-flags";
@ -40,7 +40,6 @@ import {
getThreadsFlagValue,
initializeEnvironment,
isHostedRunner,
Mode,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -137,7 +136,7 @@ async function sendSuccessStatusReport(
async function run() {
const startedAt = new Date();
const logger = getActionsLogger();
initializeEnvironment(Mode.actions, pkg.version);
initializeEnvironment(pkg.version);
await checkActionVersion(pkg.version);
let config: configUtils.Config;
@ -151,19 +150,14 @@ async function run() {
apiURL: getRequiredEnvParam("GITHUB_API_URL"),
};
const gitHubVersion = await getGitHubVersionActionsOnly();
checkGitHubVersionInRange(gitHubVersion, logger, Mode.actions);
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY")
);
const features = new Features(
gitHubVersion,
apiDetails,
repositoryNwo,
logger
);
const features = new Features(gitHubVersion, repositoryNwo, logger);
try {
const workflowErrors = await validateWorkflow();
@ -191,7 +185,7 @@ async function run() {
);
codeql = initCodeQLResult.codeql;
toolsVersion = initCodeQLResult.toolsVersion;
await enrichEnvironment(Mode.actions, codeql);
await enrichEnvironment(codeql);
config = await initConfig(
getOptionalInput("languages"),
@ -276,7 +270,6 @@ async function run() {
config,
sourceRoot,
"Runner.Worker.exe",
undefined,
features,
logger
);

2
src/init.ts
View file

@ -89,7 +89,6 @@ export async function runInit(
config: configUtils.Config,
sourceRoot: string,
processName: string | undefined,
processLevel: number | undefined,
featureEnablement: FeatureEnablement,
logger: Logger
): Promise<TracerConfig | undefined> {
@ -102,7 +101,6 @@ export async function runInit(
config,
sourceRoot,
processName,
processLevel,
featureEnablement,
logger
);

615
src/runner.ts
View file

@ -1,615 +0,0 @@
import * as fs from "fs";
import * as path from "path";
import { Command } from "commander";
import del from "del";
import { runFinalize, runQueries } from "./analyze";
import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import { CodeQL, CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
import { Language, parseLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { createFeatures } from "./testing-utils";
import * as upload_lib from "./upload-lib";
import {
checkGitHubVersionInRange,
getAddSnippetsFlag,
getGitHubVersion,
getMemoryFlag,
getThreadsFlag,
parseGitHubUrl,
getGitHubAuth,
initializeEnvironment,
Mode,
codeQlVersionAbove,
enrichEnvironment,
getMemoryFlagValue,
getThreadsFlagValue,
} from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
const program = new Command();
program.version(pkg.version).hook("preAction", () => {
initializeEnvironment(Mode.runner, pkg.version);
});
function getTempDir(userInput: string | undefined): string {
const tempDir = path.join(userInput || process.cwd(), "codeql-runner");
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
return tempDir;
}
const codeqlEnvJsonFilename = "codeql-env.json";
function loadTracerEnvironment(config: Config): { [name: string]: string } {
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
return JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
}
// Imports the environment from codeqlEnvJsonFilename if not already present
function importTracerEnvironment(config: Config) {
if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
const env = loadTracerEnvironment(config);
for (const key of Object.keys(env)) {
process.env[key] = env[key];
}
}
}
// Allow the user to specify refs in full refs/heads/branch format
// or just the short branch name and prepend "refs/heads/" to it.
function parseRef(userInput: string): string {
if (userInput.startsWith("refs/")) {
return userInput;
} else {
return `refs/heads/${userInput}`;
}
}
// Parses the --trace-process-name arg from process.argv, or returns undefined
function parseTraceProcessName(): string | undefined {
for (let i = 0; i < process.argv.length - 1; i++) {
if (process.argv[i] === "--trace-process-name") {
return process.argv[i + 1];
}
}
return undefined;
}
// Parses the --trace-process-level arg from process.argv, or returns undefined
function parseTraceProcessLevel(): number | undefined {
for (let i = 0; i < process.argv.length - 1; i++) {
if (process.argv[i] === "--trace-process-level") {
const v = parseInt(process.argv[i + 1], 10);
return isNaN(v) ? undefined : v;
}
}
return undefined;
}
interface InitArgs {
languages: string | undefined;
queries: string | undefined;
packs: string | undefined;
configFile: string | undefined;
codeqlPath: string | undefined;
tempDir: string | undefined;
toolsDir: string | undefined;
checkoutPath: string | undefined;
repository: string;
githubUrl: string;
githubAuth: string;
githubAuthStdin: boolean;
debug: boolean;
ram: string | undefined;
threads: string | undefined;
}
program
.command("init")
.description("Initializes CodeQL")
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.option(
"--github-auth <auth>",
"GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead."
)
.option(
"--github-auth-stdin",
"Read GitHub Apps token or personal access token from stdin."
)
.option(
"--languages <languages>",
"Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo."
)
.option(
"--queries <queries>",
"Comma-separated list of additional queries to run. This overrides the same setting in a configuration file."
)
.option(
"--packs <packs>",
`[Experimental] Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
specified, then the latest version of the pack is used. By default, this overrides the same setting in a
configuration file; prefix with "+" to use both sets of packs.
This option is only available in single-language analyses. To use packs in multi-language
analyses, you must specify packs in the codeql-config.yml file.`
)
.option("--config-file <file>", "Path to config file.")
.option(
"--codeql-path <path>",
"Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option(
"--tools-dir <dir>",
"Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory."
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--debug", "Print more verbose output", false)
.option(
"--trace-process-name <string>",
"(Advanced, windows-only) Inject a windows tracer of this process into a process with the given process name."
)
.option(
"--trace-process-level <number>",
"(Advanced, windows-only) Inject a windows tracer of this process into a parent process <number> levels up."
)
.option(
"--ram <number>",
"The amount of memory in MB that can be used by CodeQL extractors. " +
"By default, CodeQL extractors will use most of the memory available in the system. " +
'This input also sets the amount of memory that can later be used by the "analyze" command.'
)
.option(
"--threads <number>",
"The number of threads that can be used by CodeQL extractors. " +
"By default, CodeQL extractors will use all the hardware threads available in the system. " +
'This input also sets the number of threads that can later be used by the "analyze" command.'
)
.action(async (cmd: InitArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const tempDir = getTempDir(cmd.tempDir);
const checkoutPath = cmd.checkoutPath || process.cwd();
// Wipe the temp dir
logger.info(`Cleaning temp directory ${tempDir}`);
await del(tempDir, { force: true });
fs.mkdirSync(tempDir, { recursive: true });
const auth = await getGitHubAuth(
logger,
cmd.githubAuth,
cmd.githubAuthStdin
);
const apiDetails = {
auth,
externalRepoAuth: auth,
url: parseGitHubUrl(cmd.githubUrl),
apiURL: undefined,
};
const gitHubVersion = await getGitHubVersion(apiDetails);
checkGitHubVersionInRange(gitHubVersion, logger, Mode.runner);
// Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
// CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
// threads it would ask extractors to use. See help text for the "--ram" and "--threads"
// options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
// for details.
process.env["CODEQL_RAM"] = getMemoryFlagValue(cmd.ram).toString();
process.env["CODEQL_THREADS"] = getThreadsFlagValue(
cmd.threads,
logger
).toString();
let codeql: CodeQL;
if (cmd.codeqlPath !== undefined) {
codeql = await getCodeQL(cmd.codeqlPath);
} else {
codeql = (
await initCodeQL(
undefined,
apiDetails,
tempDir,
gitHubVersion.type,
createFeatures([]),
logger
)
).codeql;
}
await enrichEnvironment(Mode.runner, codeql);
const workspacePath = checkoutPath;
const config = await initConfig(
cmd.languages,
cmd.queries,
cmd.packs,
undefined, // we won't support registries in the runner
cmd.configFile,
undefined,
false,
false,
"",
"",
parseRepositoryNwo(cmd.repository),
tempDir,
codeql,
workspacePath,
gitHubVersion,
apiDetails,
createFeatures([]),
logger
);
const sourceRoot = checkoutPath;
const tracerConfig = await runInit(
codeql,
config,
sourceRoot,
parseTraceProcessName(),
parseTraceProcessLevel(),
createFeatures([]),
logger
);
if (tracerConfig === undefined) {
return;
}
if (
process.platform === "win32" &&
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))
) {
await injectWindowsTracer(
parseTraceProcessName(),
parseTraceProcessLevel(),
config,
codeql,
tracerConfig
);
}
// Always output a json file of the env that can be consumed programmatically
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
fs.writeFileSync(jsonEnvFile, JSON.stringify(tracerConfig.env));
if (process.platform === "win32") {
const batEnvFile = path.join(config.tempDir, "codeql-env.bat");
const batEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `Set ${key}=${value}`)
.join("\n");
fs.writeFileSync(batEnvFile, batEnvFileContents);
const powershellEnvFile = path.join(config.tempDir, "codeql-env.sh");
const powershellEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `$env:${key}="${value}"`)
.join("\n");
fs.writeFileSync(powershellEnvFile, powershellEnvFileContents);
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so that CodeQL can monitor the build. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`
);
} else {
// Assume that anything that's not windows is using a unix-style shell
const shEnvFile = path.join(config.tempDir, "codeql-env.sh");
const shEnvFileContents = Object.entries(tracerConfig.env)
// Some vars contain ${LIB} that we do not want to be expanded when executing this script
.map(
([key, value]) =>
`export ${key}='${value.replace(/'/g, "'\"'\"'")}'`
)
.join("\n");
fs.writeFileSync(shEnvFile, shEnvFileContents);
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so that CodeQL can monitor the build, ` +
`for example by running ". ${shEnvFile}".`
);
}
} catch (e) {
logger.error("Init failed");
logger.error(e instanceof Error ? e : new Error(String(e)));
process.exitCode = 1;
}
});
interface AutobuildArgs {
language: string;
tempDir: string | undefined;
debug: boolean;
}
program
.command("autobuild")
.description("Attempts to automatically build code")
.option(
"--language <language>",
"The language to build. Otherwise will detect the dominant compiled language."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: AutobuildArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error(
"Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
await enrichEnvironment(Mode.runner, await getCodeQL(config.codeQLCmd));
importTracerEnvironment(config);
let languages: Language[] | undefined = undefined;
if (cmd.language !== undefined) {
const language = parseLanguage(cmd.language);
if (language === undefined || !config.languages.includes(language)) {
throw new Error(
`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(
", "
)}.`
);
}
languages = [language];
} else {
languages = await determineAutobuildLanguages(config, logger);
}
if (languages !== undefined) {
for (const language of languages) {
await runAutobuild(language, config, logger);
}
}
} catch (e) {
logger.error("Autobuild failed");
logger.error(e instanceof Error ? e : new Error(String(e)));
process.exitCode = 1;
}
});
interface AnalyzeArgs {
repository: string;
commit: string;
ref: string;
category: string | undefined;
githubUrl: string;
githubAuth: string;
githubAuthStdin: boolean;
checkoutPath: string | undefined;
upload: boolean;
outputDir: string | undefined;
ram: string | undefined;
addSnippets: boolean;
threads: string | undefined;
tempDir: string | undefined;
debug: boolean;
}
program
.command("analyze")
.description("Finishes extracting code and runs CodeQL queries")
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption(
"--commit <commit>",
"SHA of commit that was analyzed. (Required)"
)
.requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.option(
"--github-auth <auth>",
"GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead."
)
.option(
"--github-auth-stdin",
"Read GitHub Apps token or personal access token from stdin."
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--no-upload", "Do not upload results after analysis.")
.option(
"--output-dir <dir>",
"Directory to output SARIF files to. Default is in the temp directory."
)
.option(
"--ram <ram>",
"The amount of memory in MB that can be used by CodeQL for database finalization and query execution. " +
'By default, this command will use the same amount of memory as previously set in the "init" command. ' +
'If the "init" command also does not have an explicit "ram" flag, this command will use most of the ' +
"memory available in the system."
)
.option(
"--no-add-snippets",
"Specify whether to include code snippets in the sarif output."
)
.option(
"--threads <threads>",
"The number of threads that can be used by CodeQL for database finalization and query execution. " +
'By default, this command will use the same number of threads as previously set in the "init" command. ' +
'If the "init" command also does not have an explicit "threads" flag, this command will use all the ' +
"hardware threads available in the system."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option(
"--category <category>",
"String used by Code Scanning for matching the analyses."
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: AnalyzeArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error(
"Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
await enrichEnvironment(Mode.runner, await getCodeQL(config.codeQLCmd));
const auth = await getGitHubAuth(
logger,
cmd.githubAuth,
cmd.githubAuthStdin
);
const apiDetails = {
auth,
url: parseGitHubUrl(cmd.githubUrl),
apiURL: undefined,
};
const outputDir =
cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
let initEnv: { [name: string]: string } = {};
try {
initEnv = loadTracerEnvironment(config);
} catch (err) {
// The init command did not generate a tracer environment file
}
const threads = getThreadsFlag(
cmd.threads || initEnv["CODEQL_THREADS"],
logger
);
const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
await runFinalize(outputDir, threads, memory, config, logger);
await runQueries(
outputDir,
memory,
getAddSnippetsFlag(cmd.addSnippets),
threads,
cmd.category,
config,
logger,
createFeatures([])
);
if (!cmd.upload) {
logger.info("Not uploading results");
return;
}
const sourceRoot = cmd.checkoutPath || process.cwd();
await upload_lib.uploadFromRunner(
outputDir,
parseRepositoryNwo(cmd.repository),
cmd.commit,
parseRef(cmd.ref),
cmd.category,
sourceRoot,
config.gitHubVersion,
apiDetails,
logger
);
} catch (e) {
logger.error("Analyze failed");
logger.error(e instanceof Error ? e : new Error(String(e)));
process.exitCode = 1;
}
});
interface UploadArgs {
sarifFile: string;
repository: string;
commit: string;
ref: string;
category: string | undefined;
githubUrl: string;
githubAuthStdin: boolean;
githubAuth: string;
checkoutPath: string | undefined;
debug: boolean;
}
program
.command("upload")
.description(
"Uploads a SARIF file, or all SARIF files from a directory, to code scanning"
)
.requiredOption(
"--sarif-file <file>",
"SARIF file to upload, or a directory containing multiple SARIF files. (Required)"
)
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption(
"--commit <commit>",
"SHA of commit that was analyzed. (Required)"
)
.requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.option(
"--github-auth <auth>",
"GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead."
)
.option(
"--github-auth-stdin",
"Read GitHub Apps token or personal access token from stdin."
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option(
"--category <category>",
"String used by Code Scanning for matching the analyses."
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: UploadArgs) => {
const logger = getRunnerLogger(cmd.debug);
const auth = await getGitHubAuth(
logger,
cmd.githubAuth,
cmd.githubAuthStdin
);
const apiDetails = {
auth,
url: parseGitHubUrl(cmd.githubUrl),
apiURL: undefined,
};
try {
const gitHubVersion = await getGitHubVersion(apiDetails);
const sourceRoot = cmd.checkoutPath || process.cwd();
await upload_lib.uploadFromRunner(
cmd.sarifFile,
parseRepositoryNwo(cmd.repository),
cmd.commit,
parseRef(cmd.ref),
cmd.category,
sourceRoot,
gitHubVersion,
apiDetails,
logger
);
} catch (e) {
logger.error("Upload failed");
logger.error(e instanceof Error ? e : new Error(String(e)));
process.exitCode = 1;
}
});
program.parse(process.argv);

3
src/upload-lib.test.ts
View file

@ -11,7 +11,6 @@ import {
GitHubVariant,
GitHubVersion,
initializeEnvironment,
Mode,
SarifFile,
withTmpDir,
} from "./util";
@ -19,7 +18,7 @@ import {
setupTests(test);
test.beforeEach(() => {
initializeEnvironment(Mode.actions, "1.2.3");
initializeEnvironment("1.2.3");
});
test("validateSarifFileSchema - valid", (t) => {

209
src/upload-lib.ts
View file

@ -50,7 +50,7 @@ export function combineSarifFiles(sarifFiles: string[]): SarifFile {
export function populateRunAutomationDetails(
sarif: SarifFile,
category: string | undefined,
analysis_key: string | undefined,
analysis_key: string,
environment: string | undefined
): SarifFile {
const automationID = getAutomationID(category, analysis_key, environment);
@ -70,7 +70,7 @@ export function populateRunAutomationDetails(
function getAutomationID(
category: string | undefined,
analysis_key: string | undefined,
analysis_key: string,
environment: string | undefined
): string | undefined {
if (category !== undefined) {
@ -81,12 +81,7 @@ function getAutomationID(
return automationID;
}
// analysis_key is undefined for the runner.
if (analysis_key !== undefined) {
return actionsUtil.computeAutomationID(analysis_key, environment);
}
return undefined;
return actionsUtil.computeAutomationID(analysis_key, environment);
}
// Upload the given payload.
@ -94,7 +89,6 @@ function getAutomationID(
async function uploadPayload(
payload: any,
repositoryNwo: RepositoryNwo,
apiDetails: api.GitHubApiDetails,
logger: Logger
) {
logger.info("Uploading results");
@ -113,16 +107,16 @@ async function uploadPayload(
return;
}
const client = api.getApiClient(apiDetails);
const client = api.getApiClient();
const reqURL = util.isActions()
? "PUT /repos/:owner/:repo/code-scanning/analysis"
: "POST /repos/:owner/:repo/code-scanning/sarifs";
const response = await client.request(reqURL, {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
data: payload,
});
const response = await client.request(
"PUT /repos/:owner/:repo/code-scanning/analysis",
{
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
data: payload,
}
);
logger.debug(`response status: ${response.status}`);
logger.info("Successfully uploaded results");
@ -168,7 +162,6 @@ export function findSarifFilesInDir(sarifPath: string): string[] {
export async function uploadFromActions(
sarifPath: string,
gitHubVersion: util.GitHubVersion,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<UploadResult> {
return await uploadFiles(
@ -185,38 +178,6 @@ export async function uploadFromActions(
actionsUtil.getRequiredInput("checkout_path"),
actionsUtil.getRequiredInput("matrix"),
gitHubVersion,
apiDetails,
logger
);
}
// Uploads a single sarif file or a directory of sarif files
// depending on what the path happens to refer to.
// Returns true iff the upload occurred and succeeded
export async function uploadFromRunner(
sarifPath: string,
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
category: string | undefined,
sourceRoot: string,
gitHubVersion: util.GitHubVersion,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<UploadResult> {
return await uploadFiles(
getSarifFilePaths(sarifPath),
repositoryNwo,
commitOid,
ref,
undefined,
category,
undefined,
undefined,
sourceRoot,
undefined,
gitHubVersion,
apiDetails,
logger
);
}
@ -305,61 +266,51 @@ export function buildPayload(
gitHubVersion: util.GitHubVersion,
mergeBaseCommitOid: string | undefined
) {
if (util.isActions()) {
const payloadObj = {
commit_oid: commitOid,
ref,
analysis_key: analysisKey,
analysis_name: analysisName,
sarif: zippedSarif,
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined as undefined | string,
base_sha: undefined as undefined | string,
};
const payloadObj = {
commit_oid: commitOid,
ref,
analysis_key: analysisKey,
analysis_name: analysisName,
sarif: zippedSarif,
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined as undefined | string,
base_sha: undefined as undefined | string,
};
// This behaviour can be made the default when support for GHES 3.0 is discontinued.
if (
gitHubVersion.type !== util.GitHubVariant.GHES ||
semver.satisfies(gitHubVersion.version, `>=3.1`)
) {
if (actionsUtil.workflowEventName() === "pull_request") {
if (
commitOid === util.getRequiredEnvParam("GITHUB_SHA") &&
mergeBaseCommitOid
) {
// We're uploading results for the merge commit
// and were able to determine the merge base.
// So we use that as the most accurate base.
payloadObj.base_ref = `refs/heads/${util.getRequiredEnvParam(
"GITHUB_BASE_REF"
)}`;
payloadObj.base_sha = mergeBaseCommitOid;
} else if (process.env.GITHUB_EVENT_PATH) {
// Either we're not uploading results for the merge commit
// or we could not determine the merge base.
// Using the PR base is the only option here
const githubEvent = JSON.parse(
fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")
);
payloadObj.base_ref = `refs/heads/${githubEvent.pull_request.base.ref}`;
payloadObj.base_sha = githubEvent.pull_request.base.sha;
}
// This behaviour can be made the default when support for GHES 3.0 is discontinued.
if (
gitHubVersion.type !== util.GitHubVariant.GHES ||
semver.satisfies(gitHubVersion.version, `>=3.1`)
) {
if (actionsUtil.workflowEventName() === "pull_request") {
if (
commitOid === util.getRequiredEnvParam("GITHUB_SHA") &&
mergeBaseCommitOid
) {
// We're uploading results for the merge commit
// and were able to determine the merge base.
// So we use that as the most accurate base.
payloadObj.base_ref = `refs/heads/${util.getRequiredEnvParam(
"GITHUB_BASE_REF"
)}`;
payloadObj.base_sha = mergeBaseCommitOid;
} else if (process.env.GITHUB_EVENT_PATH) {
// Either we're not uploading results for the merge commit
// or we could not determine the merge base.
// Using the PR base is the only option here
const githubEvent = JSON.parse(
fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")
);
payloadObj.base_ref = `refs/heads/${githubEvent.pull_request.base.ref}`;
payloadObj.base_sha = githubEvent.pull_request.base.sha;
}
}
return payloadObj;
} else {
return {
commit_sha: commitOid,
ref,
sarif: zippedSarif,
checkout_uri: checkoutURI,
tool_name: toolNames[0],
};
}
return payloadObj;
}
// Uploads the given set of sarif files.
@ -369,14 +320,13 @@ async function uploadFiles(
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string | undefined,
analysisKey: string,
category: string | undefined,
analysisName: string | undefined,
workflowRunID: number | undefined,
sourceRoot: string,
environment: string | undefined,
gitHubVersion: util.GitHubVersion,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<UploadResult> {
logger.startGroup("Uploading results");
@ -430,12 +380,7 @@ async function uploadFiles(
logger.debug(`Number of results in upload: ${numResultInSarif}`);
// Make the upload
const sarifID = await uploadPayload(
payload,
repositoryNwo,
apiDetails,
logger
);
const sarifID = await uploadPayload(payload, repositoryNwo, logger);
logger.endGroup();
@ -456,11 +401,10 @@ const STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1000;
export async function waitForProcessing(
repositoryNwo: RepositoryNwo,
sarifID: string,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<void> {
logger.startGroup("Waiting for processing to finish");
const client = api.getApiClient(apiDetails);
const client = api.getApiClient();
const statusCheckingStarted = Date.now();
// eslint-disable-next-line no-constant-condition
@ -510,31 +454,28 @@ export async function waitForProcessing(
}
export function validateUniqueCategory(sarif: SarifFile): void {
// This check only works on actions as env vars don't persist between calls to the runner
if (util.isActions()) {
// duplicate categories are allowed in the same sarif file
// but not across multiple sarif files
const categories = {} as Record<string, { id?: string; tool?: string }>;
// duplicate categories are allowed in the same sarif file
// but not across multiple sarif files
const categories = {} as Record<string, { id?: string; tool?: string }>;
for (const run of sarif.runs) {
const id = run?.automationDetails?.id;
const tool = run.tool?.driver?.name;
const category = `${sanitize(id)}_${sanitize(tool)}`;
categories[category] = { id, tool };
}
for (const run of sarif.runs) {
const id = run?.automationDetails?.id;
const tool = run.tool?.driver?.name;
const category = `${sanitize(id)}_${sanitize(tool)}`;
categories[category] = { id, tool };
}
for (const [category, { id, tool }] of Object.entries(categories)) {
const sentinelEnvVar = `CODEQL_UPLOAD_SARIF_${category}`;
if (process.env[sentinelEnvVar]) {
throw new Error(
"Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job per tool/category. " +
"The easiest fix is to specify a unique value for the `category` input. If .runs[].automationDetails.id is specified " +
"in the sarif file, that will take precedence over your configured `category`. " +
`Category: (${id ? id : "none"}) Tool: (${tool ? tool : "none"})`
);
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
for (const [category, { id, tool }] of Object.entries(categories)) {
const sentinelEnvVar = `CODEQL_UPLOAD_SARIF_${category}`;
if (process.env[sentinelEnvVar]) {
throw new Error(
"Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job per tool/category. " +
"The easiest fix is to specify a unique value for the `category` input. If .runs[].automationDetails.id is specified " +
"in the sarif file, that will take precedence over your configured `category`. " +
`Category: (${id ? id : "none"}) Tool: (${tool ? tool : "none"})`
);
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
}
}

10
src/upload-sarif-action.ts
View file

@ -1,7 +1,7 @@
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { getApiDetails, getGitHubVersionActionsOnly } from "./api-client";
import { getGitHubVersion } from "./api-client";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
@ -10,7 +10,6 @@ import {
getRequiredEnvParam,
initializeEnvironment,
isInTestMode,
Mode,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -38,7 +37,7 @@ async function sendSuccessStatusReport(
async function run() {
const startedAt = new Date();
initializeEnvironment(Mode.actions, pkg.version);
initializeEnvironment(pkg.version);
await checkActionVersion(pkg.version);
if (
!(await actionsUtil.sendStatusReport(
@ -53,13 +52,11 @@ async function run() {
}
try {
const apiDetails = getApiDetails();
const gitHubVersion = await getGitHubVersionActionsOnly();
const gitHubVersion = await getGitHubVersion();
const uploadResult = await upload_lib.uploadFromActions(
actionsUtil.getRequiredInput("sarif_file"),
gitHubVersion,
apiDetails,
getActionsLogger()
);
core.setOutput("sarif-id", uploadResult.sarifID);
@ -71,7 +68,6 @@ async function run() {
await upload_lib.waitForProcessing(
parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
uploadResult.sarifID,
apiDetails,
getActionsLogger()
);
}

68
src/util.test.ts
View file

@ -1,16 +1,15 @@
import * as fs from "fs";
import * as os from "os";
import path from "path";
import * as stream from "stream";
import * as core from "@actions/core";
import * as github from "@actions/github";
import test, { ExecutionContext } from "ava";
import test from "ava";
import * as sinon from "sinon";
import * as api from "./api-client";
import { Config } from "./config-utils";
import { getRunnerLogger, Logger } from "./logging";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
@ -240,65 +239,6 @@ test("getGitHubVersion", async (t) => {
t.deepEqual({ type: util.GitHubVariant.DOTCOM }, v3);
});
test("getGitHubAuth", async (t) => {
const msgs: string[] = [];
const mockLogger = {
warning: (msg: string) => msgs.push(msg),
} as unknown as Logger;
// eslint-disable-next-line @typescript-eslint/no-floating-promises
t.throwsAsync(async () => util.getGitHubAuth(mockLogger, "abc", true));
process.env.GITHUB_TOKEN = "123";
t.is("123", await util.getGitHubAuth(mockLogger, undefined, undefined));
t.is(msgs.length, 0);
t.is("abc", await util.getGitHubAuth(mockLogger, "abc", undefined));
t.is(msgs.length, 1); // warning expected
msgs.length = 0;
await mockStdInForAuth(t, mockLogger, "def", "def");
await mockStdInForAuth(t, mockLogger, "def", "", "def");
await mockStdInForAuth(
t,
mockLogger,
"def",
"def\n some extra garbage",
"ghi"
);
await mockStdInForAuth(t, mockLogger, "defghi", "def", "ghi\n123");
await mockStdInForAuthExpectError(t, mockLogger, "");
await mockStdInForAuthExpectError(t, mockLogger, "", " ", "abc");
await mockStdInForAuthExpectError(
t,
mockLogger,
" def\n some extra garbage",
"ghi"
);
t.is(msgs.length, 0);
});
async function mockStdInForAuth(
t: ExecutionContext<any>,
mockLogger: Logger,
expected: string,
...text: string[]
) {
const stdin = stream.Readable.from(text) as any;
t.is(expected, await util.getGitHubAuth(mockLogger, undefined, true, stdin));
}
async function mockStdInForAuthExpectError(
t: ExecutionContext<unknown>,
mockLogger: Logger,
...text: string[]
) {
const stdin = stream.Readable.from(text) as any;
await t.throwsAsync(async () =>
util.getGitHubAuth(mockLogger, undefined, true, stdin)
);
}
const ML_POWERED_JS_STATUS_TESTS: Array<[string[], string]> = [
// If no packs are loaded, status is false.
[[], "false"],
@ -406,9 +346,8 @@ for (const [
test(`checkActionVersion ${reportWarningDescription} for ${versionsDescription}`, async (t) => {
const warningSpy = sinon.spy(core, "warning");
const versionStub = sinon
.stub(api, "getGitHubVersionActionsOnly")
.stub(api, "getGitHubVersion")
.resolves(githubVersion);
const isActionsStub = sinon.stub(util, "isActions").returns(true);
await util.checkActionVersion(version);
if (shouldReportWarning) {
t.true(
@ -420,7 +359,6 @@ for (const [
t.false(warningSpy.called);
}
versionStub.restore();
isActionsStub.restore();
});
}

137
src/util.ts
View file

@ -1,7 +1,6 @@
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import { Readable } from "stream";
import { promisify } from "util";
import * as core from "@actions/core";
@ -329,7 +328,7 @@ export async function getGitHubVersion(
// Doesn't strictly have to be the meta endpoint as we're only
// using the response headers which are available on every request.
const apiClient = getApiClient(apiDetails);
const apiClient = getApiClient();
const response = await apiClient.meta.get();
// This happens on dotcom, although we expect to have already returned in that
@ -348,8 +347,7 @@ export async function getGitHubVersion(
export function checkGitHubVersionInRange(
version: GitHubVersion,
logger: Logger,
toolName: Mode
logger: Logger
) {
if (hasBeenWarnedAboutVersion || version.type !== GitHubVariant.GHES) {
return;
@ -365,20 +363,18 @@ export function checkGitHubVersionInRange(
disallowedAPIVersionReason === DisallowedAPIVersionReason.ACTION_TOO_OLD
) {
logger.warning(
`The CodeQL ${toolName} version you are using is too old to be compatible with GitHub Enterprise ${version.version}. If you experience issues, please upgrade to a more recent version of the CodeQL ${toolName}.`
`The CodeQL Action version you are using is too old to be compatible with GitHub Enterprise ${version.version}. If you experience issues, please upgrade to a more recent version of the CodeQL Action.`
);
}
if (
disallowedAPIVersionReason === DisallowedAPIVersionReason.ACTION_TOO_NEW
) {
logger.warning(
`GitHub Enterprise ${version.version} is too old to be compatible with this version of the CodeQL ${toolName}. If you experience issues, please upgrade to a more recent version of GitHub Enterprise or use an older version of the CodeQL ${toolName}.`
`GitHub Enterprise ${version.version} is too old to be compatible with this version of the CodeQL Action. If you experience issues, please upgrade to a more recent version of GitHub Enterprise or use an older version of the CodeQL Action.`
);
}
hasBeenWarnedAboutVersion = true;
if (isActions()) {
core.exportVariable(CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR, true);
}
core.exportVariable(CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR, true);
}
export enum DisallowedAPIVersionReason {
@ -400,71 +396,6 @@ export function apiVersionInRange(
return undefined;
}
/**
* Retrieves the github auth token for use with the runner. There are
* three possible locations for the token:
*
* 1. from the cli (considered insecure)
* 2. from stdin
* 3. from the GITHUB_TOKEN environment variable
*
* If both 1 & 2 are specified, then an error is thrown.
* If 1 & 3 or 2 & 3 are specified, then the environment variable is ignored.
*
* @param githubAuth a github app token or PAT
* @param fromStdIn read the github app token or PAT from stdin up to, but excluding the first whitespace
* @param readable the readable stream to use for getting the token (defaults to stdin)
*
* @return a promise resolving to the auth token.
*/
export async function getGitHubAuth(
logger: Logger,
githubAuth: string | undefined,
fromStdIn: boolean | undefined,
readable = process.stdin as Readable
): Promise<string> {
if (githubAuth && fromStdIn) {
throw new Error(
"Cannot specify both `--github-auth` and `--github-auth-stdin`. Please use `--github-auth-stdin`, which is more secure."
);
}
if (githubAuth) {
logger.warning(
"Using `--github-auth` via the CLI is insecure. Use `--github-auth-stdin` instead."
);
return githubAuth;
}
if (fromStdIn) {
return new Promise((resolve, reject) => {
let token = "";
readable.on("data", (data) => {
token += data.toString("utf8");
});
readable.on("end", () => {
token = token.split(/\s+/)[0].trim();
if (token) {
resolve(token);
} else {
reject(new Error("Standard input is empty"));
}
});
readable.on("error", (err) => {
reject(err);
});
});
}
if (process.env.GITHUB_TOKEN) {
return process.env.GITHUB_TOKEN;
}
throw new Error(
"No GitHub authentication token was specified. Please provide a token via the GITHUB_TOKEN environment variable, or by adding the `--github-auth-stdin` flag and passing the token via standard input."
);
}
/**
* This error is used to indicate a runtime failure of an exhaustivity check enforced at compile time.
*/
@ -482,22 +413,11 @@ export function assertNever(value: never): never {
throw new ExhaustivityCheckingError(value);
}
export enum Mode {
actions = "Action",
runner = "Runner",
}
/**
* Environment variables to be set by codeql-action and used by the
* CLI. These environment variables are relevant for both the runner
* and the action.
* CLI.
*/
export enum EnvVar {
/**
* The mode of the codeql-action, either 'actions' or 'runner'.
*/
RUN_MODE = "CODEQL_ACTION_RUN_MODE",
/**
* Semver of the codeql-action as specified in package.json.
*/
@ -529,55 +449,30 @@ export enum EnvVar {
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
}
const exportVar = (mode: Mode, name: string, value: string) => {
if (mode === Mode.actions) {
core.exportVariable(name, value);
} else {
process.env[name] = value;
}
};
/**
* Set some initial environment variables that we can set even without
* knowing what version of CodeQL we're running.
*/
export function initializeEnvironment(mode: Mode, version: string) {
exportVar(mode, EnvVar.RUN_MODE, mode);
exportVar(mode, EnvVar.VERSION, version);
exportVar(mode, EnvVar.FEATURE_SARIF_COMBINE, "true");
exportVar(mode, EnvVar.FEATURE_WILL_UPLOAD, "true");
export function initializeEnvironment(version: string) {
core.exportVariable(EnvVar.VERSION, version);
core.exportVariable(EnvVar.FEATURE_SARIF_COMBINE, "true");
core.exportVariable(EnvVar.FEATURE_WILL_UPLOAD, "true");
}
/**
* Enrich the environment variables with further flags that we cannot
* know the value of until we know what version of CodeQL we're running.
*/
export async function enrichEnvironment(mode: Mode, codeql: CodeQL) {
export async function enrichEnvironment(codeql: CodeQL) {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
exportVar(mode, EnvVar.FEATURE_MULTI_LANGUAGE, "false");
exportVar(mode, EnvVar.FEATURE_SANDWICH, "false");
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "false");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "false");
} else {
exportVar(mode, EnvVar.FEATURE_MULTI_LANGUAGE, "true");
exportVar(mode, EnvVar.FEATURE_SANDWICH, "true");
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "true");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "true");
}
}
export function getMode(): Mode {
// Make sure we fail fast if the env var is missing. This should
// only happen if there is a bug in our code and we neglected
// to set the mode early in the process.
const mode = getRequiredEnvParam(EnvVar.RUN_MODE);
if (mode !== Mode.actions && mode !== Mode.runner) {
throw new Error(`Unknown mode: ${mode}.`);
}
return mode;
}
export function isActions(): boolean {
return getMode() === Mode.actions;
}
/**
* Get an environment parameter, but throw an error if it is not set.
*/
@ -739,7 +634,7 @@ export function getMlPoweredJsQueriesStatus(config: Config): string {
*/
export async function checkActionVersion(version: string) {
if (!semver.satisfies(version, ">=2")) {
const githubVersion = await api.getGitHubVersionActionsOnly();
const githubVersion = await api.getGitHubVersion();
// Only log a warning for versions of GHES that are compatible with CodeQL Action version 2.
//
// GHES 3.4 shipped without the v2 tag, but it also shipped without this warning message code.