robojerk/codeql-action

rename new parameter from configuration to config

Browse source
This commit is contained in:
tgrall 2023-04-01 07:13:01 +02:00
commit fe4a785361
1005 changed files with 731919 additions and 985086 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/actions-util.test.ts
View file

@ -214,6 +214,10 @@ test("initializeEnvironment", (t) => {
});
test("isAnalyzingDefaultBranch()", async (t) => {
process.env["CODE_SCANNING_IS_ANALYZING_DEFAULT_BRANCH"] = "true";
t.deepEqual(await actionsutil.isAnalyzingDefaultBranch(), true);
process.env["CODE_SCANNING_IS_ANALYZING_DEFAULT_BRANCH"] = "false";
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const envFile = path.join(tmpDir, "event.json");

32
src/actions-util.ts
View file

@ -627,9 +627,15 @@ function removeRefsHeadsPrefix(ref: string): string {
return ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) : ref;
}
// Is the version of the repository we are currently analyzing from the default branch,
// or alternatively from another branch or a pull request.
// Returns whether we are analyzing the default branch for the repository.
// For cases where the repository information might not be available (e.g.,
// dynamic workflows), this can be forced by the environment variable
// CODE_SCANNING_IS_ANALYZING_DEFAULT_BRANCH.
export async function isAnalyzingDefaultBranch(): Promise<boolean> {
if (process.env.CODE_SCANNING_IS_ANALYZING_DEFAULT_BRANCH === "true") {
return true;
}
// Get the current ref and trim and refs/heads/ prefix
let currentRef = await getRef();
currentRef = removeRefsHeadsPrefix(currentRef);
@ -674,3 +680,25 @@ export async function printDebugLogs(config: Config) {
walkLogFiles(logsDirectory);
}
}
export type UploadKind = "always" | "failure-only" | "never";
// Parses the `upload` input into an `UploadKind`, converting unspecified and deprecated upload inputs appropriately.
export function getUploadValue(input: string | undefined): UploadKind {
switch (input) {
case undefined:
case "true":
case "always":
return "always";
case "false":
case "failure-only":
return "failure-only";
case "never":
return "never";
default:
core.warning(
`Unrecognized 'upload' input to 'analyze' Action: ${input}. Defaulting to 'always'.`
);
return "always";
}
}

17
src/analyze-action.ts
View file

@ -16,14 +16,17 @@ import {
} from "./analyze";
import { getApiDetails, getGitHubVersion } from "./api-client";
import { runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { enrichEnvironment, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY } from "./shared-environment";
import {
CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY,
CODEQL_ACTION_DID_AUTOBUILD_GOLANG,
} from "./shared-environment";
import { getTotalCacheSize, uploadTrapCaches } from "./trap-caching";
import * as upload_lib from "./upload-lib";
import { UploadResult } from "./upload-lib";
@ -130,7 +133,7 @@ function doesGoExtractionOutputExist(config: Config): boolean {
* an autobuild step or manual build steps.
*
* - We detect whether an autobuild step is present by checking the
* `util.DID_AUTOBUILD_GO_ENV_VAR_NAME` environment variable, which is set
* `CODEQL_ACTION_DID_AUTOBUILD_GOLANG` environment variable, which is set
* when the autobuilder is invoked.
* - We detect whether the Go database has already been finalized in case it
* has been manually set in a prior Action step.
@ -141,7 +144,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
if (!config.languages.includes(Language.go)) {
return;
}
if (process.env[util.DID_AUTOBUILD_GO_ENV_VAR_NAME] === "true") {
if (process.env[CODEQL_ACTION_DID_AUTOBUILD_GOLANG] === "true") {
logger.debug("Won't run Go autobuild since it has already been run.");
return;
}
@ -204,7 +207,7 @@ async function run() {
);
}
await util.enrichEnvironment(await getCodeQL(config.codeQLCmd));
await enrichEnvironment(await getCodeQL(config.codeQLCmd));
const apiDetails = getApiDetails();
const outputDir = actionsUtil.getRequiredInput("output");
@ -265,8 +268,8 @@ async function run() {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
}
core.setOutput("db-locations", dbLocations);
if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
const uploadInput = actionsUtil.getOptionalInput("upload");
if (runStats && actionsUtil.getUploadValue(uploadInput) === "always") {
uploadResult = await upload_lib.uploadFromActions(
outputDir,
actionsUtil.getRequiredInput("checkout_path"),

3
src/analyze.ts
View file

@ -369,7 +369,8 @@ export async function runQueries(
enableDebugLogging ? "-vv" : "-v",
automationDetailsId,
config,
features
features,
logger
);
}

2
src/api-compatibility.json
View file

@ -1 +1 @@
{"maximumVersion": "3.8", "minimumVersion": "3.4"}
{"maximumVersion": "3.9", "minimumVersion": "3.5"}

9
src/autobuild-action.ts
View file

@ -14,11 +14,8 @@ import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import {
DID_AUTOBUILD_GO_ENV_VAR_NAME,
checkGitHubVersionInRange,
initializeEnvironment,
} from "./util";
import { CODEQL_ACTION_DID_AUTOBUILD_GOLANG } from "./shared-environment";
import { checkGitHubVersionInRange, initializeEnvironment } from "./util";
interface AutobuildStatusReport extends StatusReportBase {
/** Comma-separated set of languages being auto-built. */
@ -88,7 +85,7 @@ async function run() {
currentLanguage = language;
await runAutobuild(language, config, logger);
if (language === Language.go) {
core.exportVariable(DID_AUTOBUILD_GO_ENV_VAR_NAME, "true");
core.exportVariable(CODEQL_ACTION_DID_AUTOBUILD_GOLANG, "true");
}
}
}

12
src/codeql.test.ts
View file

@ -637,7 +637,8 @@ test("databaseInterpretResults() does not set --sarif-add-query-help for 2.7.0",
"-v",
"",
stubConfig,
createFeatures([])
createFeatures([]),
getRunnerLogger(true)
);
t.false(
runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"),
@ -660,7 +661,8 @@ test("databaseInterpretResults() sets --sarif-add-query-help for 2.7.1", async (
"-v",
"",
stubConfig,
createFeatures([])
createFeatures([]),
getRunnerLogger(true)
);
t.true(
runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"),
@ -1158,7 +1160,8 @@ test("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3"
"-v",
"",
stubConfig,
createFeatures([])
createFeatures([]),
getRunnerLogger(true)
);
t.true(
runnerConstructorStub.firstCall.args[1].includes(
@ -1183,7 +1186,8 @@ test("databaseInterpretResults() does not set --sarif-add-baseline-file-info for
"-v",
"",
stubConfig,
createFeatures([])
createFeatures([]),
getRunnerLogger(true)
);
t.false(
runnerConstructorStub.firstCall.args[1].includes(

92
src/codeql.ts
View file

@ -1,6 +1,7 @@
import * as fs from "fs";
import * as path from "path";
import * as core from "@actions/core";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as yaml from "js-yaml";
@ -17,6 +18,7 @@ import { ToolsSource } from "./init";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { EnvVar } from "./shared-environment";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import {
getTrapCachingExtractorConfigArgs,
@ -179,12 +181,26 @@ export interface CodeQL {
verbosityFlag: string | undefined,
automationDetailsId: string | undefined,
config: Config,
features: FeatureEnablement
features: FeatureEnablement,
logger: Logger
): Promise<string>;
/**
* Run 'codeql database print-baseline'.
*/
databasePrintBaseline(databasePath: string): Promise<string>;
/**
* Run 'codeql database export-diagnostics'
*
* Note that the "--sarif-include-diagnostics" option is always used, as the command should
* only be run if the ExportDiagnosticsEnabled feature flag is on.
*/
databaseExportDiagnostics(
databasePath: string,
sarifFile: string,
automationDetailsId: string | undefined,
tempDir: string,
logger: Logger
): Promise<void>;
/**
* Run 'codeql diagnostics export'.
*/
@ -429,6 +445,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
partialCodeql,
"databasePrintBaseline"
),
databaseExportDiagnostics: resolveFunction(
partialCodeql,
"databaseExportDiagnostics"
),
diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
};
return cachedCodeQL;
@ -851,15 +871,23 @@ export async function getCodeQLForCmd(
verbosityFlag: string,
automationDetailsId: string | undefined,
config: Config,
features: FeatureEnablement
features: FeatureEnablement,
logger: Logger
): Promise<string> {
const shouldExportDiagnostics = await features.getValue(
Feature.ExportDiagnosticsEnabled,
this
);
const codeqlOutputFile = shouldExportDiagnostics
? path.join(config.tempDir, "codeql-intermediate-results.sarif")
: sarifFile;
const codeqlArgs = [
"database",
"interpret-results",
threadsFlag,
"--format=sarif-latest",
verbosityFlag,
`--output=${sarifFile}`,
`--output=${codeqlOutputFile}`,
addSnippetsFlag,
"--print-diagnostics-summary",
"--print-metrics-summary",
@ -880,6 +908,11 @@ export async function getCodeQLForCmd(
) {
codeqlArgs.push("--sarif-add-baseline-file-info");
}
if (shouldExportDiagnostics) {
codeqlArgs.push("--sarif-include-diagnostics");
} else if (await util.codeQlVersionAbove(this, "2.12.4")) {
codeqlArgs.push("--no-sarif-include-diagnostics");
}
codeqlArgs.push(databasePath);
if (querySuitePaths) {
codeqlArgs.push(...querySuitePaths);
@ -890,6 +923,11 @@ export async function getCodeQLForCmd(
codeqlArgs,
errorMatchers
);
if (shouldExportDiagnostics) {
util.fixInvalidNotificationsInFile(codeqlOutputFile, sarifFile, logger);
}
return returnState.stdout;
},
async databasePrintBaseline(databasePath: string): Promise<string> {
@ -982,6 +1020,40 @@ export async function getCodeQLForCmd(
];
await new toolrunner.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(
databasePath: string,
sarifFile: string,
automationDetailsId: string | undefined,
tempDir: string,
logger: Logger
): Promise<void> {
const intermediateSarifFile = path.join(
tempDir,
"codeql-intermediate-results.sarif"
);
const args = [
"database",
"export-diagnostics",
`${databasePath}`,
"--db-cluster", // Database is always a cluster for CodeQL versions that support diagnostics.
"--format=sarif-latest",
`--output=${intermediateSarifFile}`,
"--sarif-include-diagnostics", // ExportDiagnosticsEnabled is always true if this command is run.
"-vvv",
...getExtraOptionsFromEnv(["diagnostics", "export"]),
];
if (automationDetailsId !== undefined) {
args.push("--sarif-category", automationDetailsId);
}
await new toolrunner.ToolRunner(cmd, args).exec();
// Fix invalid notifications in the SARIF file output by CodeQL.
util.fixInvalidNotificationsInFile(
intermediateSarifFile,
sarifFile,
logger
);
},
async diagnosticsExport(
sarifFile: string,
automationDetailsId: string | undefined,
@ -1231,3 +1303,17 @@ async function getCodeScanningConfigExportArguments(
}
return [];
}
/**
* Enrich the environment variables with further flags that we cannot
* know the value of until we know what version of CodeQL we're running.
*/
export async function enrichEnvironment(codeql: CodeQL) {
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "false");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "false");
} else {
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "true");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "true");
}
}

10
src/config-utils.ts
View file

@ -1690,7 +1690,7 @@ export async function initConfig(
registriesInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
configuration: string | undefined,
configAsParameter: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
@ -1706,15 +1706,15 @@ export async function initConfig(
): Promise<Config> {
let config: Config;
// if configuration is set, it takes precedence over configFile
if (configuration) {
// if configAsParameter is set, it takes precedence over configFile
if (configAsParameter) {
const configFileToCreate = path.resolve(
workspacePath,
"user-config-from-action.yml"
);
fs.writeFileSync(configFileToCreate, configuration);
fs.writeFileSync(configFileToCreate, configAsParameter);
configFile = configFileToCreate;
logger.debug(`Using configuration from action input: ${configFile}`);
logger.debug(`Using config from action input: ${configFile}`);
}
// If no config file was provided create an empty one

8
src/defaults.json
View file

@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-20230304",
"cliVersion": "2.12.4",
"priorBundleVersion": "codeql-bundle-20230217",
"priorCliVersion": "2.12.3"
"bundleVersion": "codeql-bundle-20230317",
"cliVersion": "2.12.5",
"priorBundleVersion": "codeql-bundle-20230304",
"priorCliVersion": "2.12.4"
}

16
src/feature-flags.ts
View file

@ -22,7 +22,10 @@ export type CodeQLDefaultVersionInfo =
| {
cliVersion: string;
tagName: string;
variant: util.GitHubVariant.GHAE | util.GitHubVariant.GHES;
variant:
| util.GitHubVariant.GHAE
| util.GitHubVariant.GHES
| util.GitHubVariant.GHE_DOTCOM;
};
export interface FeatureEnablement {
@ -37,6 +40,7 @@ export enum Feature {
CliConfigFileEnabled = "cli_config_file_enabled",
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
ExportCodeScanningConfigEnabled = "export_code_scanning_config_enabled",
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
}
@ -58,8 +62,14 @@ export const featureConfig: Record<
[Feature.ExportCodeScanningConfigEnabled]: {
envVar: "CODEQL_ACTION_EXPORT_CODE_SCANNING_CONFIG",
minimumVersion: "2.12.3",
defaultValue: false,
defaultValue: true,
},
[Feature.ExportDiagnosticsEnabled]: {
envVar: "CODEQL_ACTION_EXPORT_DIAGNOSTICS",
minimumVersion: "2.12.4",
defaultValue: true,
},
[Feature.MlPoweredQueriesEnabled]: {
envVar: "CODEQL_ML_POWERED_QUERIES",
minimumVersion: "2.7.5",
@ -68,7 +78,7 @@ export const featureConfig: Record<
[Feature.UploadFailedSarifEnabled]: {
envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
minimumVersion: "2.11.3",
defaultValue: false,
defaultValue: true,
},
};

154
src/init-action-post-helper.test.ts
View file

@ -83,7 +83,7 @@ test("post: init action with debug mode on", async (t) => {
});
});
test("uploads failed SARIF run for typical workflow", async (t) => {
test("uploads failed SARIF run with `diagnostics export` if feature flag is off", async (t) => {
const actionsWorkflow = createTestWorkflow([
{
name: "Checkout repository",
@ -107,7 +107,7 @@ test("uploads failed SARIF run for typical workflow", async (t) => {
await testFailedSarifUpload(t, actionsWorkflow, { category: "my-category" });
});
test("doesn't upload failed SARIF for workflow with upload: false", async (t) => {
test("uploads failed SARIF run with `diagnostics export` if the database doesn't exist", async (t) => {
const actionsWorkflow = createTestWorkflow([
{
name: "Checkout repository",
@ -125,16 +125,107 @@ test("doesn't upload failed SARIF for workflow with upload: false", async (t) =>
uses: "github/codeql-action/analyze@v2",
with: {
category: "my-category",
upload: false,
},
},
]);
const result = await testFailedSarifUpload(t, actionsWorkflow, {
expectUpload: false,
await testFailedSarifUpload(t, actionsWorkflow, {
category: "my-category",
databaseExists: false,
});
t.is(result.upload_failed_run_skipped_because, "SARIF upload is disabled");
});
test("uploads failed SARIF run with database export-diagnostics if the database exists and feature flag is on", async (t) => {
const actionsWorkflow = createTestWorkflow([
{
name: "Checkout repository",
uses: "actions/checkout@v3",
},
{
name: "Initialize CodeQL",
uses: "github/codeql-action/init@v2",
with: {
languages: "javascript",
},
},
{
name: "Perform CodeQL Analysis",
uses: "github/codeql-action/analyze@v2",
with: {
category: "my-category",
},
},
]);
await testFailedSarifUpload(t, actionsWorkflow, {
category: "my-category",
exportDiagnosticsEnabled: true,
});
});
const UPLOAD_INPUT_TEST_CASES = [
{
uploadInput: "true",
shouldUpload: true,
},
{
uploadInput: "false",
shouldUpload: true,
},
{
uploadInput: "always",
shouldUpload: true,
},
{
uploadInput: "failure-only",
shouldUpload: true,
},
{
uploadInput: "never",
shouldUpload: false,
},
{
uploadInput: "unrecognized-value",
shouldUpload: true,
},
];
for (const { uploadInput, shouldUpload } of UPLOAD_INPUT_TEST_CASES) {
test(`does ${
shouldUpload ? "" : "not "
}upload failed SARIF run for workflow with upload: ${uploadInput}`, async (t) => {
const actionsWorkflow = createTestWorkflow([
{
name: "Checkout repository",
uses: "actions/checkout@v3",
},
{
name: "Initialize CodeQL",
uses: "github/codeql-action/init@v2",
with: {
languages: "javascript",
},
},
{
name: "Perform CodeQL Analysis",
uses: "github/codeql-action/analyze@v2",
with: {
category: "my-category",
upload: uploadInput,
},
},
]);
const result = await testFailedSarifUpload(t, actionsWorkflow, {
category: "my-category",
expectUpload: shouldUpload,
});
if (!shouldUpload) {
t.is(
result.upload_failed_run_skipped_because,
"SARIF upload is disabled"
);
}
});
}
test("uploading failed SARIF run succeeds when workflow uses an input with a matrix var", async (t) => {
const actionsWorkflow = createTestWorkflow([
{
@ -239,11 +330,15 @@ async function testFailedSarifUpload(
actionsWorkflow: workflow.Workflow,
{
category,
databaseExists = true,
expectUpload = true,
exportDiagnosticsEnabled = false,
matrix = {},
}: {
category?: string;
databaseExists?: boolean;
expectUpload?: boolean;
exportDiagnosticsEnabled?: boolean;
matrix?: { [key: string]: string };
} = {}
): Promise<initActionPostHelper.UploadFailedSarifResult> {
@ -253,6 +348,9 @@ async function testFailedSarifUpload(
languages: [],
packs: [],
} as unknown as configUtils.Config;
if (databaseExists) {
config.dbLocation = "path/to/database";
}
process.env["GITHUB_JOB"] = "analyze";
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
process.env["GITHUB_WORKSPACE"] =
@ -264,6 +362,10 @@ async function testFailedSarifUpload(
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeql, "getCodeQL").resolves(codeqlObject);
const databaseExportDiagnosticsStub = sinon.stub(
codeqlObject,
"databaseExportDiagnostics"
);
const diagnosticsExportStub = sinon.stub(codeqlObject, "diagnosticsExport");
sinon.stub(workflow, "getWorkflow").resolves(actionsWorkflow);
@ -275,10 +377,15 @@ async function testFailedSarifUpload(
} as uploadLib.UploadResult);
const waitForProcessing = sinon.stub(uploadLib, "waitForProcessing");
const features = [Feature.UploadFailedSarifEnabled];
if (exportDiagnosticsEnabled) {
features.push(Feature.ExportDiagnosticsEnabled);
}
const result = await initActionPostHelper.tryUploadSarifIfRunFailed(
config,
parseRepositoryNwo("github/codeql-action"),
createFeatures([Feature.UploadFailedSarifEnabled]),
createFeatures(features),
getRunnerLogger(true)
);
if (expectUpload) {
@ -286,17 +393,28 @@ async function testFailedSarifUpload(
raw_upload_size_bytes: 20,
zipped_upload_size_bytes: 10,
});
}
if (expectUpload) {
t.true(
diagnosticsExportStub.calledOnceWith(
sinon.match.string,
category,
sinon.match.any,
sinon.match.any
),
`Actual args were: ${diagnosticsExportStub.args}`
);
if (databaseExists && exportDiagnosticsEnabled) {
t.true(
databaseExportDiagnosticsStub.calledOnceWith(
config.dbLocation,
sinon.match.string,
category,
sinon.match.any,
sinon.match.any
),
`Actual args were: ${databaseExportDiagnosticsStub.args}`
);
} else {
t.true(
diagnosticsExportStub.calledOnceWith(
sinon.match.string,
category,
config,
sinon.match.any
),
`Actual args were: ${diagnosticsExportStub.args}`
);
}
t.true(
uploadFromActions.calledOnceWith(
sinon.match.string,

25
src/init-action-post-helper.ts
View file

@ -56,17 +56,37 @@ async function maybeUploadFailedSarif(
const workflow = await getWorkflow();
const jobName = getRequiredEnvParam("GITHUB_JOB");
const matrix = parseMatrixInput(actionsUtil.getRequiredInput("matrix"));
const shouldUpload = getUploadInputOrThrow(workflow, jobName, matrix);
if (
getUploadInputOrThrow(workflow, jobName, matrix) !== "true" ||
!["always", "failure-only"].includes(
actionsUtil.getUploadValue(shouldUpload)
) ||
isInTestMode()
) {
return { upload_failed_run_skipped_because: "SARIF upload is disabled" };
}
const category = getCategoryInputOrThrow(workflow, jobName, matrix);
const checkoutPath = getCheckoutPathInputOrThrow(workflow, jobName, matrix);
const databasePath = config.dbLocation;
const sarifFile = "../codeql-failed-run.sarif";
await codeql.diagnosticsExport(sarifFile, category, config, features);
// If there is no database or the feature flag is off, we run 'export diagnostics'
if (
databasePath === undefined ||
!(await features.getValue(Feature.ExportDiagnosticsEnabled, codeql))
) {
await codeql.diagnosticsExport(sarifFile, category, config, features);
} else {
// We call 'database export-diagnostics' to find any per-database diagnostics.
await codeql.databaseExportDiagnostics(
databasePath,
sarifFile,
category,
config.tempDir,
logger
);
}
core.info(`Uploading failed SARIF file ${sarifFile}`);
const uploadResult = await uploadLib.uploadFromActions(
@ -134,6 +154,7 @@ export async function run(
features,
logger
);
if (uploadFailedSarifResult.upload_failed_run_skipped_because) {
logger.debug(
"Won't upload a failed SARIF file for this CodeQL code scanning run because: " +

9
src/init-action.ts
View file

@ -14,7 +14,11 @@ import {
StatusReportBase,
} from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import {
CodeQL,
CODEQL_VERSION_NEW_TRACING,
enrichEnvironment,
} from "./codeql";
import * as configUtils from "./config-utils";
import { Feature, Features } from "./feature-flags";
import {
@ -35,7 +39,6 @@ import {
codeQlVersionAbove,
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
enrichEnvironment,
getMemoryFlagValue,
getMlPoweredJsQueriesStatus,
getRequiredEnvParam,
@ -256,7 +259,7 @@ async function run() {
registriesInput,
getOptionalInput("config-file"),
getOptionalInput("db-location"),
getOptionalInput("configuration"),
getOptionalInput("config"),
getTrapCachingEnabled(),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.

4
src/init.ts
View file

@ -58,7 +58,7 @@ export async function initConfig(
registriesInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
queryFilters: string | undefined,
configAsParameter: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
@ -80,7 +80,7 @@ export async function initConfig(
registriesInput,
configFile,
dbLocation,
queryFilters,
configAsParameter,
trapCachingEnabled,
debugMode,
debugArtifactName,

50
src/shared-environment.ts
View file

@ -1,3 +1,46 @@
/**
* Environment variables to be set by codeql-action and used by the
* CLI.
*/
export enum EnvVar {
/**
* Semver of the codeql-action as specified in package.json.
*/
VERSION = "CODEQL_ACTION_VERSION",
/**
* If set to a truthy value, then the codeql-action might combine SARIF
* output from several `interpret-results` runs for the same Language.
*/
FEATURE_SARIF_COMBINE = "CODEQL_ACTION_FEATURE_SARIF_COMBINE",
/**
* If set to the "true" string, then the codeql-action will upload SARIF,
* not the cli.
*/
FEATURE_WILL_UPLOAD = "CODEQL_ACTION_FEATURE_WILL_UPLOAD",
/**
* If set to the "true" string, then the codeql-action is using its
* own deprecated and non-standard way of scanning for multiple
* languages.
*/
FEATURE_MULTI_LANGUAGE = "CODEQL_ACTION_FEATURE_MULTI_LANGUAGE",
/**
* If set to the "true" string, then the codeql-action is using its
* own sandwiched workflow mechanism
*/
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
}
/**
* Environment variable that is set to true when the CodeQL Action has invoked
* the Go autobuilder.
*/
export const CODEQL_ACTION_DID_AUTOBUILD_GOLANG =
"CODEQL_ACTION_DID_AUTOBUILD_GOLANG";
/**
* This environment variable is set to true when the `analyze` Action
* completes successfully.
@ -11,6 +54,13 @@ export const CODEQL_ACTION_TESTING_ENVIRONMENT =
/** Used to disable uploading SARIF results or status reports to the GitHub API */
export const CODEQL_ACTION_TEST_MODE = "CODEQL_ACTION_TEST_MODE";
/**
* Used to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
export const CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX =
"CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX";
/**
* The time at which the first action (normally init) started executing.
* If a workflow invokes a different action without first invoking the init

4
src/upload-lib.ts
View file

@ -13,7 +13,7 @@ import * as api from "./api-client";
import * as fingerprints from "./fingerprints";
import { Logger } from "./logging";
import { parseRepositoryNwo, RepositoryNwo } from "./repository";
import * as sharedEnv from "./shared-environment";
import { CODEQL_WORKFLOW_STARTED_AT } from "./shared-environment";
import * as util from "./util";
import { SarifFile, SarifResult, SarifRun } from "./util";
import * as workflow from "./workflow";
@ -272,7 +272,7 @@ export function buildPayload(
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
started_at: process.env[CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined as undefined | string,
base_sha: undefined as undefined | string,

67
src/util.test.ts
View file

@ -9,7 +9,7 @@ import * as sinon from "sinon";
import * as api from "./api-client";
import { Config } from "./config-utils";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
@ -237,6 +237,14 @@ test("getGitHubVersion", async (t) => {
apiURL: undefined,
});
t.deepEqual({ type: util.GitHubVariant.DOTCOM }, v3);
mockGetMetaVersionHeader("ghe.com");
const gheDotcom = await util.getGitHubVersion({
auth: "",
url: "https://foo.ghe.com",
apiURL: undefined,
});
t.deepEqual({ type: util.GitHubVariant.GHE_DOTCOM }, gheDotcom);
});
const ML_POWERED_JS_STATUS_TESTS: Array<[string[], string]> = [
@ -392,3 +400,60 @@ test("withTimeout doesn't call callback if promise resolves", async (t) => {
t.deepEqual(shortTaskTimedOut, false);
t.deepEqual(result, 99);
});
function createMockSarifWithNotification(
locations: util.SarifLocation[]
): util.SarifFile {
return {
runs: [
{
tool: {
driver: {
name: "CodeQL",
},
},
invocations: [
{
toolExecutionNotifications: [
{
locations,
},
],
},
],
},
],
};
}
const stubLocation: util.SarifLocation = {
physicalLocation: {
artifactLocation: {
uri: "file1",
},
},
};
test("fixInvalidNotifications leaves notifications with unique locations alone", (t) => {
const messages: LoggedMessage[] = [];
const result = util.fixInvalidNotifications(
createMockSarifWithNotification([stubLocation]),
getRecordingLogger(messages)
);
t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
t.is(messages.length, 0);
});
test("fixInvalidNotifications removes duplicate locations", (t) => {
const messages: LoggedMessage[] = [];
const result = util.fixInvalidNotifications(
createMockSarifWithNotification([stubLocation, stubLocation]),
getRecordingLogger(messages)
);
t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
t.is(messages.length, 1);
t.deepEqual(messages[0], {
type: "info",
message: "Removed 1 duplicate locations from SARIF notification objects.",
});
});

177
src/util.ts
View file

@ -10,7 +10,7 @@ import * as semver from "semver";
import { getApiClient, GitHubApiDetails } from "./api-client";
import * as apiCompatibility from "./api-compatibility.json";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import { CodeQL } from "./codeql";
import {
Config,
parsePacksSpecification,
@ -19,7 +19,11 @@ import {
import { Feature, FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { Logger } from "./logging";
import { CODEQL_ACTION_TEST_MODE } from "./shared-environment";
import {
CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX,
CODEQL_ACTION_TEST_MODE,
EnvVar,
} from "./shared-environment";
/**
* Specifies bundle versions that are known to be broken
@ -42,13 +46,6 @@ export const DEFAULT_DEBUG_ARTIFACT_NAME = "debug-artifacts";
*/
export const DEFAULT_DEBUG_DATABASE_NAME = "db";
/**
* Environment variable that is set to "true" when the CodeQL Action has invoked
* the Go autobuilder.
*/
export const DID_AUTOBUILD_GO_ENV_VAR_NAME =
"CODEQL_ACTION_DID_AUTOBUILD_GOLANG";
export interface SarifFile {
version?: string | null;
runs: SarifRun[];
@ -65,9 +62,14 @@ export interface SarifRun {
id?: string;
};
artifacts?: string[];
invocations?: SarifInvocation[];
results?: SarifResult[];
}
export interface SarifInvocation {
toolExecutionNotifications?: SarifNotification[];
}
export interface SarifResult {
ruleId?: string;
message?: {
@ -88,6 +90,18 @@ export interface SarifResult {
};
}
export interface SarifNotification {
locations?: SarifLocation[];
}
export interface SarifLocation {
physicalLocation?: {
artifactLocation?: {
uri?: string;
};
};
}
/**
* Get the extra options for the codeql commands.
*/
@ -312,10 +326,12 @@ export enum GitHubVariant {
DOTCOM,
GHES,
GHAE,
GHE_DOTCOM,
}
export type GitHubVersion =
| { type: GitHubVariant.DOTCOM }
| { type: GitHubVariant.GHAE }
| { type: GitHubVariant.GHE_DOTCOM }
| { type: GitHubVariant.GHES; version: string };
export async function getGitHubVersion(
@ -341,6 +357,10 @@ export async function getGitHubVersion(
return { type: GitHubVariant.GHAE };
}
if (response.headers[GITHUB_ENTERPRISE_VERSION_HEADER] === "ghe.com") {
return { type: GitHubVariant.GHE_DOTCOM };
}
const version = response.headers[GITHUB_ENTERPRISE_VERSION_HEADER] as string;
return { type: GitHubVariant.GHES, version };
}
@ -413,42 +433,6 @@ export function assertNever(value: never): never {
throw new ExhaustivityCheckingError(value);
}
/**
* Environment variables to be set by codeql-action and used by the
* CLI.
*/
export enum EnvVar {
/**
* Semver of the codeql-action as specified in package.json.
*/
VERSION = "CODEQL_ACTION_VERSION",
/**
* If set to a truthy value, then the codeql-action might combine SARIF
* output from several `interpret-results` runs for the same Language.
*/
FEATURE_SARIF_COMBINE = "CODEQL_ACTION_FEATURE_SARIF_COMBINE",
/**
* If set to the "true" string, then the codeql-action will upload SARIF,
* not the cli.
*/
FEATURE_WILL_UPLOAD = "CODEQL_ACTION_FEATURE_WILL_UPLOAD",
/**
* If set to the "true" string, then the codeql-action is using its
* own deprecated and non-standard way of scanning for multiple
* languages.
*/
FEATURE_MULTI_LANGUAGE = "CODEQL_ACTION_FEATURE_MULTI_LANGUAGE",
/**
* If set to the "true" string, then the codeql-action is using its
* own sandwiched workflow mechanism
*/
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
}
/**
* Set some initial environment variables that we can set even without
* knowing what version of CodeQL we're running.
@ -459,20 +443,6 @@ export function initializeEnvironment(version: string) {
core.exportVariable(EnvVar.FEATURE_WILL_UPLOAD, "true");
}
/**
* Enrich the environment variables with further flags that we cannot
* know the value of until we know what version of CodeQL we're running.
*/
export async function enrichEnvironment(codeql: CodeQL) {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "false");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "false");
} else {
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "true");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "true");
}
}
/**
* Get an environment parameter, but throw an error if it is not set.
*/
@ -829,3 +799,92 @@ export function parseMatrixInput(
}
return JSON.parse(matrixInput);
}
function removeDuplicateLocations(locations: SarifLocation[]): SarifLocation[] {
const newJsonLocations = new Set<string>();
return locations.filter((location) => {
const jsonLocation = JSON.stringify(location);
if (!newJsonLocations.has(jsonLocation)) {
newJsonLocations.add(jsonLocation);
return true;
}
return false;
});
}
export function fixInvalidNotifications(
sarif: SarifFile,
logger: Logger
): SarifFile {
if (process.env[CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`
);
return sarif;
}
if (!Array.isArray(sarif.runs)) {
return sarif;
}
// Ensure that the array of locations for each SARIF notification contains unique locations.
// This is a workaround for a bug in the CodeQL CLI that causes duplicate locations to be
// emitted in some cases.
let numDuplicateLocationsRemoved = 0;
const newSarif = {
...sarif,
runs: sarif.runs.map((run) => {
if (
run.tool?.driver?.name !== "CodeQL" ||
!Array.isArray(run.invocations)
) {
return run;
}
return {
...run,
invocations: run.invocations.map((invocation) => {
if (!Array.isArray(invocation.toolExecutionNotifications)) {
return invocation;
}
return {
...invocation,
toolExecutionNotifications:
invocation.toolExecutionNotifications.map((notification) => {
if (!Array.isArray(notification.locations)) {
return notification;
}
const newLocations = removeDuplicateLocations(
notification.locations
);
numDuplicateLocationsRemoved +=
notification.locations.length - newLocations.length;
return {
...notification,
locations: newLocations,
};
}),
};
}),
};
}),
};
if (numDuplicateLocationsRemoved > 0) {
logger.info(
`Removed ${numDuplicateLocationsRemoved} duplicate locations from SARIF notification ` +
"objects."
);
}
return newSarif;
}
export function fixInvalidNotificationsInFile(
inputPath: string,
outputPath: string,
logger: Logger
): void {
let sarif = JSON.parse(fs.readFileSync(inputPath, "utf8")) as SarifFile;
sarif = fixInvalidNotifications(sarif, logger);
fs.writeFileSync(outputPath, JSON.stringify(sarif));
}

17
src/workflow.test.ts
View file

@ -56,17 +56,6 @@ test("getWorkflowErrors() when on.push is a valid superset", (t) => {
t.deepEqual(...errorCodes(errors, []));
});
test("getWorkflowErrors() when on.push should not have a path", (t) => {
const errors = getWorkflowErrors({
on: {
push: { branches: ["main"], paths: ["test/*"] },
pull_request: { branches: ["main"] },
},
});
t.deepEqual(...errorCodes(errors, [WorkflowErrors.PathsSpecified]));
});
test("getWorkflowErrors() when on.push is a correct object", (t) => {
const errors = getWorkflowErrors({
on: { push: { branches: ["main"] }, pull_request: { branches: ["main"] } },
@ -317,7 +306,7 @@ test("formatWorkflowErrors() when there is one error", (t) => {
test("formatWorkflowErrors() when there are multiple errors", (t) => {
const message = formatWorkflowErrors([
WorkflowErrors.CheckoutWrongHead,
WorkflowErrors.PathsSpecified,
WorkflowErrors.MismatchedBranches,
]);
t.true(message.startsWith("2 issues were detected with this workflow:"));
});
@ -331,10 +320,10 @@ test("formatWorkflowCause() with no errors", (t) => {
test("formatWorkflowCause()", (t) => {
const message = formatWorkflowCause([
WorkflowErrors.CheckoutWrongHead,
WorkflowErrors.PathsSpecified,
WorkflowErrors.MismatchedBranches,
]);
t.deepEqual(message, "CheckoutWrongHead,PathsSpecified");
t.deepEqual(message, "CheckoutWrongHead,MismatchedBranches");
t.deepEqual(formatWorkflowCause([]), undefined);
});

33
src/workflow.ts
View file

@ -108,8 +108,6 @@ function toCodedErrors(errors: {
export const WorkflowErrors = toCodedErrors({
MismatchedBranches: `Please make sure that every branch in on.pull_request is also in on.push so that Code Scanning can compare pull requests against the state of the base branch.`,
MissingPushHook: `Please specify an on.push hook so that Code Scanning can compare pull requests against the state of the base branch.`,
PathsSpecified: `Using on.push.paths can prevent Code Scanning annotating new alerts in your pull requests.`,
PathsIgnoreSpecified: `Using on.push.paths-ignore can prevent Code Scanning annotating new alerts in your pull requests.`,
CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.`,
});
@ -162,19 +160,6 @@ export function getWorkflowErrors(doc: Workflow): CodedError[] {
if (!hasPush && hasPullRequest) {
missingPush = true;
}
if (hasPush && hasPullRequest) {
const paths = doc.on.push?.paths;
// if you specify paths or paths-ignore you can end up with commits that have no baseline
// if they didn't change any files
// currently we cannot go back through the history and find the most recent baseline
if (Array.isArray(paths) && paths.length > 0) {
errors.push(WorkflowErrors.PathsSpecified);
}
const pathsIgnore = doc.on.push?.["paths-ignore"];
if (Array.isArray(pathsIgnore) && pathsIgnore.length > 0) {
errors.push(WorkflowErrors.PathsIgnoreSpecified);
}
}
// if doc.on.pull_request is null that means 'all branches'
// if doc.on.pull_request is undefined that means 'off'
@ -426,22 +411,20 @@ export function getCategoryInputOrThrow(
*
* Typically you'll want to wrap this function in a try/catch block and handle the error.
*
* @returns the upload input
* @returns the user input to upload, or undefined if input was unspecified
* @throws an error if the upload input could not be determined
*/
export function getUploadInputOrThrow(
workflow: Workflow,
jobName: string,
matrixVars: { [key: string]: string } | undefined
): string {
return (
getInputOrThrow(
workflow,
jobName,
getAnalyzeActionName(),
"upload",
matrixVars
) || "true" // if unspecified, upload defaults to true
): string | undefined {
return getInputOrThrow(
workflow,
jobName,
getAnalyzeActionName(),
"upload",
matrixVars
);
}