robojerk/codeql-action

Merge branch 'main' into dbartol/remove-actions-extractor

Browse source
This commit is contained in:
Andrew Eisenberg 2025-04-02 11:14:54 -07:00
commit 4a19b5125b
6279 changed files with 434033 additions and 624185 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/analyze-action-post.ts
View file

@ -3,6 +3,8 @@
* It will run after the all steps in this job, in reverse order in relation to
* other `post:` hooks.
*/
import * as fs from "fs";
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
@ -10,6 +12,7 @@ import { getGitHubVersion } from "./api-client";
import { getCodeQL } from "./codeql";
import { getConfig } from "./config-utils";
import * as debugArtifacts from "./debug-artifacts";
import { getJavaTempDependencyDir } from "./dependency-caching";
import { EnvVar } from "./environment";
import { getActionsLogger } from "./logging";
import { checkGitHubVersionInRange, getErrorMessage } from "./util";
@ -38,6 +41,20 @@ async function runWrapper() {
);
}
}
// If we analysed Java in build-mode: none, we may have downloaded dependencies
// to the temp directory. Clean these up so they don't persist unnecessarily
// long on self-hosted runners.
const javaTempDependencyDir = getJavaTempDependencyDir();
if (fs.existsSync(javaTempDependencyDir)) {
try {
fs.rmSync(javaTempDependencyDir, { recursive: true });
} catch (error) {
logger.info(
`Failed to remove temporary Java dependencies directory: ${getErrorMessage(error)}`,
);
}
}
} catch (error) {
core.setFailed(
`analyze post-action step failed: ${getErrorMessage(error)}`,

40
src/analyze-action.ts
View file

@ -3,7 +3,6 @@ import path from "path";
import { performance } from "perf_hooks";
import * as core from "@actions/core";
import * as github from "@actions/github";
import * as actionsUtil from "./actions-util";
import {
@ -23,11 +22,12 @@ import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { uploadDependencyCaches } from "./dependency-caching";
import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { getRepositoryNwo } from "./repository";
import * as statusReport from "./status-report";
import {
ActionName,
@ -252,9 +252,7 @@ async function run() {
logger,
);
const repositoryNwo = parseRepositoryNwo(
util.getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
const gitHubVersion = await getGitHubVersion();
@ -272,16 +270,14 @@ async function run() {
logger,
);
const pull_request = github.context.payload.pull_request;
const diffRangePackDir =
pull_request &&
(await setupDiffInformedQueryRun(
pull_request.base.ref as string,
pull_request.head.label as string,
codeql,
logger,
features,
));
const branches = await getDiffInformedAnalysisBranches(
codeql,
features,
logger,
);
const diffRangePackDir = branches
? await setupDiffInformedQueryRun(branches, logger)
: undefined;
await warnIfGoInstalledAfterInit(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
@ -295,12 +291,16 @@ async function run() {
logger,
);
const cleanupLevel =
actionsUtil.getOptionalInput("cleanup-level") || "brutal";
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await runQueries(
outputDir,
memory,
util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
threads,
cleanupLevel,
diffRangePackDir,
actionsUtil.getOptionalInput("category"),
config,
@ -309,12 +309,8 @@ async function run() {
);
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await runCleanup(
config,
actionsUtil.getOptionalInput("cleanup-level") || "brutal",
logger,
);
if (cleanupLevel !== "none") {
await runCleanup(config, cleanupLevel, logger);
}
const dbLocations: { [lang: string]: string } = {};
@ -365,7 +361,7 @@ async function run() {
actionsUtil.getRequiredInput("wait-for-processing") === "true"
) {
await uploadLib.waitForProcessing(
parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY")),
getRepositoryNwo(),
uploadResult.sarifID,
getActionsLogger(),
);

1
src/analyze.test.ts
View file

@ -101,6 +101,7 @@ test("status report fields", async (t) => {
memoryFlag,
addSnippetsFlag,
threadsFlag,
"brutal",
undefined,
undefined,
config,

89
src/analyze.ts
View file

@ -11,15 +11,18 @@ import { getApiClient } from "./api-client";
import { setupCppAutobuild } from "./autobuild";
import { CodeQL, getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { getJavaTempDependencyDir } from "./dependency-caching";
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import {
DiffThunkRange,
PullRequestBranches,
writeDiffRangesJsonFile,
} from "./diff-filtering-utils";
} from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { FeatureEnablement, Feature } from "./feature-flags";
import { isScannedLanguage, Language } from "./languages";
import { Logger, withGroupAsync } from "./logging";
import { getRepositoryNwoFromEnv } from "./repository";
import { DatabaseCreationTimings, EventReport } from "./status-report";
import { ToolsFeature } from "./tools-features";
import { endTracingForCluster } from "./tracer-config";
@ -166,6 +169,16 @@ export async function runExtraction(
) {
await setupCppAutobuild(codeql, logger);
}
// The Java `build-mode: none` extractor places dependencies (.jar files) in the
// database scratch directory by default. For dependency caching purposes, we want
// a stable path that caches can be restored into and that we can cache at the
// end of the workflow (i.e. that does not get removed when the scratch directory is).
if (language === Language.java && config.buildMode === BuildMode.None) {
process.env["CODEQL_EXTRACTOR_JAVA_OPTION_BUILDLESS_DEPENDENCY_DIR"] =
getJavaTempDependencyDir();
}
await codeql.extractUsingBuildMode(config, language);
} else {
await codeql.extractScannedLanguage(config, language);
@ -245,33 +258,20 @@ async function finalizeDatabaseCreation(
/**
* Set up the diff-informed analysis feature.
*
* @param baseRef The base branch name, used for calculating the diff range.
* @param headLabel The label that uniquely identifies the head branch across
* repositories, used for calculating the diff range.
* @param codeql
* @param logger
* @param features
* @returns Absolute path to the directory containing the extension pack for
* the diff range information, or `undefined` if the feature is disabled.
*/
export async function setupDiffInformedQueryRun(
baseRef: string,
headLabel: string,
codeql: CodeQL,
branches: PullRequestBranches,
logger: Logger,
features: FeatureEnablement,
): Promise<string | undefined> {
if (!(await features.getValue(Feature.DiffInformedQueries, codeql))) {
return undefined;
}
return await withGroupAsync(
"Generating diff range extension pack",
async () => {
const diffRanges = await getPullRequestEditedDiffRanges(
baseRef,
headLabel,
logger,
logger.info(
`Calculating diff ranges for ${branches.base}...${branches.head}`,
);
const diffRanges = await getPullRequestEditedDiffRanges(branches, logger);
const packDir = writeDiffRangeDataExtensionPack(logger, diffRanges);
if (packDir === undefined) {
logger.warning(
@ -291,9 +291,7 @@ export async function setupDiffInformedQueryRun(
/**
* Return the file line ranges that were added or modified in the pull request.
*
* @param baseRef The base branch name, used for calculating the diff range.
* @param headLabel The label that uniquely identifies the head branch across
* repositories, used for calculating the diff range.
* @param branches The base and head branches of the pull request.
* @param logger
* @returns An array of tuples, where each tuple contains the absolute path of a
* file, the start line and the end line (both 1-based and inclusive) of an
@ -301,11 +299,10 @@ export async function setupDiffInformedQueryRun(
* not triggered by a pull request or if there was an error.
*/
async function getPullRequestEditedDiffRanges(
baseRef: string,
headLabel: string,
branches: PullRequestBranches,
logger: Logger,
): Promise<DiffThunkRange[] | undefined> {
const fileDiffs = await getFileDiffsWithBasehead(baseRef, headLabel, logger);
const fileDiffs = await getFileDiffsWithBasehead(branches, logger);
if (fileDiffs === undefined) {
return undefined;
}
@ -344,19 +341,21 @@ interface FileDiff {
}
async function getFileDiffsWithBasehead(
baseRef: string,
headLabel: string,
branches: PullRequestBranches,
logger: Logger,
): Promise<FileDiff[] | undefined> {
const ownerRepo = util.getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
const owner = ownerRepo[0];
const repo = ownerRepo[1];
const basehead = `${baseRef}...${headLabel}`;
// Check CODE_SCANNING_REPOSITORY first. If it is empty or not set, fall back
// to GITHUB_REPOSITORY.
const repositoryNwo = getRepositoryNwoFromEnv(
"CODE_SCANNING_REPOSITORY",
"GITHUB_REPOSITORY",
);
const basehead = `${branches.base}...${branches.head}`;
try {
const response = await getApiClient().rest.repos.compareCommitsWithBasehead(
{
owner,
repo,
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
basehead,
per_page: 1,
},
@ -486,6 +485,15 @@ function writeDiffRangeDataExtensionPack(
return undefined;
}
if (ranges.length === 0) {
// An empty diff range means that there are no added or modified lines in
// the pull request. But the `restrictAlertsTo` extensible predicate
// interprets an empty data extension differently, as an indication that
// all alerts should be included. So we need to specifically set the diff
// range to a non-empty list that cannot match any alert location.
ranges = [{ path: "", startLine: 0, endLine: 0 }];
}
const diffRangeDir = path.join(
actionsUtil.getTemporaryDirectory(),
"pr-diff-range",
@ -548,6 +556,7 @@ export async function runQueries(
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string,
cleanupLevel: string,
diffRangePackDir: string | undefined,
automationDetailsId: string | undefined,
config: configUtils.Config,
@ -555,20 +564,22 @@ export async function runQueries(
features: FeatureEnablement,
): Promise<QueriesStatusReport> {
const statusReport: QueriesStatusReport = {};
const queryFlags = [memoryFlag, threadsFlag];
if (cleanupLevel !== "overlay") {
queryFlags.push("--expect-discarded-cache");
}
statusReport.analysis_is_diff_informed = diffRangePackDir !== undefined;
const dataExtensionFlags = diffRangePackDir
? [
`--additional-packs=${diffRangePackDir}`,
"--extension-packs=codeql-action/pr-diff-range",
]
: [];
if (diffRangePackDir) {
queryFlags.push(`--additional-packs=${diffRangePackDir}`);
queryFlags.push("--extension-packs=codeql-action/pr-diff-range");
}
const sarifRunPropertyFlag = diffRangePackDir
? "--sarif-run-property=incrementalMode=diff-informed"
: undefined;
const codeql = await getCodeQL(config.codeQLCmd);
const queryFlags = [memoryFlag, threadsFlag, ...dataExtensionFlags];
for (const language of config.languages) {
try {

46
src/api-client.test.ts
View file

@ -104,3 +104,49 @@ test("getGitHubVersion for GHE_DOTCOM", async (t) => {
});
t.deepEqual({ type: util.GitHubVariant.GHE_DOTCOM }, gheDotcom);
});
test("wrapApiConfigurationError correctly wraps specific configuration errors", (t) => {
// We don't reclassify arbitrary errors
const arbitraryError = new Error("arbitrary error");
let res = api.wrapApiConfigurationError(arbitraryError);
t.is(res, arbitraryError);
// Same goes for arbitrary errors
const configError = new util.ConfigurationError("arbitrary error");
res = api.wrapApiConfigurationError(configError);
t.is(res, configError);
// If an HTTP error doesn't contain a specific error message, we don't
// wrap is an an API error.
const httpError = new util.HTTPError("arbitrary HTTP error", 456);
res = api.wrapApiConfigurationError(httpError);
t.is(res, httpError);
// For other HTTP errors, we wrap them as Configuration errors if they contain
// specific error messages.
const httpNotFoundError = new util.HTTPError("commit not found", 404);
res = api.wrapApiConfigurationError(httpNotFoundError);
t.deepEqual(res, new util.ConfigurationError("commit not found"));
const refNotFoundError = new util.HTTPError(
"ref 'refs/heads/jitsi' not found in this repository - https://docs.github.com/rest",
404,
);
res = api.wrapApiConfigurationError(refNotFoundError);
t.deepEqual(
res,
new util.ConfigurationError(
"ref 'refs/heads/jitsi' not found in this repository - https://docs.github.com/rest",
),
);
const apiRateLimitError = new util.HTTPError(
"API rate limit exceeded for installation",
403,
);
res = api.wrapApiConfigurationError(apiRateLimitError);
t.deepEqual(
res,
new util.ConfigurationError("API rate limit exceeded for installation"),
);
});

22
src/api-client.ts
View file

@ -4,7 +4,7 @@ import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getActionVersion, getRequiredInput } from "./actions-util";
import { parseRepositoryNwo } from "./repository";
import { getRepositoryNwo } from "./repository";
import {
ConfigurationError,
getRequiredEnvParam,
@ -123,17 +123,15 @@ export async function getGitHubVersion(): Promise<GitHubVersion> {
* Get the path of the currently executing workflow relative to the repository root.
*/
export async function getWorkflowRelativePath(): Promise<string> {
const repo_nwo = getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
const owner = repo_nwo[0];
const repo = repo_nwo[1];
const repo_nwo = getRepositoryNwo();
const run_id = Number(getRequiredEnvParam("GITHUB_RUN_ID"));
const apiClient = getApiClient();
const runsResponse = await apiClient.request(
"GET /repos/:owner/:repo/actions/runs/:run_id?exclude_pull_requests=true",
{
owner,
repo,
owner: repo_nwo.owner,
repo: repo_nwo.repo,
run_id,
},
);
@ -218,9 +216,7 @@ export async function listActionsCaches(
key: string,
ref: string,
): Promise<ActionsCacheItem[]> {
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
return await getApiClient().paginate(
"GET /repos/{owner}/{repo}/actions/caches",
@ -235,9 +231,7 @@ export async function listActionsCaches(
/** Delete an Actions cache item by its ID. */
export async function deleteActionsCache(id: number) {
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
await getApiClient().rest.actions.deleteActionsCacheById({
owner: repositoryNwo.owner,
@ -249,9 +243,9 @@ export async function deleteActionsCache(id: number) {
export function wrapApiConfigurationError(e: unknown) {
if (isHTTPError(e)) {
if (
e.message.includes("API rate limit exceeded for site ID installation") ||
e.message.includes("API rate limit exceeded for installation") ||
e.message.includes("commit not found") ||
/^ref .* not found in this repository$/.test(e.message)
/ref .* not found in this repository/.test(e.message)
) {
return new ConfigurationError(e.message);
}

2
src/api-compatibility.json
View file

@ -1 +1 @@
{"maximumVersion": "3.16", "minimumVersion": "3.12"}
{"maximumVersion": "3.17", "minimumVersion": "3.12"}

8
src/autobuild.ts
View file

@ -9,9 +9,9 @@ import { EnvVar } from "./environment";
import { Feature, featureConfig, Features } from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { getRepositoryNwo } from "./repository";
import { ToolsFeature } from "./tools-features";
import { BuildMode, getRequiredEnvParam } from "./util";
import { BuildMode } from "./util";
export async function determineAutobuildLanguages(
codeql: CodeQL,
@ -117,9 +117,7 @@ export async function setupCppAutobuild(codeql: CodeQL, logger: Logger) {
const envVar = featureConfig[Feature.CppDependencyInstallation].envVar;
const featureName = "C++ automatic installation of dependencies";
const gitHubVersion = await getGitHubVersion();
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
const features = new Features(
gitHubVersion,
repositoryNwo,

4
src/cli-errors.ts
View file

@ -119,6 +119,7 @@ function extractAutobuildErrors(error: string): string | undefined {
/** Error messages from the CLI that we consider configuration errors and handle specially. */
export enum CliConfigErrorCategory {
AutobuildError = "AutobuildError",
CouldNotCreateTempDir = "CouldNotCreateTempDir",
ExternalRepositoryCloneFailed = "ExternalRepositoryCloneFailed",
GradleBuildFailed = "GradleBuildFailed",
IncompatibleWithActionVersion = "IncompatibleWithActionVersion",
@ -159,6 +160,9 @@ export const cliErrorsConfig: Record<
new RegExp("We were unable to automatically build your code"),
],
},
[CliConfigErrorCategory.CouldNotCreateTempDir]: {
cliErrorMessageCandidates: [new RegExp("Could not create temp directory")],
},
[CliConfigErrorCategory.ExternalRepositoryCloneFailed]: {
cliErrorMessageCandidates: [
new RegExp("Failed to clone external Git repository"),

7
src/codeql.test.ts
View file

@ -20,6 +20,7 @@ import { DocUrl } from "./doc-url";
import { FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { ToolsSource } from "./setup-codeql";
import {
setupTests,
@ -510,6 +511,7 @@ const injectedConfigMacro = test.macro({
"",
undefined,
undefined,
OverlayDatabaseMode.None,
getRunnerLogger(true),
);
@ -723,6 +725,7 @@ test("passes a code scanning config AND qlconfig to the CLI", async (t: Executio
"",
undefined,
"/path/to/qlconfig.yml",
OverlayDatabaseMode.None,
getRunnerLogger(true),
);
@ -752,6 +755,7 @@ test("does not pass a qlconfig to the CLI when it is undefined", async (t: Execu
"",
undefined,
undefined, // undefined qlconfigFile
OverlayDatabaseMode.None,
getRunnerLogger(true),
);
@ -957,7 +961,7 @@ test("runTool recognizes fatal internal errors", async (t) => {
await codeqlObject.databaseRunQueries(stubConfig.dbLocation, []),
{
instanceOf: CliError,
message: `Encountered a fatal error while running "codeql-for-testing database run-queries --expect-discarded-cache --intra-layer-parallelism --min-disk-free=1024 -v". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details:
message: `Encountered a fatal error while running "codeql-for-testing database run-queries --intra-layer-parallelism --min-disk-free=1024 -v". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details:
com.semmle.util.exception.CatastrophicError: An error occurred while evaluating ControlFlowGraph::ControlFlow::Root.isRootOf/1#dispred#f610e6ed/2@86282cc8
Severe disk cache trouble (corruption or out of space) at /home/runner/work/_temp/codeql_databases/go/db-go/default/cache/pages/28/33.pack: Failed to write item to disk. See the logs for more details.`,
},
@ -1005,6 +1009,7 @@ test("Avoids duplicating --overwrite flag if specified in CODEQL_ACTION_EXTRA_OP
"sourceRoot",
undefined,
undefined,
OverlayDatabaseMode.None,
getRunnerLogger(false),
);

44
src/codeql.ts
View file

@ -24,6 +24,11 @@ import {
import { isAnalyzingDefaultBranch } from "./git-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import {
OverlayDatabaseMode,
writeBaseDatabaseOidsFile,
writeOverlayChangesFile,
} from "./overlay-database-utils";
import * as setupCodeql from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
@ -82,6 +87,7 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
overlayDatabaseMode: OverlayDatabaseMode,
logger: Logger,
): Promise<void>;
/**
@ -377,7 +383,13 @@ export async function setupCodeQL(
zstdAvailability,
};
} catch (e) {
throw new Error(
const ErrorClass =
e instanceof util.ConfigurationError ||
(e instanceof Error && e.message.includes("ENOSPC")) // out of disk space
? util.ConfigurationError
: Error;
throw new ErrorClass(
`Unable to download and extract CodeQL CLI: ${getErrorMessage(e)}${
e instanceof Error && e.stack ? `\n\nDetails: ${e.stack}` : ""
}`,
@ -546,6 +558,7 @@ export async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
overlayDatabaseMode: OverlayDatabaseMode,
logger: Logger,
) {
const extraArgs = config.languages.map(
@ -586,12 +599,25 @@ export async function getCodeQLForCmd(
? "--force-overwrite"
: "--overwrite";
if (overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
const overlayChangesFile = await writeOverlayChangesFile(
config,
sourceRoot,
logger,
);
extraArgs.push(`--overlay-changes=${overlayChangesFile}`);
} else if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
extraArgs.push("--overlay-base");
}
await runCli(
cmd,
[
"database",
"init",
overwriteFlag,
...(overlayDatabaseMode === OverlayDatabaseMode.Overlay
? []
: [overwriteFlag]),
"--db-cluster",
config.dbLocation,
`--source-root=${sourceRoot}`,
@ -605,6 +631,10 @@ export async function getCodeQLForCmd(
],
{ stdin: externalRepositoryToken },
);
if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
await writeBaseDatabaseOidsFile(config, sourceRoot);
}
},
async runAutobuild(config: Config, language: Language) {
applyAutobuildAzurePipelinesTimeoutFix();
@ -785,7 +815,6 @@ export async function getCodeQLForCmd(
"run-queries",
...flags,
databasePath,
"--expect-discarded-cache",
"--intra-layer-parallelism",
"--min-disk-free=1024", // Try to leave at least 1GB free
"-v",
@ -1231,6 +1260,15 @@ async function generateCodeScanningConfig(
if (Array.isArray(augmentedConfig.packs) && !augmentedConfig.packs.length) {
delete augmentedConfig.packs;
}
augmentedConfig["query-filters"] = [
...(config.augmentationProperties.defaultQueryFilters || []),
...(augmentedConfig["query-filters"] || []),
];
if (augmentedConfig["query-filters"]?.length === 0) {
delete augmentedConfig["query-filters"];
}
logger.info(
`Writing augmented user configuration file to ${codeScanningConfigFile}`,
);

24
src/config-utils.test.ts
View file

@ -809,11 +809,15 @@ const calculateAugmentationMacro = test.macro({
languages: Language[],
expectedAugmentationProperties: configUtils.AugmentationProperties,
) => {
const actualAugmentationProperties = configUtils.calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages,
);
const actualAugmentationProperties =
await configUtils.calculateAugmentation(
getCachedCodeQL(),
createFeatures([]),
rawPacksInput,
rawQueriesInput,
languages,
mockLogger,
);
t.deepEqual(actualAugmentationProperties, expectedAugmentationProperties);
},
title: (_, title) => `Calculate Augmentation: ${title}`,
@ -830,6 +834,7 @@ test(
queriesInput: undefined,
packsInputCombines: false,
packsInput: undefined,
defaultQueryFilters: [],
} as configUtils.AugmentationProperties,
);
@ -844,6 +849,7 @@ test(
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
packsInputCombines: false,
packsInput: undefined,
defaultQueryFilters: [],
} as configUtils.AugmentationProperties,
);
@ -858,6 +864,7 @@ test(
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
packsInputCombines: false,
packsInput: undefined,
defaultQueryFilters: [],
} as configUtils.AugmentationProperties,
);
@ -872,6 +879,7 @@ test(
queriesInput: undefined,
packsInputCombines: false,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
defaultQueryFilters: [],
} as configUtils.AugmentationProperties,
);
@ -886,6 +894,7 @@ test(
queriesInput: undefined,
packsInputCombines: true,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
defaultQueryFilters: [],
} as configUtils.AugmentationProperties,
);
@ -898,12 +907,15 @@ const calculateAugmentationErrorMacro = test.macro({
languages: Language[],
expectedError: RegExp | string,
) => {
t.throws(
await t.throwsAsync(
() =>
configUtils.calculateAugmentation(
getCachedCodeQL(),
createFeatures([]),
rawPacksInput,
rawQueriesInput,
languages,
mockLogger,
),
{ message: expectedError },
);

34
src/config-utils.ts
View file

@ -8,6 +8,7 @@ import * as semver from "semver";
import * as api from "./api-client";
import { CachingKind, getCachingKind } from "./caching-utils";
import { CodeQL } from "./codeql";
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
@ -173,10 +174,16 @@ export interface AugmentationProperties {
* Whether or not the packs input combines with the packs in the config.
*/
packsInputCombines: boolean;
/**
* The packs input from the `with` block of the action declaration
*/
packsInput?: string[];
/**
* Default query filters to apply to the queries in the config.
*/
defaultQueryFilters?: QueryFilter[];
}
/**
@ -188,6 +195,7 @@ export const defaultAugmentationProperties: AugmentationProperties = {
packsInputCombines: false,
packsInput: undefined,
queriesInput: undefined,
defaultQueryFilters: [],
};
export type Packs = Partial<Record<Language, string[]>>;
@ -461,10 +469,13 @@ export async function getDefaultConfig({
logger,
);
const augmentationProperties = calculateAugmentation(
const augmentationProperties = await calculateAugmentation(
codeql,
features,
packsInput,
queriesInput,
languages,
logger,
);
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
@ -567,10 +578,13 @@ async function loadConfig({
logger,
);
const augmentationProperties = calculateAugmentation(
const augmentationProperties = await calculateAugmentation(
codeql,
features,
packsInput,
queriesInput,
languages,
logger,
);
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
@ -605,11 +619,14 @@ async function loadConfig({
* and the CLI does not know about these inputs so we need to inject them into
* the config file sent to the CLI.
*
* @param codeql The CodeQL object.
* @param features The feature enablement object.
* @param rawPacksInput The packs input from the action configuration.
* @param rawQueriesInput The queries input from the action configuration.
* @param languages The languages that the config file is for. If the packs input
* is non-empty, then there must be exactly one language. Otherwise, an
* error is thrown.
* @param logger The logger to use for logging.
*
* @returns The properties that need to be augmented in the config file.
*
@ -617,11 +634,14 @@ async function loadConfig({
* not have exactly one language.
*/
// exported for testing.
export function calculateAugmentation(
export async function calculateAugmentation(
codeql: CodeQL,
features: FeatureEnablement,
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
): AugmentationProperties {
logger: Logger,
): Promise<AugmentationProperties> {
const packsInputCombines = shouldCombine(rawPacksInput);
const packsInput = parsePacksFromInput(
rawPacksInput,
@ -634,11 +654,17 @@ export function calculateAugmentation(
queriesInputCombines,
);
const defaultQueryFilters: QueryFilter[] = [];
if (await shouldPerformDiffInformedAnalysis(codeql, features, logger)) {
defaultQueryFilters.push({ exclude: { tags: "exclude-from-incremental" } });
}
return {
packsInputCombines,
packsInput: packsInput?.[languages[0]],
queriesInput,
queriesInputCombines,
defaultQueryFilters,
};
}

8
src/defaults.json
View file

@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.20.5",
"cliVersion": "2.20.5",
"priorBundleVersion": "codeql-bundle-v2.20.4",
"priorCliVersion": "2.20.4"
"bundleVersion": "codeql-bundle-v2.20.7",
"cliVersion": "2.20.7",
"priorBundleVersion": "codeql-bundle-v2.20.6",
"priorCliVersion": "2.20.6"
}

12
src/dependency-caching.ts
View file

@ -4,6 +4,7 @@ import { join } from "path";
import * as actionsCache from "@actions/cache";
import * as glob from "@actions/glob";
import { getTemporaryDirectory } from "./actions-util";
import { getTotalCacheSize } from "./caching-utils";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
@ -28,6 +29,15 @@ interface CacheConfig {
const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies";
const CODEQL_DEPENDENCY_CACHE_VERSION = 1;
/**
* Returns a path to a directory intended to be used to store .jar files
* for the Java `build-mode: none` extractor.
* @returns The path to the directory that should be used by the `build-mode: none` extractor.
*/
export function getJavaTempDependencyDir(): string {
return join(getTemporaryDirectory(), "codeql_java", "repository");
}
/**
* Default caching configurations per language.
*/
@ -38,6 +48,8 @@ const CODEQL_DEFAULT_CACHE_CONFIG: { [language: string]: CacheConfig } = {
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
],
hash: [
// Maven

42
src/diff-filtering-utils.ts
View file

@ -1,42 +0,0 @@
import * as fs from "fs";
import * as path from "path";
import * as actionsUtil from "./actions-util";
import { Logger } from "./logging";
export interface DiffThunkRange {
path: string;
startLine: number;
endLine: number;
}
function getDiffRangesJsonFilePath(): string {
return path.join(actionsUtil.getTemporaryDirectory(), "pr-diff-range.json");
}
export function writeDiffRangesJsonFile(
logger: Logger,
ranges: DiffThunkRange[],
): void {
const jsonContents = JSON.stringify(ranges, null, 2);
const jsonFilePath = getDiffRangesJsonFilePath();
fs.writeFileSync(jsonFilePath, jsonContents);
logger.debug(
`Wrote pr-diff-range JSON file to ${jsonFilePath}:\n${jsonContents}`,
);
}
export function readDiffRangesJsonFile(
logger: Logger,
): DiffThunkRange[] | undefined {
const jsonFilePath = getDiffRangesJsonFilePath();
if (!fs.existsSync(jsonFilePath)) {
logger.debug(`Diff ranges JSON file does not exist at ${jsonFilePath}`);
return undefined;
}
const jsonContents = fs.readFileSync(jsonFilePath, "utf8");
logger.debug(
`Read pr-diff-range JSON file from ${jsonFilePath}:\n${jsonContents}`,
);
return JSON.parse(jsonContents) as DiffThunkRange[];
}

118
src/diff-informed-analysis-utils.ts Normal file
View file

@ -0,0 +1,118 @@
import * as fs from "fs";
import * as path from "path";
import * as github from "@actions/github";
import * as actionsUtil from "./actions-util";
import type { CodeQL } from "./codeql";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
export interface PullRequestBranches {
base: string;
head: string;
}
function getPullRequestBranches(): PullRequestBranches | undefined {
const pullRequest = github.context.payload.pull_request;
if (pullRequest) {
return {
base: pullRequest.base.ref,
// We use the head label instead of the head ref here, because the head
// ref lacks owner information and by itself does not uniquely identify
// the head branch (which may be in a forked repository).
head: pullRequest.head.label,
};
}
// PR analysis under Default Setup does not have the pull_request context,
// but it should set CODE_SCANNING_REF and CODE_SCANNING_BASE_BRANCH.
const codeScanningRef = process.env.CODE_SCANNING_REF;
const codeScanningBaseBranch = process.env.CODE_SCANNING_BASE_BRANCH;
if (codeScanningRef && codeScanningBaseBranch) {
return {
base: codeScanningBaseBranch,
// PR analysis under Default Setup analyzes the PR head commit instead of
// the merge commit, so we can use the provided ref directly.
head: codeScanningRef,
};
}
return undefined;
}
/**
* Check if the action should perform diff-informed analysis.
*/
export async function shouldPerformDiffInformedAnalysis(
codeql: CodeQL,
features: FeatureEnablement,
logger: Logger,
): Promise<boolean> {
return (
(await getDiffInformedAnalysisBranches(codeql, features, logger)) !==
undefined
);
}
/**
* Get the branches to use for diff-informed analysis.
*
* @returns If the action should perform diff-informed analysis, return
* the base and head branches that should be used to compute the diff ranges.
* Otherwise return `undefined`.
*/
export async function getDiffInformedAnalysisBranches(
codeql: CodeQL,
features: FeatureEnablement,
logger: Logger,
): Promise<PullRequestBranches | undefined> {
if (!(await features.getValue(Feature.DiffInformedQueries, codeql))) {
return undefined;
}
const branches = getPullRequestBranches();
if (!branches) {
logger.info(
"Not performing diff-informed analysis " +
"because we are not analyzing a pull request.",
);
}
return branches;
}
export interface DiffThunkRange {
path: string;
startLine: number;
endLine: number;
}
function getDiffRangesJsonFilePath(): string {
return path.join(actionsUtil.getTemporaryDirectory(), "pr-diff-range.json");
}
export function writeDiffRangesJsonFile(
logger: Logger,
ranges: DiffThunkRange[],
): void {
const jsonContents = JSON.stringify(ranges, null, 2);
const jsonFilePath = getDiffRangesJsonFilePath();
fs.writeFileSync(jsonFilePath, jsonContents);
logger.debug(
`Wrote pr-diff-range JSON file to ${jsonFilePath}:\n${jsonContents}`,
);
}
export function readDiffRangesJsonFile(
logger: Logger,
): DiffThunkRange[] | undefined {
const jsonFilePath = getDiffRangesJsonFilePath();
if (!fs.existsSync(jsonFilePath)) {
logger.debug(`Diff ranges JSON file does not exist at ${jsonFilePath}`);
return undefined;
}
const jsonContents = fs.readFileSync(jsonFilePath, "utf8");
logger.debug(
`Read pr-diff-range JSON file from ${jsonFilePath}:\n${jsonContents}`,
);
return JSON.parse(jsonContents) as DiffThunkRange[];
}

87
src/git-utils.test.ts
View file

@ -305,3 +305,90 @@ test("decodeGitFilePath quoted strings", async (t) => {
"\x07\b\f\n\r\t\v",
);
});
test("getFileOidsUnderPath returns correct file mapping", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
"d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map\n" +
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
);
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
t.deepEqual(runGitCommandStub.firstCall.args, [
"/fake/path",
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
]);
} finally {
runGitCommandStub.restore();
}
});
test("getFileOidsUnderPath handles quoted paths", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/normal-file.js\n" +
'd89514599a9a99f22b4085766d40af7b99974827_"lib/file with spaces.js"\n' +
'a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_"lib/file\\twith\\ttabs.js"',
);
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
} finally {
runGitCommandStub.restore();
}
});
test("getFileOidsUnderPath handles empty output", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves("");
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {});
} finally {
runGitCommandStub.restore();
}
});
test("getFileOidsUnderPath throws on unexpected output format", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
"invalid-line-format\n" +
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
);
try {
await t.throwsAsync(
async () => {
await gitUtils.getFileOidsUnderPath("/fake/path");
},
{
instanceOf: Error,
message: 'Unexpected "git ls-files" output: invalid-line-format',
},
);
} finally {
runGitCommandStub.restore();
}
});

140
src/git-utils.ts
View file

@ -9,8 +9,8 @@ import {
} from "./actions-util";
import { ConfigurationError, getRequiredEnvParam } from "./util";
async function runGitCommand(
checkoutPath: string | undefined,
export const runGitCommand = async function (
workingDirectory: string | undefined,
args: string[],
customErrorMessage: string,
): Promise<string> {
@ -28,7 +28,7 @@ async function runGitCommand(
stderr += data.toString();
},
},
cwd: checkoutPath,
cwd: workingDirectory,
}).exec();
return stdout;
} catch (error) {
@ -40,7 +40,7 @@ async function runGitCommand(
core.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
throw error;
}
}
};
/**
* Gets the SHA of the commit that is currently checked out.
@ -183,75 +183,6 @@ export const gitRepack = async function (flags: string[]) {
}
};
/**
* Compute the all merge bases between the given refs. Returns an empty array
* if no merge base is found, or if there is an error.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const getAllGitMergeBases = async function (
refs: string[],
): Promise<string[]> {
try {
const stdout = await runGitCommand(
getOptionalInput("checkout_path"),
["merge-base", "--all", ...refs],
`Cannot get merge base of ${refs}.`,
);
return stdout.trim().split("\n");
} catch {
return [];
}
};
/**
* Compute the diff hunk headers between the two given refs.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*
* @returns an array of diff hunk headers (one element per line), or undefined
* if the action was not triggered by a pull request, or if the diff could not
* be determined.
*/
export const getGitDiffHunkHeaders = async function (
fromRef: string,
toRef: string,
): Promise<string[] | undefined> {
let stdout = "";
try {
stdout = await runGitCommand(
getOptionalInput("checkout_path"),
[
"-c",
"core.quotePath=false",
"diff",
"--no-renames",
"--irreversible-delete",
"-U0",
fromRef,
toRef,
],
`Cannot get diff from ${fromRef} to ${toRef}.`,
);
} catch {
return undefined;
}
const headers: string[] = [];
for (const line of stdout.split("\n")) {
if (
line.startsWith("--- ") ||
line.startsWith("+++ ") ||
line.startsWith("@@ ")
) {
headers.push(line);
}
}
return headers;
};
/**
* Decode, if necessary, a file path produced by Git. See
* https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
@ -300,6 +231,69 @@ export const decodeGitFilePath = function (filePath: string): string {
return filePath;
};
/**
* Get the root of the Git repository.
*
* @param sourceRoot The source root of the code being analyzed.
* @returns The root of the Git repository.
*/
export const getGitRoot = async function (
sourceRoot: string,
): Promise<string | undefined> {
try {
const stdout = await runGitCommand(
sourceRoot,
["rev-parse", "--show-toplevel"],
`Cannot find Git repository root from the source root ${sourceRoot}.`,
);
return stdout.trim();
} catch {
// Errors are already logged by runGitCommand()
return undefined;
}
};
/**
* Returns the Git OIDs of all tracked files (in the index and in the working
* tree) that are under the given base path, including files in active
* submodules. Untracked files and files not under the given base path are
* ignored.
*
* @param basePath A path into the Git repository.
* @returns a map from file paths (relative to `basePath`) to Git OIDs.
* @throws {Error} if "git ls-tree" produces unexpected output.
*/
export const getFileOidsUnderPath = async function (
basePath: string,
): Promise<{ [key: string]: string }> {
// Without the --full-name flag, the path is relative to the current working
// directory of the git command, which is basePath.
const stdout = await runGitCommand(
basePath,
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
);
const fileOidMap: { [key: string]: string } = {};
// With --format=%(objectname)_%(path), the output is a list of lines like:
// 30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js
// d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map
const regex = /^([0-9a-f]{40})_(.+)$/;
for (const line of stdout.split("\n")) {
if (line) {
const match = line.match(regex);
if (match) {
const oid = match[1];
const path = decodeGitFilePath(match[2]);
fileOidMap[path] = oid;
} else {
throw new Error(`Unexpected "git ls-files" output: ${line}`);
}
}
}
return fileOidMap;
};
function getRefFromEnv(): string {
// To workaround a limitation of Actions dynamic workflows not setting
// the GITHUB_REF in some cases, we accept also the ref within the

6
src/init-action-post-helper.ts
View file

@ -10,7 +10,7 @@ import { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
import { RepositoryNwo, parseRepositoryNwo } from "./repository";
import { RepositoryNwo, getRepositoryNwo } from "./repository";
import { JobStatus } from "./status-report";
import * as uploadLib from "./upload-lib";
import {
@ -255,9 +255,7 @@ async function removeUploadedSarif(
const client = getApiClient();
try {
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
// Wait to make sure the analysis is ready for download before requesting it.
await delay(5000);

13
src/init-action-post.ts
View file

@ -17,7 +17,7 @@ import * as debugArtifacts from "./debug-artifacts";
import { Features } from "./feature-flags";
import * as initActionPostHelper from "./init-action-post-helper";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { getRepositoryNwo } from "./repository";
import {
StatusReportBase,
sendStatusReport,
@ -26,12 +26,7 @@ import {
ActionName,
getJobStatusDisplayName,
} from "./status-report";
import {
checkDiskUsage,
checkGitHubVersionInRange,
getRequiredEnvParam,
wrapError,
} from "./util";
import { checkDiskUsage, checkGitHubVersionInRange, wrapError } from "./util";
interface InitPostStatusReport
extends StatusReportBase,
@ -52,9 +47,7 @@ async function runWrapper() {
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
const features = new Features(
gitHubVersion,
repositoryNwo,

31
src/init-action.ts
View file

@ -36,13 +36,15 @@ import { Feature, featureConfig, Features } from "./feature-flags";
import {
checkInstallPython311,
cleanupDatabaseClusterDirectory,
getOverlayDatabaseMode,
initCodeQL,
initConfig,
runInit,
} from "./init";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { getRepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
import {
ActionName,
@ -279,9 +281,7 @@ async function run() {
checkGitHubVersionInRange(gitHubVersion, logger);
checkActionVersion(getActionVersion(), gitHubVersion);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
const features = new Features(
gitHubVersion,
@ -395,7 +395,22 @@ async function run() {
}
try {
cleanupDatabaseClusterDirectory(config, logger);
const sourceRoot = path.resolve(
getRequiredEnvParam("GITHUB_WORKSPACE"),
getOptionalInput("source-root") || "",
);
const overlayDatabaseMode = await getOverlayDatabaseMode(
(await codeql.getVersion()).version,
config,
sourceRoot,
logger,
);
logger.info(`Using overlay database mode: ${overlayDatabaseMode}`);
if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
cleanupDatabaseClusterDirectory(config, logger);
}
if (zstdAvailability) {
await recordZstdAvailability(config, zstdAvailability);
@ -675,11 +690,6 @@ async function run() {
}
}
const sourceRoot = path.resolve(
getRequiredEnvParam("GITHUB_WORKSPACE"),
getOptionalInput("source-root") || "",
);
const tracerConfig = await runInit(
codeql,
config,
@ -687,6 +697,7 @@ async function run() {
"Runner.Worker.exe",
getOptionalInput("registries"),
apiDetails,
overlayDatabaseMode,
logger,
);
if (tracerConfig !== undefined) {

49
src/init.ts
View file

@ -3,14 +3,20 @@ import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as io from "@actions/io";
import * as semver from "semver";
import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { getGitRoot } from "./git-utils";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import {
CODEQL_OVERLAY_MINIMUM_VERSION,
OverlayDatabaseMode,
} from "./overlay-database-utils";
import { ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
@ -79,6 +85,47 @@ export async function initConfig(
return config;
}
export async function getOverlayDatabaseMode(
codeqlVersion: string,
config: configUtils.Config,
sourceRoot: string,
logger: Logger,
): Promise<OverlayDatabaseMode> {
const overlayDatabaseMode = process.env.CODEQL_OVERLAY_DATABASE_MODE;
if (
overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
overlayDatabaseMode === OverlayDatabaseMode.OverlayBase
) {
if (config.buildMode !== util.BuildMode.None) {
logger.warning(
`Cannot build an ${overlayDatabaseMode} database because ` +
`build-mode is set to "${config.buildMode}" instead of "none". ` +
"Falling back to creating a normal full database instead.",
);
return OverlayDatabaseMode.None;
}
if (semver.lt(codeqlVersion, CODEQL_OVERLAY_MINIMUM_VERSION)) {
logger.warning(
`Cannot build an ${overlayDatabaseMode} database because ` +
`the CodeQL CLI is older than ${CODEQL_OVERLAY_MINIMUM_VERSION}. ` +
"Falling back to creating a normal full database instead.",
);
return OverlayDatabaseMode.None;
}
if ((await getGitRoot(sourceRoot)) === undefined) {
logger.warning(
`Cannot build an ${overlayDatabaseMode} database because ` +
`the source root "${sourceRoot}" is not inside a git repository. ` +
"Falling back to creating a normal full database instead.",
);
return OverlayDatabaseMode.None;
}
return overlayDatabaseMode as OverlayDatabaseMode;
}
return OverlayDatabaseMode.None;
}
export async function runInit(
codeql: CodeQL,
config: configUtils.Config,
@ -86,6 +133,7 @@ export async function runInit(
processName: string | undefined,
registriesInput: string | undefined,
apiDetails: GitHubApiCombinedDetails,
overlayDatabaseMode: OverlayDatabaseMode,
logger: Logger,
): Promise<TracerConfig | undefined> {
fs.mkdirSync(config.dbLocation, { recursive: true });
@ -109,6 +157,7 @@ export async function runInit(
sourceRoot,
processName,
qlconfigFile,
overlayDatabaseMode,
logger,
),
);

77
src/overlay-database-utils.test.ts Normal file
View file

@ -0,0 +1,77 @@
import * as fs from "fs";
import * as path from "path";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as gitUtils from "./git-utils";
import { getRunnerLogger } from "./logging";
import {
writeBaseDatabaseOidsFile,
writeOverlayChangesFile,
} from "./overlay-database-utils";
import { createTestConfig, setupTests } from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
test("writeOverlayChangesFile generates correct changes file", async (t) => {
await withTmpDir(async (tmpDir) => {
const dbLocation = path.join(tmpDir, "db");
await fs.promises.mkdir(dbLocation, { recursive: true });
const sourceRoot = path.join(tmpDir, "src");
await fs.promises.mkdir(sourceRoot, { recursive: true });
const tempDir = path.join(tmpDir, "temp");
await fs.promises.mkdir(tempDir, { recursive: true });
const logger = getRunnerLogger(true);
const config = createTestConfig({ dbLocation });
// Mock the getFileOidsUnderPath function to return base OIDs
const baseOids = {
"unchanged.js": "aaa111",
"modified.js": "bbb222",
"deleted.js": "ccc333",
};
const getFileOidsStubForBase = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(baseOids);
// Write the base database OIDs file
await writeBaseDatabaseOidsFile(config, sourceRoot);
getFileOidsStubForBase.restore();
// Mock the getFileOidsUnderPath function to return overlay OIDs
const currentOids = {
"unchanged.js": "aaa111",
"modified.js": "ddd444", // Changed OID
"added.js": "eee555", // New file
};
const getFileOidsStubForOverlay = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(currentOids);
// Write the overlay changes file, which uses the mocked overlay OIDs
// and the base database OIDs file
const getTempDirStub = sinon
.stub(actionsUtil, "getTemporaryDirectory")
.returns(tempDir);
const changesFilePath = await writeOverlayChangesFile(
config,
sourceRoot,
logger,
);
getFileOidsStubForOverlay.restore();
getTempDirStub.restore();
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
t.deepEqual(
parsedContent.changes.sort(),
["added.js", "deleted.js", "modified.js"],
"Should identify added, deleted, and modified files",
);
});
});

124
src/overlay-database-utils.ts Normal file
View file

@ -0,0 +1,124 @@
import * as fs from "fs";
import * as path from "path";
import { getTemporaryDirectory } from "./actions-util";
import { type Config } from "./config-utils";
import { getFileOidsUnderPath } from "./git-utils";
import { Logger } from "./logging";
export enum OverlayDatabaseMode {
Overlay = "overlay",
OverlayBase = "overlay-base",
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";
/**
* Writes a JSON file containing Git OIDs for all tracked files (represented
* by path relative to the source root) under the source root. The file is
* written into the database location specified in the config.
*
* @param config The configuration object containing the database location
* @param sourceRoot The root directory containing the source files to process
* @throws {Error} If the Git repository root cannot be determined
*/
export async function writeBaseDatabaseOidsFile(
config: Config,
sourceRoot: string,
): Promise<void> {
const gitFileOids = await getFileOidsUnderPath(sourceRoot);
const gitFileOidsJson = JSON.stringify(gitFileOids);
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
await fs.promises.writeFile(baseDatabaseOidsFilePath, gitFileOidsJson);
}
/**
* Reads and parses the JSON file containing the base database Git OIDs.
* This file contains the mapping of file paths to their corresponding Git OIDs
* that was previously written by writeBaseDatabaseOidsFile().
*
* @param config The configuration object containing the database location
* @param logger The logger instance to use for error reporting
* @returns An object mapping file paths (relative to source root) to their Git OIDs
* @throws {Error} If the file cannot be read or parsed
*/
async function readBaseDatabaseOidsFile(
config: Config,
logger: Logger,
): Promise<{ [key: string]: string }> {
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
try {
const contents = await fs.promises.readFile(
baseDatabaseOidsFilePath,
"utf-8",
);
return JSON.parse(contents) as { [key: string]: string };
} catch (e) {
logger.error(
"Failed to read overlay-base file OIDs from " +
`${baseDatabaseOidsFilePath}: ${(e as any).message || e}`,
);
throw e;
}
}
function getBaseDatabaseOidsFilePath(config: Config): string {
return path.join(config.dbLocation, "base-database-oids.json");
}
/**
* Writes a JSON file containing the source-root-relative paths of files under
* `sourceRoot` that have changed (added, removed, or modified) from the overlay
* base database.
*
* This function uses the Git index to determine which files have changed, so it
* requires the following preconditions, both when this function is called and
* when the overlay-base database was initialized:
*
* - It requires that `sourceRoot` is inside a Git repository.
* - It assumes that all changes in the working tree are staged in the index.
* - It assumes that all files of interest are tracked by Git, e.g. not covered
* by `.gitignore`.
*/
export async function writeOverlayChangesFile(
config: Config,
sourceRoot: string,
logger: Logger,
): Promise<string> {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`,
);
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path.join(
getTemporaryDirectory(),
"overlay-changes.json",
);
logger.debug(
`Writing overlay changed files to ${overlayChangesFile}: ${changedFilesJson}`,
);
await fs.promises.writeFile(overlayChangesFile, changedFilesJson);
return overlayChangesFile;
}
function computeChangedFiles(
baseFileOids: { [key: string]: string },
overlayFileOids: { [key: string]: string },
): string[] {
const changes: string[] = [];
for (const [file, oid] of Object.entries(overlayFileOids)) {
if (!(file in baseFileOids) || baseFileOids[file] !== oid) {
changes.push(file);
}
}
for (const file of Object.keys(baseFileOids)) {
if (!(file in overlayFileOids)) {
changes.push(file);
}
}
return changes;
}

32
src/repository.ts
View file

@ -1,4 +1,4 @@
import { ConfigurationError } from "./util";
import { ConfigurationError, getRequiredEnvParam } from "./util";
// A repository name with owner, parsed into its two parts
export interface RepositoryNwo {
@ -6,6 +6,36 @@ export interface RepositoryNwo {
repo: string;
}
/**
* Get the repository name with owner from the environment variable
* `GITHUB_REPOSITORY`.
*
* @returns The repository name with owner.
*/
export function getRepositoryNwo(): RepositoryNwo {
return getRepositoryNwoFromEnv("GITHUB_REPOSITORY");
}
/**
* Get the repository name with owner from the first environment variable that
* is set and non-empty.
*
* @param envVarNames The names of the environment variables to check.
* @returns The repository name with owner.
* @throws ConfigurationError if none of the environment variables are set.
*/
export function getRepositoryNwoFromEnv(
...envVarNames: string[]
): RepositoryNwo {
const envVarName = envVarNames.find((name) => process.env[name]);
if (!envVarName) {
throw new ConfigurationError(
`None of the env vars ${envVarNames.join(", ")} are set`,
);
}
return parseRepositoryNwo(getRequiredEnvParam(envVarName));
}
export function parseRepositoryNwo(input: string): RepositoryNwo {
const parts = input.split("/");
if (parts.length !== 2) {

61
src/status-report.test.ts
View file

@ -5,13 +5,17 @@ import * as actionsUtil from "./actions-util";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { ActionName, createStatusReportBase } from "./status-report";
import {
ActionName,
createStatusReportBase,
getActionsStatus,
} from "./status-report";
import {
setupTests,
setupActionsVars,
createTestConfig,
} from "./testing-utils";
import { BuildMode, withTmpDir } from "./util";
import { BuildMode, ConfigurationError, withTmpDir, wrapError } from "./util";
setupTests(test);
@ -186,3 +190,56 @@ test("createStatusReportBase_firstParty", async (t) => {
);
});
});
test("getActionStatus handling correctly various types of errors", (t) => {
t.is(
getActionsStatus(new Error("arbitrary error")),
"failure",
"We categorise an arbitrary error as a failure",
);
t.is(
getActionsStatus(new ConfigurationError("arbitrary error")),
"user-error",
"We categorise a ConfigurationError as a user error",
);
t.is(
getActionsStatus(new Error("exit code 1"), "multiple things went wrong"),
"failure",
"getActionsStatus should return failure if passed an arbitrary error and an additional failure cause",
);
t.is(
getActionsStatus(
new ConfigurationError("exit code 1"),
"multiple things went wrong",
),
"user-error",
"getActionsStatus should return failure if passed a configuration error and an additional failure cause",
);
t.is(
getActionsStatus(),
"success",
"getActionsStatus should return success if no error is passed",
);
t.is(
getActionsStatus(new Object()),
"failure",
"getActionsStatus should return failure if passed an arbitrary object",
);
t.is(
getActionsStatus(null, "an error occurred"),
"failure",
"getActionsStatus should return failure if passed null and an additional failure cause",
);
t.is(
getActionsStatus(wrapError(new ConfigurationError("arbitrary error"))),
"user-error",
"We still recognise a wrapped ConfigurationError as a user error",
);
});

8
src/status-report.ts
View file

@ -17,6 +17,7 @@ import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
import { getRef } from "./git-utils";
import { Logger } from "./logging";
import { getRepositoryNwo } from "./repository";
import {
ConfigurationError,
isHTTPError,
@ -393,16 +394,15 @@ export async function sendStatusReport<S extends StatusReportBase>(
return;
}
const nwo = getRequiredEnvParam("GITHUB_REPOSITORY");
const [owner, repo] = nwo.split("/");
const nwo = getRepositoryNwo();
const client = getApiClient();
try {
await client.request(
"PUT /repos/:owner/:repo/code-scanning/analysis/status",
{
owner,
repo,
owner: nwo.owner,
repo: nwo.repo,
data: statusReportJSON,
},
);

10
src/upload-lib.ts
View file

@ -14,14 +14,14 @@ import * as api from "./api-client";
import { getGitHubVersion, wrapApiConfigurationError } from "./api-client";
import { CodeQL, getCodeQL } from "./codeql";
import { getConfig } from "./config-utils";
import { readDiffRangesJsonFile } from "./diff-filtering-utils";
import { readDiffRangesJsonFile } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { FeatureEnablement } from "./feature-flags";
import * as fingerprints from "./fingerprints";
import * as gitUtils from "./git-utils";
import { initCodeQL } from "./init";
import { Logger } from "./logging";
import { parseRepositoryNwo, RepositoryNwo } from "./repository";
import { getRepositoryNwo, RepositoryNwo } from "./repository";
import { ToolsFeature } from "./tools-features";
import * as util from "./util";
import {
@ -624,11 +624,7 @@ export async function uploadFiles(
logger.debug(`Number of results in upload: ${numResultInSarif}`);
// Make the upload
const sarifID = await uploadPayload(
payload,
parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY")),
logger,
);
const sarifID = await uploadPayload(payload, getRepositoryNwo(), logger);
logger.endGroup();

9
src/upload-sarif-action.ts
View file

@ -5,7 +5,7 @@ import { getActionVersion, getTemporaryDirectory } from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { Features } from "./feature-flags";
import { Logger, getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { getRepositoryNwo } from "./repository";
import {
createStatusReportBase,
sendStatusReport,
@ -20,7 +20,6 @@ import {
checkActionVersion,
checkDiskUsage,
getErrorMessage,
getRequiredEnvParam,
initializeEnvironment,
isInTestMode,
wrapError,
@ -63,9 +62,7 @@ async function run() {
// Make inputs accessible in the `post` step.
actionsUtil.persistInputs();
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const repositoryNwo = getRepositoryNwo();
const features = new Features(
gitHubVersion,
repositoryNwo,
@ -100,7 +97,7 @@ async function run() {
core.debug("In test mode. Waiting for processing is disabled.");
} else if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
await upload_lib.waitForProcessing(
parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
getRepositoryNwo(),
uploadResult.sarifID,
logger,
);