robojerk/codeql-action

Merge branch 'main' into update-bundle/codeql-bundle-v2.19.1

Browse source
This commit is contained in:
Angela P Wen 2024-10-02 17:00:15 -07:00 • committed by GitHub
commit 868284ba01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5429 changed files with 2177310 additions and 72049 deletions
Download patch file Download diff file Expand all files Collapse all files

86
src/actions-util.ts
View file

@ -480,3 +480,89 @@ export const getFileType = async (filePath: string): Promise<string> => {
export function isSelfHostedRunner() {
return process.env.RUNNER_ENVIRONMENT === "self-hosted";
}
export function prettyPrintInvocation(cmd: string, args: string[]): string {
return [cmd, ...args].map((x) => (x.includes(" ") ? `'${x}'` : x)).join(" ");
}
/**
* An error from a tool invocation, with associated exit code, stderr, etc.
*/
export class CommandInvocationError extends Error {
constructor(
public cmd: string,
public args: string[],
public exitCode: number,
public stderr: string,
public stdout: string,
) {
const prettyCommand = prettyPrintInvocation(cmd, args);
const lastLine = ensureEndsInPeriod(
stderr.trim().split("\n").pop()?.trim() || "n/a",
);
super(
`Failed to run "${prettyCommand}". ` +
`Exit code was ${exitCode} and last log line was: ${lastLine} See the logs for more details.`,
);
}
}
export function ensureEndsInPeriod(text: string): string {
return text[text.length - 1] === "." ? text : `${text}.`;
}
/**
* A constant defining the maximum number of characters we will keep from
* the programs stderr for logging.
*
* This serves two purposes:
* 1. It avoids an OOM if a program fails in a way that results it
* printing many log lines.
* 2. It avoids us hitting the limit of how much data we can send in our
* status reports on GitHub.com.
*/
const MAX_STDERR_BUFFER_SIZE = 20000;
/**
* Runs a CLI tool.
*
* @returns Standard output produced by the tool.
* @throws A `CommandInvocationError` if the tool exits with a non-zero status code.
*/
export async function runTool(
cmd: string,
args: string[] = [],
opts: { stdin?: string; noStreamStdout?: boolean } = {},
): Promise<string> {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {
stdout: (data: Buffer) => {
stdout += data.toString("utf8");
if (!opts.noStreamStdout) {
process.stdout.write(data);
}
},
stderr: (data: Buffer) => {
let readStartIndex = 0;
// If the error is too large, then we only take the last MAX_STDERR_BUFFER_SIZE characters
if (data.length - MAX_STDERR_BUFFER_SIZE > 0) {
// Eg: if we have MAX_STDERR_BUFFER_SIZE the start index should be 2.
readStartIndex = data.length - MAX_STDERR_BUFFER_SIZE + 1;
}
stderr += data.toString("utf8", readStartIndex);
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
},
},
silent: true,
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
throw new CommandInvocationError(cmd, args, exitCode, stderr, stdout);
}
return stdout;
}

35
src/analyze-action-post.ts
View file

@ -5,21 +5,48 @@
*/
import * as core from "@actions/core";
import { getTemporaryDirectory } from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { getConfig } from "./config-utils";
import * as debugArtifacts from "./debug-artifacts";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { getActionsLogger, withGroup } from "./logging";
import { getErrorMessage } from "./util";
import { parseRepositoryNwo } from "./repository";
import {
checkGitHubVersionInRange,
getErrorMessage,
getRequiredEnvParam,
} from "./util";
async function runWrapper() {
try {
const logger = getActionsLogger();
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const features = new Features(
gitHubVersion,
repositoryNwo,
getTemporaryDirectory(),
logger,
);
// Upload SARIF artifacts if we determine that this is a first-party analysis run.
// For third-party runs, this artifact will be uploaded in the `upload-sarif-post` step.
if (process.env[EnvVar.INIT_ACTION_HAS_RUN] === "true") {
await withGroup("Uploading combined SARIF debug artifact", () =>
debugArtifacts.uploadCombinedSarifArtifacts(logger),
);
const config = await getConfig(getTemporaryDirectory(), logger);
if (config !== undefined) {
await withGroup("Uploading combined SARIF debug artifact", () =>
debugArtifacts.uploadCombinedSarifArtifacts(
logger,
config.gitHubVersion.type,
features,
),
);
}
}
} catch (error) {
core.setFailed(

39
src/cli-errors.ts
View file

@ -1,21 +1,20 @@
import {
CommandInvocationError,
ensureEndsInPeriod,
prettyPrintInvocation,
} from "./actions-util";
import { DocUrl } from "./doc-url";
import { ConfigurationError } from "./util";
/**
* A class of Error that we can classify as an error stemming from a CLI
* invocation, with associated exit code, stderr,etc.
* An error from a CodeQL CLI invocation, with associated exit code, stderr, etc.
*/
export class CommandInvocationError extends Error {
constructor(
cmd: string,
args: string[],
public exitCode: number,
public stderr: string,
public stdout: string,
) {
const prettyCommand = [cmd, ...args]
.map((x) => (x.includes(" ") ? `'${x}'` : x))
.join(" ");
export class CliError extends Error {
public readonly exitCode: number;
public readonly stderr: string;
constructor({ cmd, args, exitCode, stderr }: CommandInvocationError) {
const prettyCommand = prettyPrintInvocation(cmd, args);
const fatalErrors = extractFatalErrors(stderr);
const autobuildErrors = extractAutobuildErrors(stderr);
@ -42,6 +41,8 @@ export class CommandInvocationError extends Error {
}
super(message);
this.exitCode = exitCode;
this.stderr = stderr;
}
}
@ -115,10 +116,6 @@ function extractAutobuildErrors(error: string): string | undefined {
return errorLines.join("\n") || undefined;
}
function ensureEndsInPeriod(text: string): string {
return text[text.length - 1] === "." ? text : `${text}.`;
}
/** Error messages from the CLI that we consider configuration errors and handle specially. */
export enum CliConfigErrorCategory {
AutobuildError = "AutobuildError",
@ -282,7 +279,7 @@ export const cliErrorsConfig: Record<
* if not, return undefined.
*/
export function getCliConfigCategoryIfExists(
cliError: CommandInvocationError,
cliError: CliError,
): CliConfigErrorCategory | undefined {
for (const [category, configuration] of Object.entries(cliErrorsConfig)) {
if (
@ -308,11 +305,7 @@ export function getCliConfigCategoryIfExists(
* error message appended, if it exists in a known set of configuration errors. Otherwise,
* simply returns the original error.
*/
export function wrapCliConfigurationError(cliError: Error): Error {
if (!(cliError instanceof CommandInvocationError)) {
return cliError;
}
export function wrapCliConfigurationError(cliError: CliError): Error {
const cliConfigErrorCategory = getCliConfigCategoryIfExists(cliError);
if (cliConfigErrorCategory === undefined) {
return cliError;

4
src/codeql.test.ts
View file

@ -12,7 +12,7 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { GitHubApiDetails } from "./api-client";
import { CommandInvocationError } from "./cli-errors";
import { CliError } from "./cli-errors";
import * as codeql from "./codeql";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
@ -961,7 +961,7 @@ test("runTool recognizes fatal internal errors", async (t) => {
async () =>
await codeqlObject.databaseRunQueries(stubConfig.dbLocation, []),
{
instanceOf: CommandInvocationError,
instanceOf: CliError,
message: `Encountered a fatal error while running "codeql-for-testing database run-queries --expect-discarded-cache --min-disk-free=1024 -v --intra-layer-parallelism". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details:
com.semmle.util.exception.CatastrophicError: An error occurred while evaluating ControlFlowGraph::ControlFlow::Root.isRootOf/1#dispred#f610e6ed/2@86282cc8
Severe disk cache trouble (corruption or out of space) at /home/runner/work/_temp/codeql_databases/go/db-go/default/cache/pages/28/33.pack: Failed to write item to disk. See the logs for more details.`,

93
src/codeql.ts
View file

@ -7,15 +7,14 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import {
CommandInvocationError,
getActionVersion,
getOptionalInput,
isAnalyzingDefaultBranch,
runTool,
} from "./actions-util";
import * as api from "./api-client";
import {
CommandInvocationError,
wrapCliConfigurationError,
} from "./cli-errors";
import { CliError, wrapCliConfigurationError } from "./cli-errors";
import { type Config } from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
@ -544,7 +543,7 @@ export async function getCodeQLForCmd(
async getVersion() {
let result = util.getCachedCodeQlVersion();
if (result === undefined) {
const output = await runTool(cmd, ["version", "--format=json"]);
const output = await runCli(cmd, ["version", "--format=json"]);
try {
result = JSON.parse(output) as VersionInfo;
} catch {
@ -557,7 +556,7 @@ export async function getCodeQLForCmd(
return result;
},
async printVersion() {
await runTool(cmd, ["version", "--format=json"]);
await runCli(cmd, ["version", "--format=json"]);
},
async supportsFeature(feature: ToolsFeature) {
return isSupportedToolsFeature(await this.getVersion(), feature);
@ -627,7 +626,7 @@ export async function getCodeQLForCmd(
? "--force-overwrite"
: "--overwrite";
await runTool(
await runCli(
cmd,
[
"database",
@ -674,10 +673,10 @@ export async function getCodeQLForCmd(
// When `DYLD_INSERT_LIBRARIES` is set in the environment for a step,
// the Actions runtime introduces its own workaround for SIP
// (https://github.com/actions/runner/pull/416).
await runTool(autobuildCmd);
await runCli(autobuildCmd);
},
async extractScannedLanguage(config: Config, language: Language) {
await runTool(cmd, [
await runCli(cmd, [
"database",
"trace-command",
"--index-traceless-dbs",
@ -692,7 +691,7 @@ export async function getCodeQLForCmd(
applyAutobuildAzurePipelinesTimeoutFix();
}
try {
await runTool(cmd, [
await runCli(cmd, [
"database",
"trace-command",
"--use-build-mode",
@ -731,7 +730,7 @@ export async function getCodeQLForCmd(
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
];
await runTool(cmd, args);
await runCli(cmd, args);
},
async resolveLanguages() {
const codeqlArgs = [
@ -740,7 +739,7 @@ export async function getCodeQLForCmd(
"--format=json",
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveLanguagesOutput;
@ -759,7 +758,7 @@ export async function getCodeQLForCmd(
...(await getLanguageAliasingArguments(this)),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as BetterResolveLanguagesOutput;
@ -783,7 +782,7 @@ export async function getCodeQLForCmd(
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveQueriesOutput;
@ -805,7 +804,7 @@ export async function getCodeQLForCmd(
if (workingDir !== undefined) {
codeqlArgs.push("--working-dir", workingDir);
}
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveBuildEnvironmentOutput;
@ -839,7 +838,7 @@ export async function getCodeQLForCmd(
) {
codeqlArgs.push("--intra-layer-parallelism");
}
await runTool(cmd, codeqlArgs);
await runCli(cmd, codeqlArgs);
},
async databaseInterpretResults(
databasePath: string,
@ -911,7 +910,7 @@ export async function getCodeQLForCmd(
}
// Capture the stdout, which contains the analysis summary. Don't stream it to the Actions
// logs to avoid printing it twice.
return await runTool(cmd, codeqlArgs, {
return await runCli(cmd, codeqlArgs, {
noStreamStdout: true,
});
},
@ -922,7 +921,7 @@ export async function getCodeQLForCmd(
...getExtraOptionsFromEnv(["database", "print-baseline"]),
databasePath,
];
return await runTool(cmd, codeqlArgs);
return await runCli(cmd, codeqlArgs);
},
/**
@ -956,7 +955,7 @@ export async function getCodeQLForCmd(
...packs,
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
const parsedOutput: PackDownloadOutput = JSON.parse(output);
@ -994,7 +993,7 @@ export async function getCodeQLForCmd(
`${cacheCleanupFlag}=${cleanupLevel}`,
...getExtraOptionsFromEnv(["database", "cleanup"]),
];
await runTool(cmd, codeqlArgs);
await runCli(cmd, codeqlArgs);
},
async databaseBundle(
databasePath: string,
@ -1103,7 +1102,7 @@ export async function getCodeQLForCmd(
args.push("--sarif-merge-runs-from-equal-category");
}
await runTool(cmd, args);
await runCli(cmd, args);
},
};
// To ensure that status reports include the CodeQL CLI version wherever
@ -1216,53 +1215,19 @@ export function getExtraOptions(
return all.concat(specific);
}
/*
* A constant defining the maximum number of characters we will keep from
* the programs stderr for logging. This serves two purposes:
* (1) It avoids an OOM if a program fails in a way that results it
* printing many log lines.
* (2) It avoids us hitting the limit of how much data we can send in our
* status reports on GitHub.com.
*/
const maxErrorSize = 20_000;
async function runTool(
async function runCli(
cmd: string,
args: string[] = [],
opts: { stdin?: string; noStreamStdout?: boolean } = {},
) {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {
stdout: (data: Buffer) => {
stdout += data.toString("utf8");
if (!opts.noStreamStdout) {
process.stdout.write(data);
}
},
stderr: (data: Buffer) => {
let readStartIndex = 0;
// If the error is too large, then we only take the last 20,000 characters
if (data.length - maxErrorSize > 0) {
// Eg: if we have 20,000 the start index should be 2.
readStartIndex = data.length - maxErrorSize + 1;
}
stderr += data.toString("utf8", readStartIndex);
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
},
},
silent: true,
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
const e = new CommandInvocationError(cmd, args, exitCode, stderr, stdout);
throw wrapCliConfigurationError(e);
): Promise<string> {
try {
return await runTool(cmd, args, opts);
} catch (e) {
if (e instanceof CommandInvocationError) {
throw wrapCliConfigurationError(new CliError(e));
}
throw e;
}
return stdout;
}
/**

15
src/debug-artifacts.test.ts
View file

@ -1,6 +1,10 @@
import test from "ava";
import * as debugArtifacts from "./debug-artifacts";
import { Feature } from "./feature-flags";
import { getActionsLogger } from "./logging";
import { createFeatures } from "./testing-utils";
import { GitHubVariant } from "./util";
test("sanitizeArtifactName", (t) => {
t.deepEqual(
@ -20,7 +24,16 @@ test("sanitizeArtifactName", (t) => {
test("uploadDebugArtifacts", async (t) => {
// Test that no error is thrown if artifacts list is empty.
const logger = getActionsLogger();
const mockFeature = createFeatures([Feature.ArtifactV4Upgrade]);
await t.notThrowsAsync(
debugArtifacts.uploadDebugArtifacts([], "rootDir", "artifactName"),
debugArtifacts.uploadDebugArtifacts(
logger,
[],
"rootDir",
"artifactName",
GitHubVariant.DOTCOM,
mockFeature,
),
);
});

74
src/debug-artifacts.ts
View file

@ -2,6 +2,7 @@ import * as fs from "fs";
import * as path from "path";
import * as artifact from "@actions/artifact";
import * as artifactLegacy from "@actions/artifact-legacy";
import * as core from "@actions/core";
import AdmZip from "adm-zip";
import del from "del";
@ -11,6 +12,7 @@ import { dbIsFinalized } from "./analyze";
import { getCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { Logger, withGroup } from "./logging";
import {
@ -18,6 +20,7 @@ import {
doesDirectoryExist,
getCodeQLDatabasePath,
getErrorMessage,
GitHubVariant,
listFolder,
} from "./util";
@ -29,7 +32,11 @@ export function sanitizeArtifactName(name: string): string {
* Upload Actions SARIF artifacts for debugging when CODEQL_ACTION_DEBUG_COMBINED_SARIF
* environment variable is set
*/
export async function uploadCombinedSarifArtifacts(logger: Logger) {
export async function uploadCombinedSarifArtifacts(
logger: Logger,
gitHubVariant: GitHubVariant,
features: FeatureEnablement,
) {
const tempDir = getTemporaryDirectory();
// Upload Actions SARIF artifacts for debugging when environment variable is set
@ -58,9 +65,12 @@ export async function uploadCombinedSarifArtifacts(logger: Logger) {
try {
await uploadDebugArtifacts(
logger,
toUpload,
baseTempDir,
"combined-sarif-artifacts",
gitHubVariant,
features,
);
} catch (e) {
logger.warning(
@ -153,6 +163,7 @@ async function tryBundleDatabase(
export async function tryUploadAllAvailableDebugArtifacts(
config: Config,
logger: Logger,
features: FeatureEnablement,
) {
const filesToUpload: string[] = [];
try {
@ -211,9 +222,12 @@ export async function tryUploadAllAvailableDebugArtifacts(
try {
await withGroup("Uploading debug artifacts", async () =>
uploadDebugArtifacts(
logger,
filesToUpload,
config.dbLocation,
config.debugArtifactName,
config.gitHubVersion.type,
features,
),
);
} catch (e) {
@ -224,9 +238,12 @@ export async function tryUploadAllAvailableDebugArtifacts(
}
export async function uploadDebugArtifacts(
logger: Logger,
toUpload: string[],
rootDir: string,
artifactName: string,
ghVariant: GitHubVariant,
features: FeatureEnablement,
) {
if (toUpload.length === 0) {
return;
@ -246,16 +263,53 @@ export async function uploadDebugArtifacts(
}
}
await artifact.create().uploadArtifact(
sanitizeArtifactName(`${artifactName}${suffix}`),
toUpload.map((file) => path.normalize(file)),
path.normalize(rootDir),
{
continueOnError: true,
// ensure we don't keep the debug artifacts around for too long since they can be large.
retentionDays: 7,
},
const artifactUploader = await getArtifactUploaderClient(
logger,
ghVariant,
features,
);
try {
await artifactUploader.uploadArtifact(
sanitizeArtifactName(`${artifactName}${suffix}`),
toUpload.map((file) => path.normalize(file)),
path.normalize(rootDir),
{
// ensure we don't keep the debug artifacts around for too long since they can be large.
retentionDays: 7,
},
);
} catch (e) {
// A failure to upload debug artifacts should not fail the entire action.
core.warning(`Failed to upload debug artifacts: ${e}`);
}
}
// `@actions/artifact@v2` is not yet supported on GHES so the legacy version of the client will be used on GHES
// until it is supported. We also use the legacy version of the client if the feature flag is disabled.
// The feature flag is named `ArtifactV4Upgrade` to reduce customer confusion; customers are primarily affected by
// `actions/download-artifact`, whose upgrade to v4 must be accompanied by the `@actions/artifact@v2` upgrade.
export async function getArtifactUploaderClient(
logger: Logger,
ghVariant: GitHubVariant,
features: FeatureEnablement,
): Promise<artifact.ArtifactClient | artifactLegacy.ArtifactClient> {
if (ghVariant === GitHubVariant.GHES) {
logger.info(
"Debug artifacts can be consumed with `actions/download-artifact@v3` because the `v4` version is not yet compatible on GHES.",
);
return artifactLegacy.create();
} else if (!(await features.getValue(Feature.ArtifactV4Upgrade))) {
logger.info(
"Debug artifacts can be consumed with `actions/download-artifact@v3`. To use the `actions/download-artifact@v4`, set the `CODEQL_ACTION_ARTIFACT_V4_UPGRADE` environment variable to true.",
);
return artifactLegacy.create();
} else {
logger.info(
"Debug artifacts can be consumed with `actions/download-artifact@v4`.",
);
return new artifact.DefaultArtifactClient();
}
}
/**

6
src/feature-flags.ts
View file

@ -45,6 +45,7 @@ export interface FeatureEnablement {
* Legacy features should end with `_enabled`.
*/
export enum Feature {
ArtifactV4Upgrade = "artifact_v4_upgrade",
CleanupTrapCaches = "cleanup_trap_caches",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
DisableCsharpBuildless = "disable_csharp_buildless",
@ -86,6 +87,11 @@ export const featureConfig: Record<
toolsFeature?: ToolsFeature;
}
> = {
[Feature.ArtifactV4Upgrade]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ARTIFACT_V4_UPGRADE",
minimumVersion: undefined,
},
[Feature.CleanupTrapCaches]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",

3
src/init-action-post-helper.ts
View file

@ -161,6 +161,7 @@ export async function run(
uploadAllAvailableDebugArtifacts: (
config: Config,
logger: Logger,
features: FeatureEnablement,
) => Promise<void>,
printDebugLogs: (config: Config) => Promise<void>,
config: Config,
@ -210,7 +211,7 @@ export async function run(
logger.info(
"Debug mode is on. Uploading available database bundles and logs as Actions debugging artifacts...",
);
await uploadAllAvailableDebugArtifacts(config, logger);
await uploadAllAvailableDebugArtifacts(config, logger, features);
await printDebugLogs(config);
}

4
src/resolve-environment-action.ts
View file

@ -7,7 +7,7 @@ import {
getTemporaryDirectory,
} from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { CommandInvocationError } from "./cli-errors";
import { CliError } from "./cli-errors";
import { Config, getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
import { runResolveBuildEnvironment } from "./resolve-environment";
@ -69,7 +69,7 @@ async function run() {
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
if (error instanceof CommandInvocationError) {
if (error instanceof CliError) {
// If the CLI failed to run successfully for whatever reason,
// we just return an empty JSON object and proceed with the workflow.
core.setOutput(ENVIRONMENT_OUTPUT_NAME, {});

66
src/setup-codeql.ts
View file

@ -4,12 +4,11 @@ import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import del from "del";
import { default as deepEqual } from "fast-deep-equal";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { isRunningLocalAction } from "./actions-util";
import { CommandInvocationError, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
@ -24,7 +23,7 @@ import {
import { Logger } from "./logging";
import * as tar from "./tar";
import * as util from "./util";
import { isGoodVersion } from "./util";
import { cleanUpGlob, isGoodVersion } from "./util";
export enum ToolsSource {
Unknown = "UNKNOWN",
@ -497,6 +496,7 @@ export const downloadCodeQL = async function (
maybeBundleVersion: string | undefined,
maybeCliVersion: string | undefined,
apiDetails: api.GitHubApiDetails,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
logger: Logger,
): Promise<{
@ -549,17 +549,25 @@ export const downloadCodeQL = async function (
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
);
logger.debug("Extracting CodeQL bundle.");
const extractionStart = performance.now();
const extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
);
const extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.debug(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.debug(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
const bundleVersion =
maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
@ -700,6 +708,10 @@ export async function setupCodeQLBundle(
);
} catch (e) {
zstdFailureReason = util.getErrorMessage(e) || "unknown error";
if (e instanceof CommandInvocationError) {
zstdFailureReason += ` Full error: ${e.stderr}`;
logger.debug(`Invocation output the following to stderr: ${e.stderr}`);
}
logger.warning(
`Failed to set up CodeQL tools with zstd. Falling back to gzipped version. Error: ${util.getErrorMessage(
e,
@ -755,7 +767,12 @@ async function setupCodeQLBundleWithCompressionMethod(
const compressionMethod = tar.inferCompressionMethod(
source.codeqlTarPath,
);
codeqlFolder = await tar.extract(source.codeqlTarPath, compressionMethod);
codeqlFolder = await tar.extract(
source.codeqlTarPath,
compressionMethod,
zstdAvailability.version,
logger,
);
toolsSource = ToolsSource.Local;
break;
}
@ -770,6 +787,7 @@ async function setupCodeQLBundleWithCompressionMethod(
source.bundleVersion,
source.cliVersion,
apiDetails,
zstdAvailability.version,
tempDir,
logger,
);
@ -791,24 +809,6 @@ async function setupCodeQLBundleWithCompressionMethod(
};
}
async function cleanUpGlob(glob: string, name: string, logger: Logger) {
logger.debug(`Cleaning up ${name}.`);
try {
const deletedPaths = await del(glob, { force: true });
if (deletedPaths.length === 0) {
logger.warning(
`Failed to clean up ${name}: no files found matching ${glob}.`,
);
} else if (deletedPaths.length === 1) {
logger.debug(`Cleaned up ${name}.`);
} else {
logger.debug(`Cleaned up ${name} (${deletedPaths.length} files).`);
}
} catch (e) {
logger.warning(`Failed to clean up ${name}: ${e}.`);
}
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),

58
src/start-proxy-action-post.ts
View file

@ -3,12 +3,20 @@
* It will run after the all steps in this job, in reverse order in relation to
* other `post:` hooks.
*/
import * as artifact from "@actions/artifact";
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { getGitHubVersion } from "./api-client";
import * as configUtils from "./config-utils";
import { getErrorMessage } from "./util";
import { getArtifactUploaderClient } from "./debug-artifacts";
import { Features } from "./feature-flags";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
checkGitHubVersionInRange,
getErrorMessage,
getRequiredEnvParam,
} from "./util";
async function runWrapper() {
try {
@ -31,18 +39,42 @@ async function runWrapper() {
core.info(
"Debug mode is on. Uploading proxy log as Actions debugging artifact...",
);
if (config?.gitHubVersion.type === undefined) {
core.warning(
`Did not upload debug artifacts because cannot determine the GitHub variant running.`,
);
return;
}
const logger = getActionsLogger();
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const features = new Features(
gitHubVersion,
repositoryNwo,
actionsUtil.getTemporaryDirectory(),
logger,
);
try {
await artifact
.create()
.uploadArtifact(
"proxy-log-file",
[logFilePath],
actionsUtil.getTemporaryDirectory(),
{
continueOnError: true,
retentionDays: 7,
},
);
const artifactUploader = await getArtifactUploaderClient(
logger,
gitHubVersion.type,
features,
);
await artifactUploader.uploadArtifact(
"proxy-log-file",
[logFilePath],
actionsUtil.getTemporaryDirectory(),
{
// ensure we don't keep the debug artifacts around for too long since they can be large.
retentionDays: 7,
},
);
} catch (e) {
// A failure to upload debug artifacts should not fail the entire action.
core.warning(`Failed to upload debug artifacts: ${e}`);

87
src/tar.ts
View file

@ -1,9 +1,14 @@
import * as fs from "fs";
import path from "path";
import { ToolRunner } from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
import { safeWhich } from "@chrisgavin/safe-which";
import { v4 as uuidV4 } from "uuid";
import { getTemporaryDirectory, runTool } from "./actions-util";
import { Logger } from "./logging";
import { assertNever } from "./util";
import { assertNever, cleanUpGlob } from "./util";
const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
@ -84,24 +89,84 @@ export async function isZstdAvailable(
export type CompressionMethod = "gzip" | "zstd";
export async function extract(
path: string,
tarPath: string,
compressionMethod: CompressionMethod,
tarVersion: TarVersion | undefined,
logger: Logger,
): Promise<string> {
switch (compressionMethod) {
case "gzip":
// While we could also ask tar to autodetect the compression method,
// we defensively keep the gzip call identical as requesting a gzipped
// bundle will soon be a fallback option.
return await toolcache.extractTar(path);
// Defensively continue to call the toolcache API as requesting a gzipped
// bundle may be a fallback option.
return await toolcache.extractTar(tarPath);
case "zstd":
// By specifying only the "x" flag, we ask tar to autodetect the
// compression method.
return await toolcache.extractTar(path, undefined, "x");
if (!tarVersion) {
throw new Error(
"Could not determine tar version, which is required to extract a Zstandard archive.",
);
}
return await extractTarZst(tarPath, tarVersion, logger);
}
}
export function inferCompressionMethod(path: string): CompressionMethod {
if (path.endsWith(".tar.gz")) {
/**
* Extract a compressed tar archive
*
* @param file path to the tar
* @param dest destination directory. Optional.
* @returns path to the destination directory
*/
export async function extractTarZst(
file: string,
tarVersion: TarVersion,
logger: Logger,
): Promise<string> {
if (!file) {
throw new Error("parameter 'file' is required");
}
// Create dest
const dest = await createExtractFolder();
try {
// Initialize args
const args = ["-x", "-v"];
let destArg = dest;
let fileArg = file;
if (process.platform === "win32" && tarVersion.type === "gnu") {
args.push("--force-local");
destArg = dest.replace(/\\/g, "/");
// Technically only the dest needs to have `/` but for aesthetic consistency
// convert slashes in the file arg too.
fileArg = file.replace(/\\/g, "/");
}
if (tarVersion.type === "gnu") {
// Suppress warnings when using GNU tar to extract archives created by BSD tar
args.push("--warning=no-unknown-keyword");
args.push("--overwrite");
}
args.push("-C", destArg, "-f", fileArg);
await runTool(`tar`, args);
} catch (e) {
await cleanUpGlob(dest, "extraction destination directory", logger);
throw e;
}
return dest;
}
async function createExtractFolder(): Promise<string> {
const dest = path.join(getTemporaryDirectory(), uuidV4());
fs.mkdirSync(dest, { recursive: true });
return dest;
}
export function inferCompressionMethod(tarPath: string): CompressionMethod {
if (tarPath.endsWith(".tar.gz")) {
return "gzip";
}
return "zstd";

34
src/upload-sarif-action-post.ts
View file

@ -5,19 +5,49 @@
*/
import * as core from "@actions/core";
import { getTemporaryDirectory } from "./actions-util";
import { getGitHubVersion } from "./api-client";
import * as debugArtifacts from "./debug-artifacts";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { getActionsLogger, withGroup } from "./logging";
import { getErrorMessage } from "./util";
import { parseRepositoryNwo } from "./repository";
import {
checkGitHubVersionInRange,
getErrorMessage,
getRequiredEnvParam,
} from "./util";
async function runWrapper() {
try {
const logger = getActionsLogger();
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const features = new Features(
gitHubVersion,
repositoryNwo,
getTemporaryDirectory(),
logger,
);
// Upload SARIF artifacts if we determine that this is a third-party analysis run.
// For first-party runs, this artifact will be uploaded in the `analyze-post` step.
if (process.env[EnvVar.INIT_ACTION_HAS_RUN] !== "true") {
if (gitHubVersion.type === undefined) {
core.warning(
`Did not upload debug artifacts because cannot determine the GitHub variant running.`,
);
return;
}
await withGroup("Uploading combined SARIF debug artifact", () =>
debugArtifacts.uploadCombinedSarifArtifacts(logger),
debugArtifacts.uploadCombinedSarifArtifacts(
logger,
gitHubVersion.type,
features,
),
);
}
} catch (error) {

18
src/util.ts
View file

@ -1161,3 +1161,21 @@ export async function checkSipEnablement(
return undefined;
}
}
export async function cleanUpGlob(glob: string, name: string, logger: Logger) {
logger.debug(`Cleaning up ${name}.`);
try {
const deletedPaths = await del(glob, { force: true });
if (deletedPaths.length === 0) {
logger.warning(
`Failed to clean up ${name}: no files found matching ${glob}.`,
);
} else if (deletedPaths.length === 1) {
logger.debug(`Cleaned up ${name}.`);
} else {
logger.debug(`Cleaned up ${name} (${deletedPaths.length} files).`);
}
} catch (e) {
logger.warning(`Failed to clean up ${name}: ${e}.`);
}
}