robojerk/codeql-action

Merge branch 'main' into update-supported-enterprise-server-versions

Browse source
This commit is contained in:
Henry Mercer 2023-07-17 13:35:17 +01:00 • committed by GitHub
commit eb1ef12e40
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5444 changed files with 663378 additions and 474489 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/actions-util.test.ts
View file

@ -5,7 +5,7 @@ import test from "ava";
import * as sinon from "sinon";
import * as actionsutil from "./actions-util";
import * as sharedEnv from "./shared-environment";
import { EnvVar } from "./environment";
import { setupActionsVars, setupTests } from "./testing-utils";
import { initializeEnvironment, withTmpDir } from "./util";
@ -211,7 +211,7 @@ test("computeAutomationID()", async (t) => {
test("initializeEnvironment", (t) => {
initializeEnvironment("1.2.3");
t.deepEqual(process.env.CODEQL_ACTION_VERSION, "1.2.3");
t.deepEqual(process.env[EnvVar.VERSION], "1.2.3");
});
test("isAnalyzingDefaultBranch()", async (t) => {
@ -303,8 +303,7 @@ test("createStatusReportBase", async (t) => {
t.assert(statusReport.action_name === "init");
t.assert(statusReport.action_oid === "unknown");
t.assert(
statusReport.started_at ===
process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT]
statusReport.started_at === process.env[EnvVar.WORKFLOW_STARTED_AT]
);
t.assert(
statusReport.action_started_at ===

19
src/actions-util.ts
View file

@ -9,7 +9,7 @@ import { JSONSchemaForNPMPackageJsonFiles } from "@schemastore/package";
import * as api from "./api-client";
import { Config } from "./config-utils";
import * as sharedEnv from "./shared-environment";
import { EnvVar } from "./environment";
import {
doesDirectoryExist,
getCachedCodeQlVersion,
@ -443,32 +443,25 @@ export async function createStatusReportBase(
): Promise<StatusReportBase> {
const commitOid = getOptionalInput("sha") || process.env["GITHUB_SHA"] || "";
const ref = await getRef();
const jobRunUUID = process.env[sharedEnv.JOB_RUN_UUID] || "";
const jobRunUUID = process.env[EnvVar.JOB_RUN_UUID] || "";
const workflowRunID = getWorkflowRunID();
const workflowRunAttempt = getWorkflowRunAttempt();
const workflowName = process.env["GITHUB_WORKFLOW"] || "";
const jobName = process.env["GITHUB_JOB"] || "";
const analysis_key = await getAnalysisKey();
let workflowStartedAt = process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT];
let workflowStartedAt = process.env[EnvVar.WORKFLOW_STARTED_AT];
if (workflowStartedAt === undefined) {
workflowStartedAt = actionStartedAt.toISOString();
core.exportVariable(
sharedEnv.CODEQL_WORKFLOW_STARTED_AT,
workflowStartedAt
);
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
}
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const codeQlCliVersion = getCachedCodeQlVersion();
const actionRef = process.env["GITHUB_ACTION_REF"];
const testingEnvironment =
process.env[sharedEnv.CODEQL_ACTION_TESTING_ENVIRONMENT] || "";
const testingEnvironment = process.env[EnvVar.TESTING_ENVIRONMENT] || "";
// re-export the testing environment variable so that it is available to subsequent steps,
// even if it was only set for this step
if (testingEnvironment !== "") {
core.exportVariable(
sharedEnv.CODEQL_ACTION_TESTING_ENVIRONMENT,
testingEnvironment
);
core.exportVariable(EnvVar.TESTING_ENVIRONMENT, testingEnvironment);
}
const statusReport: StatusReportBase = {

20
src/analyze-action.ts
View file

@ -19,14 +19,11 @@ import { runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY,
CODEQL_ACTION_DID_AUTOBUILD_GOLANG,
} from "./shared-environment";
import { getTotalCacheSize, uploadTrapCaches } from "./trap-caching";
import * as uploadLib from "./upload-lib";
import { UploadResult } from "./upload-lib";
@ -144,7 +141,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
if (!config.languages.includes(Language.go)) {
return;
}
if (process.env[CODEQL_ACTION_DID_AUTOBUILD_GOLANG] === "true") {
if (process.env[EnvVar.DID_AUTOBUILD_GOLANG] === "true") {
logger.debug("Won't run Go autobuild since it has already been run.");
return;
}
@ -213,9 +210,6 @@ async function run() {
actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
logger
);
const memory = util.getMemoryFlag(
actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
);
const repositoryNwo = parseRepositoryNwo(
util.getRequiredEnvParam("GITHUB_REPOSITORY")
@ -230,6 +224,11 @@ async function run() {
logger
);
const memory = await util.getMemoryFlag(
actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"],
features
);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await runFinalize(
@ -308,10 +307,7 @@ async function run() {
`expect-error input was set to true but no error was thrown.`
);
}
core.exportVariable(
CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY,
"true"
);
core.exportVariable(EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY, "true");
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
if (

4
src/analyze.test.ts
View file

@ -175,6 +175,8 @@ test("status report fields and search path setting", async (t) => {
if (builtinStatusReport.event_reports) {
for (const eventReport of builtinStatusReport.event_reports) {
t.deepEqual(eventReport.event, "codeql database interpret-results");
t.true("properties" in eventReport);
t.true("alertCounts" in eventReport.properties!);
}
}
@ -214,6 +216,8 @@ test("status report fields and search path setting", async (t) => {
if (customStatusReport.event_reports) {
for (const eventReport of customStatusReport.event_reports) {
t.deepEqual(eventReport.event, "codeql database interpret-results");
t.true("properties" in eventReport);
t.true("alertCounts" in eventReport.properties!);
}
}
}

4
src/analyze.ts
View file

@ -372,7 +372,9 @@ export async function runQueries(
completed_at: endTimeInterpretResults.toISOString(),
exit_status: "success",
language,
properties: perQueryAlertCounts,
properties: {
alertCounts: perQueryAlertCounts,
},
};
if (statusReport["event_reports"] === undefined) {

4
src/autobuild-action.ts
View file

@ -12,9 +12,9 @@ import {
import { getGitHubVersion } from "./api-client";
import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { CODEQL_ACTION_DID_AUTOBUILD_GOLANG } from "./shared-environment";
import {
checkGitHubVersionInRange,
initializeEnvironment,
@ -89,7 +89,7 @@ async function run() {
currentLanguage = language;
await runAutobuild(language, config, logger);
if (language === Language.go) {
core.exportVariable(CODEQL_ACTION_DID_AUTOBUILD_GOLANG, "true");
core.exportVariable(EnvVar.DID_AUTOBUILD_GOLANG, "true");
}
}
}

44
src/codeql.test.ts
View file

@ -11,7 +11,6 @@ import nock from "nock";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { GitHubApiDetails } from "./api-client";
import * as codeql from "./codeql";
import { AugmentationProperties, Config } from "./config-utils";
@ -90,7 +89,7 @@ async function installIntoToolcache({
tmpDir,
util.GitHubVariant.GHES,
cliVersion !== undefined
? { cliVersion, tagName, variant: util.GitHubVariant.GHES }
? { cliVersion, tagName }
: SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
@ -246,21 +245,11 @@ for (const {
});
}
for (const { githubReleases, toolcacheVersion } of [
for (const toolcacheVersion of [
// Test that we use the tools from the toolcache when `SAMPLE_DEFAULT_CLI_VERSION` is requested
// and `SAMPLE_DEFAULT_CLI_VERSION-` is in the toolcache.
{
toolcacheVersion: SAMPLE_DEFAULT_CLI_VERSION.cliVersion,
},
{
githubReleases: {
"codeql-bundle-20230101": `cli-version-${SAMPLE_DEFAULT_CLI_VERSION.cliVersion}.txt`,
},
toolcacheVersion: "0.0.0-20230101",
},
{
toolcacheVersion: `${SAMPLE_DEFAULT_CLI_VERSION.cliVersion}-20230101`,
},
SAMPLE_DEFAULT_CLI_VERSION.cliVersion,
`${SAMPLE_DEFAULT_CLI_VERSION.cliVersion}-20230101`,
]) {
test(
`uses tools from toolcache when ${SAMPLE_DEFAULT_CLI_VERSION.cliVersion} is requested and ` +
@ -275,26 +264,6 @@ for (const { githubReleases, toolcacheVersion } of [
.returns("path/to/cached/codeql");
sinon.stub(toolcache, "findAllVersions").returns([toolcacheVersion]);
if (githubReleases) {
sinon.stub(api, "getApiClient").value(() => ({
repos: {
listReleases: sinon.stub().resolves(undefined),
},
paginate: sinon.stub().resolves(
Object.entries(githubReleases).map(
([releaseTagName, cliVersionMarkerFile]) => ({
assets: [
{
name: cliVersionMarkerFile,
},
],
tag_name: releaseTagName,
})
)
),
}));
}
const result = await codeql.setupCodeQL(
undefined,
SAMPLE_DOTCOM_API_DETAILS,
@ -331,7 +300,6 @@ for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
},
getRunnerLogger(true),
false
@ -366,7 +334,6 @@ for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
},
getRunnerLogger(true),
false
@ -474,7 +441,6 @@ for (const isBundleVersionInUrl of [true, false]) {
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant: util.GitHubVariant.GHAE,
},
getRunnerLogger(true),
false
@ -572,7 +538,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.8.1");
sinon.stub(codeqlObject, "getVersion").resolves("2.9.4");
// safeWhich throws because of the test CodeQL object.
sinon.stub(safeWhich, "safeWhich").resolves("");

39
src/codeql.ts
View file

@ -1,12 +1,14 @@
import * as fs from "fs";
import * as path from "path";
import * as core from "@actions/core";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as yaml from "js-yaml";
import { getOptionalInput } from "./actions-util";
import * as api from "./api-client";
import { Config, getGeneratedCodeScanningConfigPath } from "./config-utils";
import { EnvVar } from "./environment";
import { errorMatchers } from "./error-matcher";
import {
CodeQLDefaultVersionInfo,
@ -268,7 +270,12 @@ let cachedCodeQL: CodeQL | undefined = undefined;
* The version flags below can be used to conditionally enable certain features
* on versions newer than this.
*/
const CODEQL_MINIMUM_VERSION = "2.8.5";
const CODEQL_MINIMUM_VERSION = "2.9.4";
/**
* This version will shortly become the oldest version of CodeQL that the Action will run with.
*/
const CODEQL_NEXT_MINIMUM_VERSION = "2.9.4";
/**
* Versions of CodeQL that version-flag certain functionality in the Action.
@ -280,13 +287,6 @@ const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
const CODEQL_VERSION_FILE_BASELINE_INFORMATION = "2.11.3";
/**
* Versions 2.9.0+ of the CodeQL CLI run machine learning models from a temporary directory, which
* resolves an issue on Windows where TensorFlow models are not correctly loaded due to the path of
* some of their files being greater than MAX_PATH (260 characters).
*/
export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
/**
* Previous versions had the option already, but were missing the
* --extractor-options-verbosity that we need.
@ -315,6 +315,11 @@ export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
*/
export const CODEQL_VERSION_RESOLVE_ENVIRONMENT = "2.13.4";
/**
* Versions 2.13.4+ of the CodeQL CLI have an associated CodeQL Bundle release that is semantically versioned.
*/
export const CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED = "2.13.4";
/**
* Versions 2.14.0+ of the CodeQL CLI support new analysis summaries.
*/
@ -1032,6 +1037,24 @@ export async function getCodeQLForCmd(
throw new Error(
`Expected a CodeQL CLI with version at least ${CODEQL_MINIMUM_VERSION} but got version ${await codeql.getVersion()}`
);
} else if (
checkVersion &&
process.env[EnvVar.SUPPRESS_DEPRECATED_SOON_WARNING] !== "true" &&
!(await util.codeQlVersionAbove(codeql, CODEQL_NEXT_MINIMUM_VERSION))
) {
core.warning(
`CodeQL CLI version ${await codeql.getVersion()} was deprecated on 2023-06-20 alongside ` +
"GitHub Enterprise Server 3.5 and will not be supported by the next release of the " +
`CodeQL Action. Please update to CodeQL CLI version ${CODEQL_NEXT_MINIMUM_VERSION} or ` +
"later. For instance, if you have specified a custom version of the CLI using the " +
"'tools' input to the 'init' Action, you can remove this input to use the default " +
"version.\n\n" +
"Alternatively, if you want to continue using CodeQL CLI version " +
`${await codeql.getVersion()}, you can replace 'github/codeql-action/*@v2' by ` +
"'github/codeql-action/*@v2.20.4' in your code scanning workflow to ensure you continue " +
"using this version of the CodeQL Action."
);
core.exportVariable(EnvVar.SUPPRESS_DEPRECATED_SOON_WARNING, "true");
}
return codeql;
}

174
src/config-utils.test.ts
View file

@ -23,7 +23,7 @@ import {
createFeatures,
mockLanguagesInRepo as mockLanguagesInRepo,
} from "./testing-utils";
import * as util from "./util";
import { GitHubVariant, GitHubVersion, UserError, withTmpDir } from "./util";
setupTests(test);
@ -35,7 +35,7 @@ const sampleApiDetails = {
registriesAuthTokens: undefined,
};
const gitHubVersion = { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion;
const gitHubVersion = { type: GitHubVariant.DOTCOM } as GitHubVersion;
// Returns the filepath of the newly-created file
function createConfigFile(inputFileContents: string, tmpDir: string): string {
@ -55,7 +55,7 @@ function mockGetContents(
data: content,
};
const spyGetContents = sinon
.stub(client.repos, "getContent")
.stub(client.rest.repos, "getContent")
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
.resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);
@ -73,12 +73,12 @@ function mockListLanguages(languages: string[]) {
response.data[language] = 123;
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
sinon.stub(client.repos, "listLanguages").resolves(response as any);
sinon.stub(client.rest.repos, "listLanguages").resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);
}
test("load empty config", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const logger = getRunnerLogger(true);
const languages = "javascript,python";
@ -145,7 +145,7 @@ test("load empty config", async (t) => {
});
test("loading config saves config", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const logger = getRunnerLogger(true);
const codeQL = setCodeQL({
@ -207,7 +207,7 @@ test("loading config saves config", async (t) => {
});
test("load input outside of workspace", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
try {
await configUtils.initConfig(
undefined,
@ -234,7 +234,7 @@ test("load input outside of workspace", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(
new UserError(
configUtils.getConfigFileOutsideWorkspaceErrorMessage(
path.join(tmpDir, "../input")
)
@ -245,7 +245,7 @@ test("load input outside of workspace", async (t) => {
});
test("load non-local input with invalid repo syntax", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
// no filename given, just a repo
const configFile = "octo-org/codeql-config@main";
@ -275,7 +275,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(
new UserError(
configUtils.getConfigFileRepoFormatInvalidMessage(
"octo-org/codeql-config@main"
)
@ -286,7 +286,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
});
test("load non-existent input", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const languages = "javascript";
const configFile = "input";
t.false(fs.existsSync(path.join(tmpDir, configFile)));
@ -317,7 +317,7 @@ test("load non-existent input", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(
new UserError(
configUtils.getConfigFileDoesNotExistErrorMessage(
path.join(tmpDir, "input")
)
@ -328,7 +328,7 @@ test("load non-existent input", async (t) => {
});
test("load non-empty input", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
@ -428,7 +428,7 @@ test("load non-empty input", async (t) => {
});
test("Default queries are used", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
// Check that the default behaviour is to add the default queries.
// In this case if a config file is specified but does not include
// the disable-default-queries field.
@ -523,7 +523,7 @@ function queriesToResolvedQueryForm(queries: string[]) {
}
test("Queries can be specified in config file", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const inputFileContents = `
name: my config
queries:
@ -598,7 +598,7 @@ test("Queries can be specified in config file", async (t) => {
});
test("Queries from config file can be overridden in workflow file", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const inputFileContents = `
name: my config
queries:
@ -677,7 +677,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
});
test("Queries in workflow file can be used in tandem with the 'disable default queries' option", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
process.env["GITHUB_WORKSPACE"] = tmpDir;
@ -751,7 +751,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
});
test("Multiple queries can be specified in workflow file, no config file required", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
fs.mkdirSync(path.join(tmpDir, "override1"));
fs.mkdirSync(path.join(tmpDir, "override2"));
@ -829,7 +829,7 @@ test("Multiple queries can be specified in workflow file, no config file require
});
test("Queries in workflow file can be added to the set of queries without overriding config file", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
process.env["GITHUB_WORKSPACE"] = tmpDir;
@ -926,7 +926,7 @@ test("Queries in workflow file can be added to the set of queries without overri
});
test("Queries can be specified using config input", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const configInput = `
name: my config
queries:
@ -1009,7 +1009,7 @@ test("Queries can be specified using config input", async (t) => {
});
test("Using config input and file together, config input should be used.", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
process.env["GITHUB_WORKSPACE"] = tmpDir;
@ -1101,7 +1101,7 @@ test("Using config input and file together, config input should be used.", async
});
test("Invalid queries in workflow file handled correctly", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const queries = "foo/bar@v1@v3";
const languages = "javascript";
@ -1148,14 +1148,16 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3"))
new UserError(
configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3")
)
);
}
});
});
test("API client used when reading remote config", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
@ -1222,7 +1224,7 @@ test("API client used when reading remote config", async (t) => {
});
test("Remote config handles the case where a directory is provided", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const dummyResponse = []; // directories are returned as arrays
mockGetContents(dummyResponse);
@ -1253,14 +1255,16 @@ test("Remote config handles the case where a directory is provided", async (t) =
} catch (err) {
t.deepEqual(
err,
new Error(configUtils.getConfigFileDirectoryGivenMessage(repoReference))
new UserError(
configUtils.getConfigFileDirectoryGivenMessage(repoReference)
)
);
}
});
});
test("Invalid format of remote config handled correctly", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const dummyResponse = {
// note no "content" property here
};
@ -1293,14 +1297,16 @@ test("Invalid format of remote config handled correctly", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(configUtils.getConfigFileFormatInvalidMessage(repoReference))
new UserError(
configUtils.getConfigFileFormatInvalidMessage(repoReference)
)
);
}
});
});
test("No detected languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
mockListLanguages([]);
const codeQL = setCodeQL({
async resolveLanguages() {
@ -1335,13 +1341,13 @@ test("No detected languages", async (t) => {
);
throw new Error("initConfig did not throw error");
} catch (err) {
t.deepEqual(err, new Error(configUtils.getNoLanguagesError()));
t.deepEqual(err, new UserError(configUtils.getNoLanguagesError()));
}
});
});
test("Unknown languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const languages = "rubbish,english";
try {
@ -1370,14 +1376,16 @@ test("Unknown languages", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(configUtils.getUnknownLanguagesError(["rubbish", "english"]))
new UserError(
configUtils.getUnknownLanguagesError(["rubbish", "english"])
)
);
}
});
});
test("Config specifies packages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
@ -1431,7 +1439,7 @@ test("Config specifies packages", async (t) => {
});
test("Config specifies packages for multiple languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
@ -1518,7 +1526,7 @@ function doInvalidInputTest(
expectedErrorMessageGenerator: (configFile: string) => string
) {
test(`load invalid input - ${testName}`, async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
@ -1561,7 +1569,10 @@ function doInvalidInputTest(
);
throw new Error("initConfig did not throw error");
} catch (err) {
t.deepEqual(err, new Error(expectedErrorMessageGenerator(inputFile)));
t.deepEqual(
err,
new UserError(expectedErrorMessageGenerator(inputFile))
);
}
});
});
@ -2104,7 +2115,7 @@ const mlPoweredQueriesMacro = test.macro({
queriesInput: string | undefined,
expectedVersionString: string | undefined
) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async getVersion() {
return codeQLVersion;
@ -2178,95 +2189,46 @@ const mlPoweredQueriesMacro = test.macro({
// Test that ML-powered queries aren't run when the feature is off.
test(
mlPoweredQueriesMacro,
"2.7.5",
"2.12.3",
false,
undefined,
"security-extended",
undefined
);
// Test that the ~0.1.0 version of ML-powered queries is run on v2.8.3 of the CLI.
test(
mlPoweredQueriesMacro,
"2.8.3",
true,
undefined,
"security-extended",
process.platform === "win32" ? undefined : "~0.1.0"
);
// Test that ML-powered queries aren't run when the user hasn't specified that we should run the
// `security-extended`, `security-and-quality`, or `security-experimental` query suite.
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
// Test that ML-powered queries are run on non-Windows platforms running `security-extended` on
// versions of the CodeQL CLI prior to 2.9.0.
test(
mlPoweredQueriesMacro,
"2.8.5",
true,
undefined,
"security-extended",
process.platform === "win32" ? undefined : "~0.2.0"
);
// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality` on
// versions of the CodeQL CLI prior to 2.9.0.
test(
mlPoweredQueriesMacro,
"2.8.5",
true,
undefined,
"security-and-quality",
process.platform === "win32" ? undefined : "~0.2.0"
);
// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL CLI
// 2.9.0+.
test(
mlPoweredQueriesMacro,
"2.9.0",
true,
undefined,
"security-extended",
"~0.2.0"
);
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.9.0+.
test(
mlPoweredQueriesMacro,
"2.9.0",
true,
undefined,
"security-and-quality",
"~0.2.0"
);
test(mlPoweredQueriesMacro, "2.12.3", true, undefined, undefined, undefined);
// Test that we don't inject an ML-powered query pack if the user has already specified one.
test(
mlPoweredQueriesMacro,
"2.9.0",
"2.12.3",
true,
"codeql/javascript-experimental-atm-queries@0.0.1",
"security-and-quality",
"0.0.1"
);
// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL
// CLI 2.9.3+.
// Test that ML-powered queries ~0.3.0 are run on all platforms running `security-extended` on
// CodeQL CLI 2.9.4+.
test(
mlPoweredQueriesMacro,
"2.9.3",
"2.9.4",
true,
undefined,
"security-extended",
"~0.3.0"
);
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.9.3+.
// Test that ML-powered queries ~0.3.0 are run on all platforms running `security-and-quality` on
// CodeQL CLI 2.9.4+.
test(
mlPoweredQueriesMacro,
"2.9.3",
"2.9.4",
true,
undefined,
"security-and-quality",
"~0.3.0"
);
// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL
// CLI 2.11.3+.
// Test that ML-powered queries ~0.4.0 are run on all platforms running `security-extended` on
// CodeQL CLI 2.11.3+.
test(
mlPoweredQueriesMacro,
"2.11.3",
@ -2275,8 +2237,8 @@ test(
"security-extended",
"~0.4.0"
);
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.11.3+.
// Test that ML-powered queries ~0.4.0 are run on all platforms running `security-and-quality` on
// CodeQL CLI 2.11.3+.
test(
mlPoweredQueriesMacro,
"2.11.3",
@ -2458,7 +2420,7 @@ test(
);
test("downloadPacks-no-registries", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs) => ({
packs,
@ -2495,7 +2457,7 @@ test("downloadPacks-no-registries", async (t) => {
test("downloadPacks-with-registries", async (t) => {
// same thing, but this time include a registries block and
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = undefined;
const logger = getRunnerLogger(true);
@ -2586,7 +2548,7 @@ test("downloadPacks-with-registries", async (t) => {
test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
// same thing, but this time include a registries block and
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
@ -2628,7 +2590,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
test("downloadPacks-with-registries fails with invalid registries block", async (t) => {
// same thing, but this time include a registries block and
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
@ -2670,7 +2632,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
// the happy path for generateRegistries is already tested in downloadPacks.
// these following tests are for the error cases and when nothing is generated.
test("no generateRegistries when CLI is too old", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const registriesInput = yaml.dump([
{
// no slash
@ -2698,7 +2660,7 @@ test("no generateRegistries when CLI is too old", async (t) => {
});
});
test("no generateRegistries when registries is undefined", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
const registriesInput = undefined;
const codeQL = setCodeQL({
// Accepted CLI versions are 2.10.4 or higher
@ -2719,7 +2681,7 @@ test("no generateRegistries when registries is undefined", async (t) => {
});
test("generateRegistries prefers original CODEQL_REGISTRIES_AUTH", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tmpDir) => {
process.env.CODEQL_REGISTRIES_AUTH = "original";
const registriesInput = yaml.dump([
{

108
src/config-utils.ts
View file

@ -9,7 +9,6 @@ import * as api from "./api-client";
import {
CodeQL,
CODEQL_VERSION_GHES_PACK_DOWNLOAD,
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS,
CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE,
ResolveQueriesOutput,
} from "./codeql";
@ -31,6 +30,7 @@ import {
logCodeScanningConfigInCli,
ML_POWERED_JS_QUERIES_PACK_NAME,
useCodeScanningConfigInCli,
UserError,
} from "./util";
// Property names from the user-supplied config file.
@ -300,7 +300,7 @@ function validateQueries(resolvedQueries: ResolveQueriesOutput) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage;
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new Error(
throw new UserError(
`${
"The following queries do not declare a language. " +
"Their qlpack.yml files are either missing or is invalid.\n"
@ -313,7 +313,7 @@ function validateQueries(resolvedQueries: ResolveQueriesOutput) {
multipleDeclaredLanguages
);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new Error(
throw new UserError(
`${
"The following queries declare multiple languages. " +
"Their qlpack.yml files are either missing or is invalid.\n"
@ -404,7 +404,7 @@ async function addBuiltinSuiteQueries(
let injectedMlQueries = false;
const found = builtinSuites.find((suite) => suite === suiteName);
if (!found) {
throw new Error(getQueryUsesInvalid(configFile, suiteName));
throw new UserError(getQueryUsesInvalid(configFile, suiteName));
}
if (
suiteName === "security-experimental" &&
@ -413,7 +413,7 @@ async function addBuiltinSuiteQueries(
CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE
))
) {
throw new Error(
throw new UserError(
`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`
@ -424,12 +424,6 @@ async function addBuiltinSuiteQueries(
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries.
if (
// Only run ML-powered queries on Windows if we have a CLI that supports it.
(process.platform !== "win32" ||
(await codeQlVersionAbove(
codeQL,
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS
))) &&
languages.includes("javascript") &&
(found === "security-experimental" ||
found === "security-extended" ||
@ -469,7 +463,7 @@ async function addLocalQueries(
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
throw new Error(getLocalPathDoesNotExist(configFile, localQueryPath));
throw new UserError(getLocalPathDoesNotExist(configFile, localQueryPath));
}
// Call this after checking file exists, because it'll fail if file doesn't exist
@ -481,7 +475,7 @@ async function addLocalQueries(
fs.realpathSync(workspacePath) + path.sep
)
) {
throw new Error(
throw new UserError(
getLocalPathOutsideOfRepository(configFile, localQueryPath)
);
}
@ -510,7 +504,7 @@ async function addRemoteQueries(
) {
let tok = queryUses.split("@");
if (tok.length !== 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
throw new UserError(getQueryUsesInvalid(configFile, queryUses));
}
const ref = tok[1];
@ -520,11 +514,11 @@ async function addRemoteQueries(
// The second token is the repo
// The rest is a path, if there is more than one token combine them to form the full path
if (tok.length < 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
throw new UserError(getQueryUsesInvalid(configFile, queryUses));
}
// Check none of the parts of the repository name are empty
if (tok[0].trim() === "" || tok[1].trim() === "") {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
throw new UserError(getQueryUsesInvalid(configFile, queryUses));
}
const nwo = `${tok[0]}/${tok[1]}`;
@ -573,7 +567,7 @@ async function parseQueryUses(
): Promise<boolean> {
queryUses = queryUses.trim();
if (queryUses === "") {
throw new Error(getQueryUsesInvalid(configFile));
throw new UserError(getQueryUsesInvalid(configFile));
}
// Check for the local path case before we start trying to parse the repository name
@ -650,7 +644,7 @@ export function validateAndSanitisePath(
// An empty path is not allowed as it's meaningless
if (newPath === "") {
throw new Error(
throw new UserError(
getConfigFilePropertyError(
configFile,
propertyName,
@ -662,7 +656,7 @@ export function validateAndSanitisePath(
// Check for illegal uses of **
if (newPath.match(pathStarsRegex)) {
throw new Error(
throw new UserError(
getConfigFilePropertyError(
configFile,
propertyName,
@ -689,7 +683,7 @@ export function validateAndSanitisePath(
// This may not play nicely with project layouts.
// This restriction can be lifted later if we determine they are ok.
if (newPath.indexOf("\\") !== -1) {
throw new Error(
throw new UserError(
getConfigFilePropertyError(
configFile,
propertyName,
@ -888,7 +882,7 @@ export async function getLanguagesInRepo(
logger: Logger
): Promise<LanguageOrAlias[]> {
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api.getApiClient().repos.listLanguages({
const response = await api.getApiClient().rest.repos.listLanguages({
owner: repository.owner,
repo: repository.repo,
});
@ -900,7 +894,7 @@ export async function getLanguagesInRepo(
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
const languages: Set<LanguageOrAlias> = new Set();
for (const lang of Object.keys(response.data)) {
for (const lang of Object.keys(response.data as Record<string, number>)) {
const parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
@ -945,7 +939,7 @@ export async function getLanguages(
// If the languages parameter was not given and no languages were
// detected then fail here as this is a workflow configuration error.
if (languages.length === 0) {
throw new Error(getNoLanguagesError());
throw new UserError(getNoLanguagesError());
}
// Make sure they are supported
@ -964,7 +958,7 @@ export async function getLanguages(
// Any unknown languages here would have come directly from the input
// since we filter unknown languages coming from the GitHub API.
if (unknownLanguages.length > 0) {
throw new Error(getUnknownLanguagesError(unknownLanguages));
throw new UserError(getUnknownLanguagesError(unknownLanguages));
}
return parsedLanguages;
@ -1193,10 +1187,10 @@ async function loadConfig(
// even though we don't use the value yet.
if (NAME_PROPERTY in parsedYAML) {
if (typeof parsedYAML[NAME_PROPERTY] !== "string") {
throw new Error(getNameInvalid(configFile));
throw new UserError(getNameInvalid(configFile));
}
if (parsedYAML[NAME_PROPERTY]!.length === 0) {
throw new Error(getNameInvalid(configFile));
throw new UserError(getNameInvalid(configFile));
}
}
@ -1220,7 +1214,7 @@ async function loadConfig(
let disableDefaultQueries = false;
if (DISABLE_DEFAULT_QUERIES_PROPERTY in parsedYAML) {
if (typeof parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY] !== "boolean") {
throw new Error(getDisableDefaultQueriesInvalid(configFile));
throw new UserError(getDisableDefaultQueriesInvalid(configFile));
}
disableDefaultQueries = parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY]!;
}
@ -1266,11 +1260,11 @@ async function loadConfig(
) {
const queriesArr = parsedYAML[QUERIES_PROPERTY];
if (!Array.isArray(queriesArr)) {
throw new Error(getQueriesInvalid(configFile));
throw new UserError(getQueriesInvalid(configFile));
}
for (const query of queriesArr) {
if (typeof query[QUERIES_USES_PROPERTY] !== "string") {
throw new Error(getQueriesMissingUses(configFile));
throw new UserError(getQueriesMissingUses(configFile));
}
await parseQueryUses(
languages,
@ -1290,11 +1284,11 @@ async function loadConfig(
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
throw new Error(getPathsIgnoreInvalid(configFile));
throw new UserError(getPathsIgnoreInvalid(configFile));
}
for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]!) {
if (typeof ignorePath !== "string" || ignorePath === "") {
throw new Error(getPathsIgnoreInvalid(configFile));
throw new UserError(getPathsIgnoreInvalid(configFile));
}
pathsIgnore.push(
validateAndSanitisePath(
@ -1309,11 +1303,11 @@ async function loadConfig(
if (PATHS_PROPERTY in parsedYAML) {
if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
throw new Error(getPathsInvalid(configFile));
throw new UserError(getPathsInvalid(configFile));
}
for (const includePath of parsedYAML[PATHS_PROPERTY]!) {
if (typeof includePath !== "string" || includePath === "") {
throw new Error(getPathsInvalid(configFile));
throw new UserError(getPathsInvalid(configFile));
}
paths.push(
validateAndSanitisePath(includePath, PATHS_PROPERTY, configFile, logger)
@ -1405,7 +1399,7 @@ function parseQueriesFromInput(
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(
throw new UserError(
getConfigFilePropertyError(
undefined,
"queries",
@ -1445,13 +1439,13 @@ export function parsePacksFromConfig(
} else {
// this is an error since multi-language analysis requires
// packs split by language
throw new Error(getPacksInvalidSplit(configFile));
throw new UserError(getPacksInvalidSplit(configFile));
}
}
for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
if (!Array.isArray(packsArr)) {
throw new Error(getPacksInvalid(configFile));
throw new UserError(getPacksInvalid(configFile));
}
if (!languages.includes(lang as Language)) {
// This particular language is not being analyzed in this run.
@ -1462,7 +1456,7 @@ export function parsePacksFromConfig(
continue;
} else {
// This language is invalid, probably a misspelling
throw new Error(getPacksRequireLanguage(configFile, lang));
throw new UserError(getPacksRequireLanguage(configFile, lang));
}
}
@ -1483,18 +1477,20 @@ function parsePacksFromInput(
}
if (languages.length > 1) {
throw new Error(
throw new UserError(
"Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by language."
);
} else if (languages.length === 0) {
throw new Error("No languages specified. Cannot process the packs input.");
throw new UserError(
"No languages specified. Cannot process the packs input."
);
}
rawPacksInput = rawPacksInput.trim();
if (packsInputCombines) {
rawPacksInput = rawPacksInput.trim().substring(1).trim();
if (!rawPacksInput) {
throw new Error(
throw new UserError(
getConfigFilePropertyError(
undefined,
"packs",
@ -1535,7 +1531,7 @@ export function parsePacksSpecification(
configFile?: string
): Pack {
if (typeof packStr !== "string") {
throw new Error(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr, configFile));
}
packStr = packStr.trim();
@ -1563,14 +1559,14 @@ export function parsePacksSpecification(
: undefined;
if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
throw new Error(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr, configFile));
}
if (version) {
try {
new semver.Range(version);
} catch (e) {
// The range string is invalid. OK to ignore the caught error
throw new Error(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr, configFile));
}
}
@ -1584,12 +1580,12 @@ export function parsePacksSpecification(
path.normalize(packPath).split(path.sep).join("/") !==
packPath.split(path.sep).join("/"))
) {
throw new Error(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr, configFile));
}
if (!packPath && pathStart) {
// 0 length path
throw new Error(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr, configFile));
}
return {
@ -1635,7 +1631,7 @@ export function parsePacks(
}
if (!packsInputCombines) {
if (!packsFromInput) {
throw new Error(getPacksInvalid(configFile));
throw new UserError(getPacksInvalid(configFile));
}
return packsFromInput;
}
@ -1775,7 +1771,7 @@ export async function initConfig(
const hasCustomQueries = config.queries[language]?.custom.length > 0;
const hasPacks = (config.packs[language]?.length || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(
throw new UserError(
`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run."
);
@ -1806,7 +1802,7 @@ function parseRegistries(
? (yaml.load(registriesInput) as RegistryConfigWithCredentials[])
: undefined;
} catch (e) {
throw new Error("Invalid registries input. Must be a YAML string.");
throw new UserError("Invalid registries input. Must be a YAML string.");
}
}
@ -1822,12 +1818,12 @@ function isLocal(configPath: string): boolean {
function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
// Error if the config file is now outside of the workspace
if (!(configFile + path.sep).startsWith(workspacePath + path.sep)) {
throw new Error(getConfigFileOutsideWorkspaceErrorMessage(configFile));
throw new UserError(getConfigFileOutsideWorkspaceErrorMessage(configFile));
}
// Error if the file does not exist
if (!fs.existsSync(configFile)) {
throw new Error(getConfigFileDoesNotExistErrorMessage(configFile));
throw new UserError(getConfigFileDoesNotExistErrorMessage(configFile));
}
return yaml.load(fs.readFileSync(configFile, "utf8")) as UserConfig;
@ -1844,12 +1840,12 @@ async function getRemoteConfig(
const pieces = format.exec(configFile);
// 5 = 4 groups + the whole expression
if (pieces === null || pieces.groups === undefined || pieces.length < 5) {
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
throw new UserError(getConfigFileRepoFormatInvalidMessage(configFile));
}
const response = await api
.getApiClientWithExternalAuth(apiDetails)
.repos.getContent({
.rest.repos.getContent({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
@ -1860,9 +1856,9 @@ async function getRemoteConfig(
if ("content" in response.data && response.data.content !== undefined) {
fileContents = response.data.content;
} else if (Array.isArray(response.data)) {
throw new Error(getConfigFileDirectoryGivenMessage(configFile));
throw new UserError(getConfigFileDirectoryGivenMessage(configFile));
} else {
throw new Error(getConfigFileFormatInvalidMessage(configFile));
throw new UserError(getConfigFileFormatInvalidMessage(configFile));
}
return yaml.load(
@ -1986,7 +1982,7 @@ export async function generateRegistries(
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
throw new UserError(
`The 'registries' input is not supported on CodeQL CLI versions earlier than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}. Please upgrade to CodeQL CLI version ${CODEQL_VERSION_GHES_PACK_DOWNLOAD} or later.`
);
}
@ -2025,7 +2021,7 @@ function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
!Array.isArray(registries) ||
registries.some((r) => !r.url || !r.packages)
) {
throw new Error(
throw new UserError(
"Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties."
);
}

8
src/defaults.json
View file

@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.13.5",
"cliVersion": "2.13.5",
"priorBundleVersion": "codeql-bundle-v2.13.4",
"priorCliVersion": "2.13.4"
"bundleVersion": "codeql-bundle-v2.14.0",
"cliVersion": "2.14.0",
"priorBundleVersion": "codeql-bundle-v2.13.5",
"priorCliVersion": "2.13.5"
}

63
src/environment.ts Normal file
View file

@ -0,0 +1,63 @@
export enum EnvVar {
/** Set to true when the `analyze` Action completes successfully. */
ANALYZE_DID_COMPLETE_SUCCESSFULLY = "CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY",
/** Set to "true" when the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
/**
* Used to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
DISABLE_DUPLICATE_LOCATION_FIX = "CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX",
/**
* If set to the "true" string, then the CodeQL Action is using its
* own deprecated and non-standard way of scanning for multiple
* languages.
*/
FEATURE_MULTI_LANGUAGE = "CODEQL_ACTION_FEATURE_MULTI_LANGUAGE",
/**
* If set to the "true" string, then the CodeQL Action is using its
* own sandwiched workflow mechanism.
*/
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
/**
* If set to a truthy value, then the CodeQL Action might combine SARIF
* output from several `interpret-results` runs for the same language.
*/
FEATURE_SARIF_COMBINE = "CODEQL_ACTION_FEATURE_SARIF_COMBINE",
/**
* If set to the "true" string, then the CodeQL Action will upload SARIF,
* not the CLI.
*/
FEATURE_WILL_UPLOAD = "CODEQL_ACTION_FEATURE_WILL_UPLOAD",
/** UUID representing the current job run. */
JOB_RUN_UUID = "JOB_RUN_UUID",
ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
/** Whether to suppress the warning if the current CLI will soon be unsupported. */
SUPPRESS_DEPRECATED_SOON_WARNING = "CODEQL_ACTION_SUPPRESS_DEPRECATED_SOON_WARNING",
/** Used to disable uploading SARIF results or status reports to the GitHub API */
TEST_MODE = "CODEQL_ACTION_TEST_MODE",
TESTING_ENVIRONMENT = "CODEQL_ACTION_TESTING_ENVIRONMENT",
/** Semver of the CodeQL Action as specified in `package.json`. */
VERSION = "CODEQL_ACTION_VERSION",
/**
* The time at which the first action (normally init) started executing.
* If a workflow invokes a different action without first invoking the init
* action (i.e. the upload action is being used by a third-party integrator)
* then this variable will be assigned the start time of the action invoked
* rather that the init action.
*/
WORKFLOW_STARTED_AT = "CODEQL_WORKFLOW_STARTED_AT",
}

70
src/feature-flags.test.ts
View file

@ -366,31 +366,48 @@ for (const variant of [GitHubVariant.GHAE, GitHubVariant.GHES]) {
t.deepEqual(defaultCliVersion, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
});
});
});
}
test("selects CLI v2.12.1 on Dotcom when feature flags enable v2.12.0 and v2.12.1", async (t) => {
test("selects CLI v2.20.1 on Dotcom when feature flags enable v2.20.0 and v2.20.1", async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_12_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_2_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_12_3_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_12_4_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_12_5_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_2_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_3_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_4_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_5_enabled"] = false;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM
);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.12.1",
cliVersion: "2.20.1",
tagName: "codeql-bundle-v2.20.1",
toolsFeatureFlagsValid: true,
});
});
});
test("includes tag name when feature flags enable version greater than v2.13.4", async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM
);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.20.0",
tagName: "codeql-bundle-v2.20.0",
toolsFeatureFlagsValid: true,
variant: GitHubVariant.DOTCOM,
});
});
});
@ -406,8 +423,27 @@ test(`selects CLI from defaults.json on Dotcom when no default version feature f
);
t.deepEqual(defaultCliVersion, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
toolsFeatureFlagsValid: false,
});
});
});
test(`selects CLI from defaults.json on Dotcom when default version feature flags are unsupported`, async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
// Doesn't have a semantically versioned bundle
expectedFeatureEnablement["default_codeql_version_2_13_3_enabled"] = true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM
);
t.deepEqual(defaultCliVersion, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
toolsFeatureFlagsValid: false,
variant: GitHubVariant.DOTCOM,
});
});
});
@ -420,9 +456,9 @@ test("ignores invalid version numbers in default version feature flags", async (
getRecordingLogger(loggedMessages)
);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_12_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_invalid_enabled"] =
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_invalid_enabled"] =
true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
@ -430,9 +466,9 @@ test("ignores invalid version numbers in default version feature flags", async (
GitHubVariant.DOTCOM
);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.12.1",
cliVersion: "2.20.1",
tagName: "codeql-bundle-v2.20.1",
toolsFeatureFlagsValid: true,
variant: GitHubVariant.DOTCOM,
});
t.assert(
@ -440,7 +476,7 @@ test("ignores invalid version numbers in default version feature flags", async (
(v: LoggedMessage) =>
v.type === "warning" &&
v.message ===
"Ignoring feature flag default_codeql_version_2_12_invalid_enabled as it does not specify a valid CodeQL version."
"Ignoring feature flag default_codeql_version_2_20_invalid_enabled as it does not specify a valid CodeQL version."
) !== undefined
);
});

75
src/feature-flags.ts
View file

@ -4,7 +4,11 @@ import * as path from "path";
import * as semver from "semver";
import { getApiClient } from "./api-client";
import { CODEQL_VERSION_NEW_ANALYSIS_SUMMARY, CodeQL } from "./codeql";
import {
CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED,
CODEQL_VERSION_NEW_ANALYSIS_SUMMARY,
CodeQL,
} from "./codeql";
import * as defaults from "./defaults.json";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
@ -13,20 +17,11 @@ import * as util from "./util";
const DEFAULT_VERSION_FEATURE_FLAG_PREFIX = "default_codeql_version_";
const DEFAULT_VERSION_FEATURE_FLAG_SUFFIX = "_enabled";
export type CodeQLDefaultVersionInfo =
| {
cliVersion: string;
toolsFeatureFlagsValid?: boolean;
variant: util.GitHubVariant.DOTCOM;
}
| {
cliVersion: string;
tagName: string;
variant:
| util.GitHubVariant.GHAE
| util.GitHubVariant.GHES
| util.GitHubVariant.GHE_DOTCOM;
};
export interface CodeQLDefaultVersionInfo {
cliVersion: string;
tagName: string;
toolsFeatureFlagsValid?: boolean;
}
export interface FeatureEnablement {
/** Gets the default version of the CodeQL tools. */
@ -44,6 +39,7 @@ export enum Feature {
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
NewAnalysisSummaryEnabled = "new_analysis_summary_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
ScalingReservedRam = "scaling_reserved_ram",
UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
}
@ -68,7 +64,7 @@ export const featureConfig: Record<
},
[Feature.MlPoweredQueriesEnabled]: {
envVar: "CODEQL_ML_POWERED_QUERIES",
minimumVersion: "2.7.5",
minimumVersion: undefined,
defaultValue: false,
},
[Feature.NewAnalysisSummaryEnabled]: {
@ -81,6 +77,11 @@ export const featureConfig: Record<
minimumVersion: undefined,
defaultValue: false,
},
[Feature.ScalingReservedRam]: {
envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
minimumVersion: undefined,
defaultValue: false,
},
[Feature.UploadFailedSarifEnabled]: {
envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
minimumVersion: "2.11.3",
@ -256,33 +257,27 @@ class GitHubFeatureFlags {
variant: util.GitHubVariant
): Promise<CodeQLDefaultVersionInfo> {
if (variant === util.GitHubVariant.DOTCOM) {
const defaultDotComCliVersion = await this.getDefaultDotcomCliVersion();
return {
cliVersion: defaultDotComCliVersion.version,
toolsFeatureFlagsValid: this.hasAccessedRemoteFeatureFlags
? defaultDotComCliVersion.toolsFeatureFlagsValid
: undefined,
variant,
};
return await this.getDefaultDotcomCliVersion();
}
return {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
};
}
async getDefaultDotcomCliVersion(): Promise<{
version: string;
toolsFeatureFlagsValid: boolean | undefined;
}> {
async getDefaultDotcomCliVersion(): Promise<CodeQLDefaultVersionInfo> {
const response = await this.getAllFeatures();
const enabledFeatureFlagCliVersions = Object.entries(response)
.map(([f, isEnabled]) =>
isEnabled ? this.getCliVersionFromFeatureFlag(f) : undefined
)
.filter((f) => f !== undefined)
.filter(
(f) =>
f !== undefined &&
// Only consider versions that have semantically versioned bundles.
semver.gte(f, CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED)
)
.map((f) => f as string);
if (enabledFeatureFlagCliVersions.length === 0) {
@ -300,12 +295,14 @@ class GitHubFeatureFlags {
"Feature flags do not specify a default CLI version. Falling back to the CLI version " +
`shipped with the Action. This is ${defaults.cliVersion}.`
);
return {
version: defaults.cliVersion,
toolsFeatureFlagsValid: this.hasAccessedRemoteFeatureFlags
? false
: undefined,
const result: CodeQLDefaultVersionInfo = {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
};
if (this.hasAccessedRemoteFeatureFlags) {
result.toolsFeatureFlagsValid = false;
}
return result;
}
const maxCliVersion = enabledFeatureFlagCliVersions.reduce(
@ -316,7 +313,11 @@ class GitHubFeatureFlags {
this.logger.debug(
`Derived default CLI version of ${maxCliVersion} from feature flags.`
);
return { version: maxCliVersion, toolsFeatureFlagsValid: true };
return {
cliVersion: maxCliVersion,
tagName: `codeql-bundle-v${maxCliVersion}`,
toolsFeatureFlagsValid: true,
};
}
async getValue(feature: Feature): Promise<boolean | undefined> {
@ -422,7 +423,7 @@ class GitHubFeatureFlags {
"This run of the CodeQL Action does not have permission to access Code Scanning API endpoints. " +
"As a result, it will not be opted into any experimental features. " +
"This could be because the Action is running on a pull request from a fork. If not, " +
`please ensure the Action has the 'security-events: write' permission. Details: ${e}`
`please ensure the Action has the 'security-events: write' permission. Details: ${e.message}`
);
this.hasAccessedRemoteFeatureFlags = false;
return {};

4
src/init-action-post-helper.ts
View file

@ -3,10 +3,10 @@ import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY } from "./shared-environment";
import * as uploadLib from "./upload-lib";
import {
getRequiredEnvParam,
@ -114,7 +114,7 @@ export async function tryUploadSarifIfRunFailed(
features: FeatureEnablement,
logger: Logger
): Promise<UploadFailedSarifResult> {
if (process.env[CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY] !== "true") {
if (process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] !== "true") {
try {
return await maybeUploadFailedSarif(
config,

11
src/init-action.ts
View file

@ -16,6 +16,7 @@ import {
import { getGitHubVersion } from "./api-client";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import {
initCodeQL,
@ -27,7 +28,6 @@ import {
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as sharedEnv from "./shared-environment";
import { getTotalCacheSize } from "./trap-caching";
import {
checkForTimeout,
@ -38,7 +38,6 @@ import {
getMlPoweredJsQueriesStatus,
getRequiredEnvParam,
getThreadsFlagValue,
GitHubVariant,
initializeEnvironment,
isHostedRunner,
wrapError,
@ -214,7 +213,7 @@ async function run() {
logger
);
core.exportVariable(sharedEnv.JOB_RUN_UUID, uuidV4());
core.exportVariable(EnvVar.JOB_RUN_UUID, uuidV4());
try {
const workflowErrors = await validateWorkflow(logger);
@ -235,9 +234,7 @@ async function run() {
const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(
gitHubVersion.type
);
if (codeQLDefaultVersionInfo.variant === GitHubVariant.DOTCOM) {
toolsFeatureFlagsValid = codeQLDefaultVersionInfo.toolsFeatureFlagsValid;
}
toolsFeatureFlagsValid = codeQLDefaultVersionInfo.toolsFeatureFlagsValid;
const initCodeQLResult = await initCodeQL(
getOptionalInput("tools"),
apiDetails,
@ -332,7 +329,7 @@ async function run() {
core.exportVariable(
"CODEQL_RAM",
process.env["CODEQL_RAM"] ||
getMemoryFlagValue(getOptionalInput("ram")).toString()
(await getMemoryFlagValue(getOptionalInput("ram"), features)).toString()
);
core.exportVariable(
"CODEQL_THREADS",

59
src/setup-codeql.test.ts
View file

@ -4,7 +4,6 @@ import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { getRunnerLogger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import {
@ -77,64 +76,6 @@ test("getCodeQLActionRepository", (t) => {
t.deepEqual(repoEnv, "xxx/yyy");
});
test("findCodeQLBundleTagDotcomOnly() matches GitHub Release with marker file", async (t) => {
// Look for GitHub Releases in github/codeql-action
sinon.stub(actionsUtil, "isRunningLocalAction").resolves(true);
sinon.stub(api, "getApiClient").value(() => ({
repos: {
listReleases: sinon.stub().resolves(undefined),
},
paginate: sinon.stub().resolves([
{
assets: [
{
name: "cli-version-2.12.0.txt",
},
],
tag_name: "codeql-bundle-20230106",
},
]),
}));
t.is(
await setupCodeql.findCodeQLBundleTagDotcomOnly(
"2.12.0",
getRunnerLogger(true)
),
"codeql-bundle-20230106"
);
});
test("findCodeQLBundleTagDotcomOnly() errors if no GitHub Release matches marker file", async (t) => {
// Look for GitHub Releases in github/codeql-action
sinon.stub(actionsUtil, "isRunningLocalAction").resolves(true);
sinon.stub(api, "getApiClient").value(() => ({
repos: {
listReleases: sinon.stub().resolves(undefined),
},
paginate: sinon.stub().resolves([
{
assets: [
{
name: "cli-version-2.12.0.txt",
},
],
tag_name: "codeql-bundle-20230106",
},
]),
}));
await t.throwsAsync(
async () =>
await setupCodeql.findCodeQLBundleTagDotcomOnly(
"2.12.1",
getRunnerLogger(true)
),
{
message:
"Failed to find a release of the CodeQL tools that contains CodeQL CLI 2.12.1.",
}
);
});
test("getCodeQLSource sets CLI version for a semver tagged bundle", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);

112
src/setup-codeql.ts
View file

@ -50,40 +50,6 @@ export function getCodeQLActionRepository(logger: Logger): string {
return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
}
/**
* Gets the tag name and, if known, the CodeQL CLI version for each CodeQL bundle release.
*
* CodeQL bundles are currently tagged in the form `codeql-bundle-yyyymmdd`, so it is not possible
* to directly find the CodeQL bundle release for a particular CLI version or find the CodeQL CLI
* version for a particular CodeQL bundle.
*
* To get around this, we add a `cli-version-x.y.z.txt` asset to each bundle release that specifies
* the CLI version for that bundle release. We can then use the GitHub Releases for the CodeQL
* Action as a source of truth.
*
* In the medium term, we should migrate to a tagging scheme that allows us to directly find the
* CodeQL bundle release for a particular CLI version, for example `codeql-bundle-vx.y.z`.
*/
async function getCodeQLBundleReleasesDotcomOnly(
logger: Logger
): Promise<Array<{ cliVersion?: string; tagName: string }>> {
logger.debug(
`Fetching CodeQL CLI version and CodeQL bundle tag name information for releases of the CodeQL tools.`
);
const apiClient = api.getApiClient();
const codeQLActionRepository = getCodeQLActionRepository(logger);
const releases = await apiClient.paginate(apiClient.repos.listReleases, {
owner: codeQLActionRepository.split("/")[0],
repo: codeQLActionRepository.split("/")[1],
});
logger.debug(`Found ${releases.length} releases.`);
return releases.map((release) => ({
cliVersion: tryGetCodeQLCliVersionForRelease(release, logger),
tagName: release.tag_name,
}));
}
function tryGetCodeQLCliVersionForRelease(
release,
logger: Logger
@ -106,26 +72,6 @@ function tryGetCodeQLCliVersionForRelease(
return cliVersionsFromMarkerFiles[0];
}
export async function findCodeQLBundleTagDotcomOnly(
cliVersion: string,
logger: Logger
): Promise<string> {
const filtered = (await getCodeQLBundleReleasesDotcomOnly(logger)).filter(
(release) => release.cliVersion === cliVersion
);
if (filtered.length === 0) {
throw new Error(
`Failed to find a release of the CodeQL tools that contains CodeQL CLI ${cliVersion}.`
);
} else if (filtered.length > 1) {
throw new Error(
`Found multiple releases of the CodeQL tools that contain CodeQL CLI ${cliVersion}. ` +
`Only one such release should exist.`
);
}
return filtered[0].tagName;
}
export async function tryFindCliVersionDotcomOnly(
tagName: string,
logger: Logger
@ -136,7 +82,7 @@ export async function tryFindCliVersionDotcomOnly(
);
const apiClient = api.getApiClient();
const codeQLActionRepository = getCodeQLActionRepository(logger);
const release = await apiClient.repos.getReleaseByTag({
const release = await apiClient.rest.repos.getReleaseByTag({
owner: codeQLActionRepository.split("/")[0],
repo: codeQLActionRepository.split("/")[1],
tag: tagName,
@ -219,7 +165,7 @@ async function getCodeQLBundleDownloadURL(
}
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getApiClient().repos.getReleaseByTag({
const release = await api.getApiClient().rest.repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: tagName,
@ -411,7 +357,7 @@ export async function getCodeQLSource(
} else {
// Otherwise, use the default CLI version passed in.
cliVersion = defaultCliVersion.cliVersion;
tagName = defaultCliVersion["tagName"];
tagName = defaultCliVersion.tagName;
}
const bundleVersion =
@ -430,7 +376,7 @@ export async function getCodeQLSource(
`URL: ${url ?? "unspecified"}.`
);
let codeqlFolder;
let codeqlFolder: string | undefined;
if (cliVersion) {
// If we find the specified CLI version, we always use that.
@ -475,26 +421,18 @@ export async function getCodeQLSource(
}
// Fall back to matching `0.0.0-<bundleVersion>`.
if (!codeqlFolder && (cliVersion || tagName)) {
if (cliVersion || tagName) {
const fallbackVersion = await tryGetFallbackToolcacheVersion(
cliVersion,
tagName,
variant,
logger
);
if (fallbackVersion) {
codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
} else {
logger.debug(
"Could not determine a fallback toolcache version number for CodeQL tools version " +
`${humanReadableVersion}.`
);
}
if (!codeqlFolder && tagName) {
const fallbackVersion = await tryGetFallbackToolcacheVersion(
cliVersion,
tagName,
logger
);
if (fallbackVersion) {
codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
} else {
logger.debug(
"Both the CLI version and the bundle version are unknown, so we will not be able to find " +
"the requested version of the CodeQL tools in the toolcache."
"Could not determine a fallback toolcache version number for CodeQL tools version " +
`${humanReadableVersion}.`
);
}
}
@ -535,16 +473,8 @@ export async function getCodeQLSource(
}
if (!url) {
if (!tagName && cliVersion && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
} else if (!tagName) {
throw new Error(
`Could not obtain the requested version (${humanReadableVersion}) of the CodeQL tools ` +
"since we could not compute the tag name."
);
}
url = await getCodeQLBundleDownloadURL(
tagName,
tagName!,
apiDetails,
variant,
logger
@ -566,19 +496,9 @@ export async function getCodeQLSource(
*/
export async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string | undefined,
variant: util.GitHubVariant,
tagName: string,
logger: Logger
): Promise<string | undefined> {
//
// If we are on Dotcom, we will make an HTTP request to the Releases API here
// to find the tag name for the requested version.
if (cliVersion && !tagName && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
if (!tagName) {
return undefined;
}
const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger);
if (!bundleVersion) {
return undefined;

76
src/shared-environment.ts
View file

@ -1,76 +0,0 @@
/**
* Environment variables to be set by codeql-action and used by the
* CLI.
*/
export enum EnvVar {
/**
* Semver of the codeql-action as specified in package.json.
*/
VERSION = "CODEQL_ACTION_VERSION",
/**
* If set to a truthy value, then the codeql-action might combine SARIF
* output from several `interpret-results` runs for the same Language.
*/
FEATURE_SARIF_COMBINE = "CODEQL_ACTION_FEATURE_SARIF_COMBINE",
/**
* If set to the "true" string, then the codeql-action will upload SARIF,
* not the cli.
*/
FEATURE_WILL_UPLOAD = "CODEQL_ACTION_FEATURE_WILL_UPLOAD",
/**
* If set to the "true" string, then the codeql-action is using its
* own deprecated and non-standard way of scanning for multiple
* languages.
*/
FEATURE_MULTI_LANGUAGE = "CODEQL_ACTION_FEATURE_MULTI_LANGUAGE",
/**
* If set to the "true" string, then the codeql-action is using its
* own sandwiched workflow mechanism
*/
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
}
/**
* Environment variable that is set to true when the CodeQL Action has invoked
* the Go autobuilder.
*/
export const CODEQL_ACTION_DID_AUTOBUILD_GOLANG =
"CODEQL_ACTION_DID_AUTOBUILD_GOLANG";
/**
* This environment variable is set to true when the `analyze` Action
* completes successfully.
*/
export const CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY =
"CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY";
export const CODEQL_ACTION_TESTING_ENVIRONMENT =
"CODEQL_ACTION_TESTING_ENVIRONMENT";
/** Used to disable uploading SARIF results or status reports to the GitHub API */
export const CODEQL_ACTION_TEST_MODE = "CODEQL_ACTION_TEST_MODE";
/**
* Used to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
export const CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX =
"CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX";
/**
* The time at which the first action (normally init) started executing.
* If a workflow invokes a different action without first invoking the init
* action (i.e. the upload action is being used by a third-party integrator)
* then this variable will be assigned the start time of the action invoked
* rather that the init action.
*/
export const CODEQL_WORKFLOW_STARTED_AT = "CODEQL_WORKFLOW_STARTED_AT";
export const ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION";
/** UUID representing the current job run. */
export const JOB_RUN_UUID = "JOB_RUN_UUID";

12
src/testing-utils.ts
View file

@ -15,7 +15,7 @@ import {
FeatureEnablement,
} from "./feature-flags";
import { Logger } from "./logging";
import { GitHubVariant, HTTPError } from "./util";
import { HTTPError } from "./util";
export const SAMPLE_DOTCOM_API_DETAILS = {
auth: "token",
@ -24,8 +24,8 @@ export const SAMPLE_DOTCOM_API_DETAILS = {
};
export const SAMPLE_DEFAULT_CLI_VERSION: CodeQLDefaultVersionInfo = {
cliVersion: "2.0.0",
variant: GitHubVariant.DOTCOM,
cliVersion: "2.20.0",
tagName: "codeql-bundle-v2.20.0",
};
type TestContext = {
@ -199,8 +199,10 @@ export function mockLanguagesInRepo(languages: string[]) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
mockClient.returns({
repos: {
listLanguages,
rest: {
repos: {
listLanguages,
},
},
} as any);
return listLanguages;

4
src/upload-lib.ts
View file

@ -10,10 +10,10 @@ import * as jsonschema from "jsonschema";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { EnvVar } from "./environment";
import * as fingerprints from "./fingerprints";
import { Logger } from "./logging";
import { parseRepositoryNwo, RepositoryNwo } from "./repository";
import { CODEQL_WORKFLOW_STARTED_AT } from "./shared-environment";
import * as util from "./util";
import { SarifFile, SarifResult, SarifRun, wrapError } from "./util";
import * as workflow from "./workflow";
@ -287,7 +287,7 @@ export function buildPayload(
workflow_run_attempt: workflowRunAttempt,
checkout_uri: checkoutURI,
environment,
started_at: process.env[CODEQL_WORKFLOW_STARTED_AT],
started_at: process.env[EnvVar.WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined as undefined | string,
base_sha: undefined as undefined | string,

44
src/util.test.ts
View file

@ -8,8 +8,14 @@ import * as sinon from "sinon";
import * as api from "./api-client";
import { Config } from "./config-utils";
import { Feature } from "./feature-flags";
import { getRunnerLogger } from "./logging";
import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
import {
createFeatures,
getRecordingLogger,
LoggedMessage,
setupTests,
} from "./testing-utils";
import * as util from "./util";
setupTests(test);
@ -23,25 +29,37 @@ test("getToolNames", (t) => {
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
test("getMemoryFlag() should return the correct --ram flag", (t) => {
const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
test("getMemoryFlag() should return the correct --ram flag", async (t) => {
const totalMem = os.totalmem() / (1024 * 1024);
const fixedAmount = process.platform === "win32" ? 1536 : 1024;
const scaledAmount = 0.02 * totalMem;
const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
const expectedMemoryValueWithScaling = Math.floor(
totalMem - fixedAmount - scaledAmount
);
const tests: Array<[string | undefined, string]> = [
[undefined, `--ram=${totalMem - expectedThreshold}`],
["", `--ram=${totalMem - expectedThreshold}`],
["512", "--ram=512"],
const tests: Array<[string | undefined, boolean, string]> = [
[undefined, false, `--ram=${expectedMemoryValue}`],
["", false, `--ram=${expectedMemoryValue}`],
["512", false, "--ram=512"],
[undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
["", true, `--ram=${expectedMemoryValueWithScaling}`],
];
for (const [input, expectedFlag] of tests) {
const flag = util.getMemoryFlag(input);
for (const [input, withScaling, expectedFlag] of tests) {
const features = createFeatures(
withScaling ? [Feature.ScalingReservedRam] : []
);
const flag = await util.getMemoryFlag(input, features);
t.deepEqual(flag, expectedFlag);
}
});
test("getMemoryFlag() throws if the ram input is < 0 or NaN", (t) => {
test("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
for (const input of ["-1", "hello!"]) {
t.throws(() => util.getMemoryFlag(input));
await t.throwsAsync(
async () => await util.getMemoryFlag(input, createFeatures([]))
);
}
});
@ -196,7 +214,7 @@ function mockGetMetaVersionHeader(
},
};
const spyGetContents = sinon
.stub(client.meta, "get")
.stub(client.rest.meta, "get")
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
.resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);

54
src/util.ts
View file

@ -16,14 +16,10 @@ import {
parsePacksSpecification,
prettyPrintPack,
} from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { Logger } from "./logging";
import {
CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX,
CODEQL_ACTION_TEST_MODE,
EnvVar,
} from "./shared-environment";
/**
* Specifies bundle versions that are known to be broken
@ -161,9 +157,21 @@ export async function withTmpDir<T>(
* from committing too much of the available memory to CodeQL.
* @returns number
*/
function getSystemReservedMemoryMegaBytes(): number {
async function getSystemReservedMemoryMegaBytes(
totalMemoryMegaBytes: number,
features: FeatureEnablement
): Promise<number> {
// Windows needs more memory for OS processes.
return 1024 * (process.platform === "win32" ? 1.5 : 1);
const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
if (await features.getValue(Feature.ScalingReservedRam)) {
// Reserve an additional 2% of the total memory, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount = 0.02 * totalMemoryMegaBytes;
return fixedAmount + scaledAmount;
} else {
return fixedAmount;
}
}
/**
@ -173,7 +181,10 @@ function getSystemReservedMemoryMegaBytes(): number {
*
* @returns {number} the amount of RAM to use, in megabytes
*/
export function getMemoryFlagValue(userInput: string | undefined): number {
export async function getMemoryFlagValue(
userInput: string | undefined,
features: FeatureEnablement
): Promise<number> {
let memoryToUseMegaBytes: number;
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
@ -183,7 +194,10 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
} else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
const reservedMemoryMegaBytes = await getSystemReservedMemoryMegaBytes(
totalMemoryMegaBytes,
features
);
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
}
return Math.floor(memoryToUseMegaBytes);
@ -196,8 +210,12 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
*
* @returns string
*/
export function getMemoryFlag(userInput: string | undefined): string {
return `--ram=${getMemoryFlagValue(userInput)}`;
export async function getMemoryFlag(
userInput: string | undefined,
features: FeatureEnablement
): Promise<string> {
const megabytes = await getMemoryFlagValue(userInput, features);
return `--ram=${megabytes}`;
}
/**
@ -348,7 +366,7 @@ export async function getGitHubVersion(
// Doesn't strictly have to be the meta endpoint as we're only
// using the response headers which are available on every request.
const apiClient = getApiClient();
const response = await apiClient.meta.get();
const response = await apiClient.rest.meta.get();
// This happens on dotcom, although we expect to have already returned in that
// case. This can also serve as a fallback in cases we haven't foreseen.
@ -568,12 +586,8 @@ export async function getMlPoweredJsQueriesPack(
let version;
if (await codeQlVersionAbove(codeQL, "2.11.3")) {
version = "~0.4.0";
} else if (await codeQlVersionAbove(codeQL, "2.9.3")) {
version = `~0.3.0`;
} else if (await codeQlVersionAbove(codeQL, "2.8.4")) {
version = `~0.2.0`;
} else {
version = `~0.1.0`;
version = `~0.3.0`;
}
return prettyPrintPack({
name: ML_POWERED_JS_QUERIES_PACK_NAME,
@ -630,7 +644,7 @@ export function getMlPoweredJsQueriesStatus(config: Config): string {
* In test mode, we don't upload SARIF results or status reports to the GitHub API.
*/
export function isInTestMode(): boolean {
return process.env[CODEQL_ACTION_TEST_MODE] === "true";
return process.env[EnvVar.TEST_MODE] === "true";
}
/**
@ -894,10 +908,10 @@ export function fixInvalidNotificationsInFile(
outputPath: string,
logger: Logger
): void {
if (process.env[CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
if (process.env[EnvVar.DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`
`${EnvVar.DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`
);
fs.renameSync(inputPath, outputPath);
} else {

4
src/workflow.ts
View file

@ -6,6 +6,7 @@ import * as core from "@actions/core";
import * as yaml from "js-yaml";
import * as api from "./api-client";
import { EnvVar } from "./environment";
import { Logger } from "./logging";
import { getRequiredEnvParam, isInTestMode } from "./util";
@ -391,8 +392,7 @@ function getInputOrThrow(
function getAnalyzeActionName() {
if (
isInTestMode() ||
process.env["CODEQL_ACTION_TESTING_ENVIRONMENT"] ===
"codeql-action-pr-checks"
process.env[EnvVar.TESTING_ENVIRONMENT] === "codeql-action-pr-checks"
) {
return "./analyze";
} else {