robojerk/codeql-action

Merge pull request #907 from github/henrymercer/report-ml-powered-query-enablement

Browse source 
Report ML-powered query enablement in the `init` status report
This commit is contained in:
Henry Mercer 2022-02-07 17:16:25 +00:00 • committed by GitHub
commit 4eb03fb6f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 398 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

13
lib/config-utils.js generated
View file

@ -120,24 +120,23 @@ const builtinSuites = ["security-extended", "security-and-quality"];
* Throws an error if suiteName is not a valid builtin suite.
*/
async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suiteName, featureFlags, configFile) {
var _a;
const found = builtinSuites.find((suite) => suite === suiteName);
if (!found) {
throw new Error(getQueryUsesInvalid(configFile, suiteName));
}
// If we're running the JavaScript security-extended analysis (or a superset of it) and the repo
// is opted into the ML-powered queries beta, then add the ML-powered query pack so that we run
// the ML-powered queries.
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries.
if (languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") &&
!((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK.packName)) &&
(await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) &&
(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) {
if (!packs.javascript) {
packs.javascript = [];
}
packs.javascript.push({
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
});
packs.javascript.push(util_1.ML_POWERED_JS_QUERIES_PACK);
}
const suites = languages.map((l) => `${l}-${suiteName}.qls`);
await runResolveQueries(codeQL, resultMap, suites, undefined);

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

30
lib/config-utils.test.js generated
View file

@ -872,7 +872,7 @@ parseInputAndConfigErrorMacro.title = (providedTitle) => `Parse Packs input and
(0, ava_1.default)("input with + only", parseInputAndConfigErrorMacro, {}, " + ", [languages_1.Language.cpp], /remove the '\+'/);
(0, ava_1.default)("input with invalid pack name", parseInputAndConfigErrorMacro, {}, " xxx", [languages_1.Language.cpp], /"xxx" is not a valid pack/);
const mlPoweredQueriesMacro = ava_1.default.macro({
exec: async (t, codeQLVersion, isMlPoweredQueriesFlagEnabled, queriesInput, shouldRunMlPoweredQueries) => {
exec: async (t, codeQLVersion, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, expectedVersionString) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = (0, codeql_1.setCodeQL)({
async getVersion() {
@ -888,15 +888,15 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
};
},
});
const { packs } = await configUtils.initConfig("javascript", queriesInput, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)(isMlPoweredQueriesFlagEnabled
const { packs } = await configUtils.initConfig("javascript", queriesInput, packsInput, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)(isMlPoweredQueriesFlagEnabled
? [feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled]
: []), (0, logging_1.getRunnerLogger)(true));
if (shouldRunMlPoweredQueries) {
if (expectedVersionString !== undefined) {
t.deepEqual(packs, {
[languages_1.Language.javascript]: [
{
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
version: expectedVersionString,
},
],
});
@ -906,17 +906,15 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
}
});
},
title: (_providedTitle, codeQLVersion, isMlPoweredQueriesFlagEnabled, queriesInput, shouldRunMlPoweredQueries) => {
const queriesInputDescription = queriesInput
? `'queries: ${queriesInput}'`
: "default config";
return `ML-powered queries ${shouldRunMlPoweredQueries ? "are" : "aren't"} loaded for ${queriesInputDescription} using CLI v${codeQLVersion} when feature flag is ${isMlPoweredQueriesFlagEnabled ? "enabled" : "disabled"}`;
},
title: (_providedTitle, codeQLVersion, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, expectedVersionString) => `ML-powered queries ${expectedVersionString !== undefined
? `${expectedVersionString} are`
: "aren't"} loaded for packs: ${packsInput}, queries: ${queriesInput} using CLI v${codeQLVersion} when feature flag is ${isMlPoweredQueriesFlagEnabled ? "enabled" : "disabled"}`,
});
// macro, isMlPoweredQueriesFlagEnabled, queriesInput, shouldRunMlPoweredQueries
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.4", true, "security-extended", false);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", false, "security-extended", false);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, false);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, "security-extended", true);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, "security-and-quality", true);
// macro, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, versionString
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.4", true, undefined, "security-extended", undefined);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", false, undefined, "security-extended", undefined);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-extended", "~0.0.2");
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-and-quality", "~0.0.2");
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, "codeql/javascript-experimental-atm-queries@0.0.1", "security-and-quality", "0.0.1");
//# sourceMappingURL=config-utils.test.js.map

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

5
lib/init-action.js generated
View file

@ -54,14 +54,15 @@ async function sendSuccessStatusReport(startedAt, config, toolsVersion) {
}
const statusReport = {
...statusReportBase,
disable_default_queries: disableDefaultQueries,
languages,
workflow_languages: workflowLanguages || "",
ml_powered_js_queries: (0, util_1.getMlPoweredJsQueriesStatus)(config),
paths,
paths_ignore: pathsIgnore,
disable_default_queries: disableDefaultQueries,
queries: queries.join(","),
tools_input: (0, actions_util_1.getOptionalInput)("tools") || "",
tools_resolved_version: toolsVersion,
workflow_languages: workflowLanguages || "",
};
await (0, actions_util_1.sendStatusReport)(statusReport);
}

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

55
lib/util.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.checkNotWindows11 = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.getMlPoweredJsQueriesStatus = exports.ML_POWERED_JS_QUERIES_PACK = exports.checkNotWindows11 = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@ -524,4 +524,57 @@ function checkNotWindows11() {
}
}
exports.checkNotWindows11 = checkNotWindows11;
/**
* The ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* queries beta.
*/
exports.ML_POWERED_JS_QUERIES_PACK = {
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
};
/**
* Get information about ML-powered JS queries to populate status reports with.
*
* This will be:
*
* - The version string if the analysis is using the ML-powered query pack that will be added to the
* analysis if the repo is opted into the ML-powered queries beta, i.e.
* {@link ML_POWERED_JS_QUERIES_PACK.version}. If the version string
* {@link ML_POWERED_JS_QUERIES_PACK.version} is undefined, then the status report string will be
* "latest", however this shouldn't occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases.
*
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings.
*
* We restrict the set of strings we report here by excluding other version strings and combinations
* of version strings. We do this to limit the cardinality of the ML-powered JS queries status
* report field, since some platforms that ingest this status report bill based on the cardinality
* of its fields.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is.
*/
function getMlPoweredJsQueriesStatus(config) {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK.packName);
if (mlPoweredJsQueryPacks.length === 0) {
return "false";
}
const firstVersionString = mlPoweredJsQueryPacks[0].version;
if (mlPoweredJsQueryPacks.length === 1 &&
exports.ML_POWERED_JS_QUERIES_PACK.version === firstVersionString) {
// We should always specify an explicit version string in `ML_POWERED_JS_QUERIES_PACK`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice, we should never hit the
// `latest` case here.
return exports.ML_POWERED_JS_QUERIES_PACK.version || "latest";
}
return "other";
}
exports.getMlPoweredJsQueriesStatus = getMlPoweredJsQueriesStatus;
//# sourceMappingURL=util.js.map

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

58
lib/util.test.js generated
View file

@ -204,4 +204,62 @@ async function mockStdInForAuthExpectError(t, mockLogger, ...text) {
const stdin = stream.Readable.from(text);
await t.throwsAsync(async () => util.getGitHubAuth(mockLogger, undefined, true, stdin));
}
const ML_POWERED_JS_STATUS_TESTS = [
[[], "false"],
[[{ packName: "someOtherPack" }], "false"],
[
[{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK],
util.ML_POWERED_JS_QUERIES_PACK.version,
],
[[util.ML_POWERED_JS_QUERIES_PACK], util.ML_POWERED_JS_QUERIES_PACK.version],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }], "other"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "~0.0.1" }],
"other",
],
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.2" },
],
"other",
],
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName },
],
"other",
],
];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
const packDescriptions = `[${packs
.map((pack) => JSON.stringify(pack))
.join(", ")}]`;
(0, ava_1.default)(`ML-powered JS queries status report is "${expectedStatus}" for packs = ${packDescriptions}`, (t) => {
return util.withTmpDir(async (tmpDir) => {
const config = {
languages: [],
queries: {},
paths: [],
pathsIgnore: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
},
dbLocation: "",
packs: {
javascript: packs,
},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);
});
});
}
//# sourceMappingURL=util.test.js.map

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long

32
src/actions-util.ts
View file

@ -548,37 +548,37 @@ type ActionName = "init" | "autobuild" | "finish" | "upload-sarif";
type ActionStatus = "starting" | "aborted" | "success" | "failure";
export interface StatusReportBase {
// ID of the workflow run containing the action run
/** ID of the workflow run containing the action run. */
workflow_run_id: number;
// Workflow name. Converted to analysis_name further down the pipeline.
/** Workflow name. Converted to analysis_name further down the pipeline.. */
workflow_name: string;
// Job name from the workflow
/** Job name from the workflow. */
job_name: string;
// Analysis key, normally composed from the workflow path and job name
/** Analysis key, normally composed from the workflow path and job name. */
analysis_key: string;
// Value of the matrix for this instantiation of the job
/** Value of the matrix for this instantiation of the job. */
matrix_vars?: string;
// Commit oid that the workflow was triggered on
/** Commit oid that the workflow was triggered on. */
commit_oid: string;
// Ref that the workflow was triggered on
/** Ref that the workflow was triggered on. */
ref: string;
// Name of the action being executed
/** Name of the action being executed. */
action_name: ActionName;
// Version of the action being executed, as a ref
/** Version of the action being executed, as a ref. */
action_ref?: string;
// Version of the action being executed, as a commit oid
/** Version of the action being executed, as a commit oid. */
action_oid: string;
// Time the first action started. Normally the init action
/** Time the first action started. Normally the init action. */
started_at: string;
// Time this action started
/** Time this action started. */
action_started_at: string;
// Time this action completed, or undefined if not yet completed
/** Time this action completed, or undefined if not yet completed. */
completed_at?: string;
// State this action is currently in
/** State this action is currently in. */
status: ActionStatus;
// Cause of the failure (or undefined if status is not failure)
/** Cause of the failure (or undefined if status is not failure). */
cause?: string;
// Stack trace of the failure (or undefined if status is not failure)
/** Stack trace of the failure (or undefined if status is not failure). */
exception?: string;
}

44
src/analyze.ts
View file

@ -25,49 +25,49 @@ export class CodeQLAnalysisError extends Error {
}
export interface QueriesStatusReport {
// Time taken in ms to run builtin queries for cpp (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for cpp (or undefined if this language was not analyzed). */
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to run builtin queries for csharp (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for csharp (or undefined if this language was not analyzed). */
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to run builtin queries for go (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for go (or undefined if this language was not analyzed). */
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to run builtin queries for java (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for java (or undefined if this language was not analyzed). */
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to run builtin queries for javascript (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for javascript (or undefined if this language was not analyzed). */
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to run builtin queries for python (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for python (or undefined if this language was not analyzed). */
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to run builtin queries for ruby (or undefined if this language was not analyzed)
/** Time taken in ms to run builtin queries for ruby (or undefined if this language was not analyzed). */
analyze_builtin_queries_ruby_duration_ms?: number;
// Time taken in ms to run custom queries for cpp (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for cpp (or undefined if this language was not analyzed). */
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to run custom queries for csharp (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for csharp (or undefined if this language was not analyzed). */
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to run custom queries for go (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for go (or undefined if this language was not analyzed). */
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to run custom queries for java (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for java (or undefined if this language was not analyzed). */
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to run custom queries for javascript (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for javascript (or undefined if this language was not analyzed). */
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to run custom queries for python (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for python (or undefined if this language was not analyzed). */
analyze_custom_queries_python_duration_ms?: number;
// Time taken in ms to run custom queries for ruby (or undefined if this language was not analyzed)
/** Time taken in ms to run custom queries for ruby (or undefined if this language was not analyzed). */
analyze_custom_queries_ruby_duration_ms?: number;
// Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed). */
interpret_results_cpp_duration_ms?: number;
// Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed). */
interpret_results_csharp_duration_ms?: number;
// Time taken in ms to interpret results for go (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for go (or undefined if this language was not analyzed). */
interpret_results_go_duration_ms?: number;
// Time taken in ms to interpret results for java (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for java (or undefined if this language was not analyzed). */
interpret_results_java_duration_ms?: number;
// Time taken in ms to interpret results for javascript (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for javascript (or undefined if this language was not analyzed). */
interpret_results_javascript_duration_ms?: number;
// Time taken in ms to interpret results for python (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for python (or undefined if this language was not analyzed). */
interpret_results_python_duration_ms?: number;
// Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed)
/** Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed). */
interpret_results_ruby_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no language failed)
/** Name of language that errored during analysis (or undefined if no language failed). */
analyze_failure_language?: string;
}

4
src/autobuild-action.ts
View file

@ -16,9 +16,9 @@ import { initializeEnvironment, Mode } from "./util";
const pkg = require("../package.json");
interface AutobuildStatusReport extends StatusReportBase {
// Comma-separated set of languages being auto-built
/** Comma-separated set of languages being auto-built. */
autobuild_languages: string;
// Language that failed autobuilding (or undefined if all languages succeeded).
/** Language that failed autobuilding (or undefined if all languages succeeded). */
autobuild_failure?: string;
}

77
src/config-utils.test.ts
View file

@ -1713,8 +1713,9 @@ const mlPoweredQueriesMacro = test.macro({
t: ExecutionContext,
codeQLVersion: string,
isMlPoweredQueriesFlagEnabled: boolean,
packsInput: string | undefined,
queriesInput: string | undefined,
shouldRunMlPoweredQueries: boolean
expectedVersionString: string | undefined
) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
@ -1735,7 +1736,7 @@ const mlPoweredQueriesMacro = test.macro({
const { packs } = await configUtils.initConfig(
"javascript",
queriesInput,
undefined,
packsInput,
undefined,
undefined,
false,
@ -1755,12 +1756,12 @@ const mlPoweredQueriesMacro = test.macro({
),
getRunnerLogger(true)
);
if (shouldRunMlPoweredQueries) {
if (expectedVersionString !== undefined) {
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
version: expectedVersionString,
},
],
});
@ -1773,24 +1774,58 @@ const mlPoweredQueriesMacro = test.macro({
_providedTitle: string | undefined,
codeQLVersion: string,
isMlPoweredQueriesFlagEnabled: boolean,
packsInput: string | undefined,
queriesInput: string | undefined,
shouldRunMlPoweredQueries: boolean
) => {
const queriesInputDescription = queriesInput
? `'queries: ${queriesInput}'`
: "default config";
return `ML-powered queries ${
shouldRunMlPoweredQueries ? "are" : "aren't"
} loaded for ${queriesInputDescription} using CLI v${codeQLVersion} when feature flag is ${
expectedVersionString: string | undefined
) =>
`ML-powered queries ${
expectedVersionString !== undefined
? `${expectedVersionString} are`
: "aren't"
} loaded for packs: ${packsInput}, queries: ${queriesInput} using CLI v${codeQLVersion} when feature flag is ${
isMlPoweredQueriesFlagEnabled ? "enabled" : "disabled"
}`;
},
}`,
});
// macro, isMlPoweredQueriesFlagEnabled, queriesInput, shouldRunMlPoweredQueries
test(mlPoweredQueriesMacro, "2.7.4", true, "security-extended", false);
test(mlPoweredQueriesMacro, "2.7.5", false, "security-extended", false);
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, false);
test(mlPoweredQueriesMacro, "2.7.5", true, "security-extended", true);
test(mlPoweredQueriesMacro, "2.7.5", true, "security-and-quality", true);
// macro, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, versionString
test(
mlPoweredQueriesMacro,
"2.7.4",
true,
undefined,
"security-extended",
undefined
);
test(
mlPoweredQueriesMacro,
"2.7.5",
false,
undefined,
"security-extended",
undefined
);
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
test(
mlPoweredQueriesMacro,
"2.7.5",
true,
undefined,
"security-extended",
"~0.0.2"
);
test(
mlPoweredQueriesMacro,
"2.7.5",
true,
undefined,
"security-and-quality",
"~0.0.2"
);
test(
mlPoweredQueriesMacro,
"2.7.5",
true,
"codeql/javascript-experimental-atm-queries@0.0.1",
"security-and-quality",
"0.0.1"
);

20
src/config-utils.ts
View file

@ -15,7 +15,11 @@ import { FeatureFlag, FeatureFlags } from "./feature-flags";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { codeQlVersionAbove, GitHubVersion } from "./util";
import {
codeQlVersionAbove,
GitHubVersion,
ML_POWERED_JS_QUERIES_PACK,
} from "./util";
// Property names from the user-supplied config file.
const NAME_PROPERTY = "name";
@ -285,22 +289,22 @@ async function addBuiltinSuiteQueries(
throw new Error(getQueryUsesInvalid(configFile, suiteName));
}
// If we're running the JavaScript security-extended analysis (or a superset of it) and the repo
// is opted into the ML-powered queries beta, then add the ML-powered query pack so that we run
// the ML-powered queries.
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries.
if (
languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") &&
!packs.javascript?.some(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK.packName
) &&
(await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) &&
(await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES))
) {
if (!packs.javascript) {
packs.javascript = [];
}
packs.javascript.push({
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
});
packs.javascript.push(ML_POWERED_JS_QUERIES_PACK);
}
const suites = languages.map((l) => `${l}-${suiteName}.qls`);

41
src/init-action.ts
View file

@ -38,29 +38,39 @@ import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
checkNotWindows11,
getMlPoweredJsQueriesStatus,
} from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
interface InitSuccessStatusReport extends StatusReportBase {
// Comma-separated list of languages that analysis was run for
// This may be from the workflow file or may be calculated from repository contents
languages: string;
// Comma-separated list of languages specified explicitly in the workflow file
workflow_languages: string;
// Comma-separated list of paths, from the 'paths' config field
paths: string;
// Comma-separated list of paths, from the 'paths-ignore' config field
paths_ignore: string;
// Commas-separated list of languages where the default queries are disabled
/** Comma-separated list of languages where the default queries are disabled. */
disable_default_queries: string;
// Comma-separated list of queries sources, from the 'queries' config field or workflow input
/**
* Comma-separated list of languages that analysis was run for.
*
* This may be from the workflow file or may be calculated from repository contents
*/
languages: string;
/**
* Information about the enablement of the ML-powered JS query pack.
*
* @see {@link getMlPoweredJsQueriesStatus}
*/
ml_powered_js_queries: string;
/** Comma-separated list of paths, from the 'paths' config field. */
paths: string;
/** Comma-separated list of paths, from the 'paths-ignore' config field. */
paths_ignore: string;
/** Comma-separated list of queries sources, from the 'queries' config field or workflow input. */
queries: string;
// Value given by the user as the "tools" input
/** Value given by the user as the "tools" input. */
tools_input: string;
// Version of the bundle used
/** Version of the bundle used. */
tools_resolved_version: string;
/** Comma-separated list of languages specified explicitly in the workflow file. */
workflow_languages: string;
}
async function sendSuccessStatusReport(
@ -102,14 +112,15 @@ async function sendSuccessStatusReport(
const statusReport: InitSuccessStatusReport = {
...statusReportBase,
disable_default_queries: disableDefaultQueries,
languages,
workflow_languages: workflowLanguages || "",
ml_powered_js_queries: getMlPoweredJsQueriesStatus(config),
paths,
paths_ignore: pathsIgnore,
disable_default_queries: disableDefaultQueries,
queries: queries.join(","),
tools_input: getOptionalInput("tools") || "",
tools_resolved_version: toolsVersion,
workflow_languages: workflowLanguages || "",
};
await sendStatusReport(statusReport);

6
src/upload-lib.ts
View file

@ -122,11 +122,11 @@ async function uploadPayload(
}
export interface UploadStatusReport {
// Size in bytes of unzipped SARIF upload
/** Size in bytes of unzipped SARIF upload. */
raw_upload_size_bytes?: number;
// Size in bytes of actual SARIF upload
/** Size in bytes of actual SARIF upload. */
zipped_upload_size_bytes?: number;
// Number of results in the SARIF upload
/** Number of results in the SARIF upload. */
num_results_in_sarif?: number;
}

62
src/util.test.ts
View file

@ -7,6 +7,7 @@ import test, { ExecutionContext } from "ava";
import * as sinon from "sinon";
import * as api from "./api-client";
import { Config, PackWithVersion } from "./config-utils";
import { getRunnerLogger, Logger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
@ -291,3 +292,64 @@ async function mockStdInForAuthExpectError(
util.getGitHubAuth(mockLogger, undefined, true, stdin)
);
}
const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [
[[], "false"],
[[{ packName: "someOtherPack" }], "false"],
[
[{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK],
util.ML_POWERED_JS_QUERIES_PACK.version!,
],
[[util.ML_POWERED_JS_QUERIES_PACK], util.ML_POWERED_JS_QUERIES_PACK.version!],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }], "other"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "~0.0.1" }],
"other",
],
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.2" },
],
"other",
],
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName },
],
"other",
],
];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
const packDescriptions = `[${packs
.map((pack) => JSON.stringify(pack))
.join(", ")}]`;
test(`ML-powered JS queries status report is "${expectedStatus}" for packs = ${packDescriptions}`, (t) => {
return util.withTmpDir(async (tmpDir) => {
const config: Config = {
languages: [],
queries: {},
paths: [],
pathsIgnore: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {
javascript: packs,
},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);
});
});
}

60
src/util.ts
View file

@ -10,7 +10,7 @@ import * as semver from "semver";
import { getApiClient, GitHubApiDetails } from "./api-client";
import * as apiCompatibility from "./api-compatibility.json";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import { Config } from "./config-utils";
import { Config, PackWithVersion } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
@ -627,3 +627,61 @@ export function checkNotWindows11() {
);
}
}
/**
* The ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* queries beta.
*/
export const ML_POWERED_JS_QUERIES_PACK: PackWithVersion = {
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
};
/**
* Get information about ML-powered JS queries to populate status reports with.
*
* This will be:
*
* - The version string if the analysis is using the ML-powered query pack that will be added to the
* analysis if the repo is opted into the ML-powered queries beta, i.e.
* {@link ML_POWERED_JS_QUERIES_PACK.version}. If the version string
* {@link ML_POWERED_JS_QUERIES_PACK.version} is undefined, then the status report string will be
* "latest", however this shouldn't occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases.
*
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings.
*
* We restrict the set of strings we report here by excluding other version strings and combinations
* of version strings. We do this to limit the cardinality of the ML-powered JS queries status
* report field, since some platforms that ingest this status report bill based on the cardinality
* of its fields.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is.
*/
export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK.packName
);
if (mlPoweredJsQueryPacks.length === 0) {
return "false";
}
const firstVersionString = mlPoweredJsQueryPacks[0].version;
if (
mlPoweredJsQueryPacks.length === 1 &&
ML_POWERED_JS_QUERIES_PACK.version === firstVersionString
) {
// We should always specify an explicit version string in `ML_POWERED_JS_QUERIES_PACK`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice, we should never hit the
// `latest` case here.
return ML_POWERED_JS_QUERIES_PACK.version || "latest";
}
return "other";
}