robojerk/codeql-action

Add more documentation for ML-powered JS queries status report

Browse source 
Also be more explicit about which version strings are reportable in
the code.
This commit is contained in:
Henry Mercer 2022-02-07 16:46:32 +00:00
parent cc622a02a9
commit 03c64ef07d
9 changed files with 84 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

7
lib/config-utils.js generated
View file

@ -130,16 +130,13 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
// pack, then add the ML-powered query pack so that we run ML-powered queries. // pack, then add the ML-powered query pack so that we run ML-powered queries.
if (languages.includes("javascript") && if (languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") && (found === "security-extended" || found === "security-and-quality") &&
!((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK_NAME)) && !((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK.packName)) &&
(await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) && (await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) &&
(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) { (await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) {
if (!packs.javascript) { if (!packs.javascript) {
packs.javascript = []; packs.javascript = [];
} }
packs.javascript.push({ packs.javascript.push(util_1.ML_POWERED_JS_QUERIES_PACK);
packName: util_1.ML_POWERED_JS_QUERIES_PACK_NAME,
version: "~0.0.2",
});
} }
const suites = languages.map((l) => `${l}-${suiteName}.qls`); const suites = languages.map((l) => `${l}-${suiteName}.qls`);
await runResolveQueries(codeQL, resultMap, suites, undefined); await runResolveQueries(codeQL, resultMap, suites, undefined);

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

44
lib/util.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod }; return (mod && mod.__esModule) ? mod : { "default": mod };
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.getMlPoweredJsQueriesStatus = exports.ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.checkNotWindows11 = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0; exports.getMlPoweredJsQueriesStatus = exports.ML_POWERED_JS_QUERIES_PACK = exports.checkNotWindows11 = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs")); const fs = __importStar(require("fs"));
const os = __importStar(require("os")); const os = __importStar(require("os"));
const path = __importStar(require("path")); const path = __importStar(require("path"));
@ -524,39 +524,55 @@ function checkNotWindows11() {
} }
} }
exports.checkNotWindows11 = checkNotWindows11; exports.checkNotWindows11 = checkNotWindows11;
exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-queries";
/** /**
* Set containing version strings of the ML-powered JS query pack that are reportable. * The ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* * queries beta.
* We restrict the set of version strings we report to limit the cardinality of the ML-powered JS
* queries status report field, since some platforms that ingest this status report charge based on
* the cardinality of the fields.
*/ */
exports.ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS = new Set(["~0.0.2"]); exports.ML_POWERED_JS_QUERIES_PACK = {
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
};
/** /**
* Get information about ML-powered JS queries to populate status reports with. * Get information about ML-powered JS queries to populate status reports with.
* *
* This will be: * This will be:
* *
* - The version string if the analysis will use a specific version of the pack and that version * - The version string if the analysis is using the ML-powered query pack that will be added to the
* string is within {@link ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS}. * analysis if the repo is opted into the ML-powered queries beta, i.e.
* {@link ML_POWERED_JS_QUERIES_PACK.version}. If the version string
* {@link ML_POWERED_JS_QUERIES_PACK.version} is undefined, then the status report string will be
* "latest", however this shouldn't occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries. * - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases. * - "other" in all other cases.
* *
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings.
*
* We restrict the set of strings we report here by excluding other version strings and combinations
* of version strings. We do this to limit the cardinality of the ML-powered JS queries status
* report field, since some platforms that ingest this status report bill based on the cardinality
* of its fields.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for * This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an * `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is. * explanation as to why this is.
*/ */
function getMlPoweredJsQueriesStatus(config) { function getMlPoweredJsQueriesStatus(config) {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK_NAME); const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK.packName);
if (mlPoweredJsQueryPacks.length === 0) { if (mlPoweredJsQueryPacks.length === 0) {
return "false"; return "false";
} }
const firstVersionString = mlPoweredJsQueryPacks[0].version; const firstVersionString = mlPoweredJsQueryPacks[0].version;
if (mlPoweredJsQueryPacks.length === 1 && if (mlPoweredJsQueryPacks.length === 1 &&
firstVersionString && exports.ML_POWERED_JS_QUERIES_PACK.version === firstVersionString) {
exports.ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS.has(firstVersionString)) { // We should always specify an explicit version string in `ML_POWERED_JS_QUERIES_PACK`,
return firstVersionString; // otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice, we should never hit the
// `latest` case here.
return exports.ML_POWERED_JS_QUERIES_PACK.version || "latest";
} }
return "other"; return "other";
} }

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

22
lib/util.test.js generated
View file

@ -208,32 +208,26 @@ const ML_POWERED_JS_STATUS_TESTS = [
[[], "false"], [[], "false"],
[[{ packName: "someOtherPack" }], "false"], [[{ packName: "someOtherPack" }], "false"],
[ [
[ [{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK],
{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK.version,
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.2" },
],
"~0.0.2",
], ],
[[util.ML_POWERED_JS_QUERIES_PACK], util.ML_POWERED_JS_QUERIES_PACK.version],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }], "other"],
[ [
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.2" }], [{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "~0.0.1" }],
"~0.0.2",
],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "other"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.1" }],
"other", "other",
], ],
[ [
[ [
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" }, { packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" }, { packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.2" },
], ],
"other", "other",
], ],
[ [
[ [
{ packName: "someOtherPack" }, { packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }, { packName: util.ML_POWERED_JS_QUERIES_PACK.packName },
], ],
"other", "other",
], ],

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long

9
src/config-utils.ts
View file

@ -18,7 +18,7 @@ import { RepositoryNwo } from "./repository";
import { import {
codeQlVersionAbove, codeQlVersionAbove,
GitHubVersion, GitHubVersion,
ML_POWERED_JS_QUERIES_PACK_NAME, ML_POWERED_JS_QUERIES_PACK,
} from "./util"; } from "./util";
// Property names from the user-supplied config file. // Property names from the user-supplied config file.
@ -296,7 +296,7 @@ async function addBuiltinSuiteQueries(
languages.includes("javascript") && languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") && (found === "security-extended" || found === "security-and-quality") &&
!packs.javascript?.some( !packs.javascript?.some(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME (pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK.packName
) && ) &&
(await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) && (await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) &&
(await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES)) (await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES))
@ -304,10 +304,7 @@ async function addBuiltinSuiteQueries(
if (!packs.javascript) { if (!packs.javascript) {
packs.javascript = []; packs.javascript = [];
} }
packs.javascript.push({ packs.javascript.push(ML_POWERED_JS_QUERIES_PACK);
packName: ML_POWERED_JS_QUERIES_PACK_NAME,
version: "~0.0.2",
});
} }
const suites = languages.map((l) => `${l}-${suiteName}.qls`); const suites = languages.map((l) => `${l}-${suiteName}.qls`);

22
src/util.test.ts
View file

@ -297,32 +297,26 @@ const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [
[[], "false"], [[], "false"],
[[{ packName: "someOtherPack" }], "false"], [[{ packName: "someOtherPack" }], "false"],
[ [
[ [{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK],
{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK.version!,
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.2" },
],
"~0.0.2",
], ],
[[util.ML_POWERED_JS_QUERIES_PACK], util.ML_POWERED_JS_QUERIES_PACK.version!],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }], "other"],
[ [
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.2" }], [{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "~0.0.1" }],
"~0.0.2",
],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "other"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.1" }],
"other", "other",
], ],
[ [
[ [
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" }, { packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" }, { packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.2" },
], ],
"other", "other",
], ],
[ [
[ [
{ packName: "someOtherPack" }, { packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }, { packName: util.ML_POWERED_JS_QUERIES_PACK.packName },
], ],
"other", "other",
], ],

46
src/util.ts
View file

@ -10,7 +10,7 @@ import * as semver from "semver";
import { getApiClient, GitHubApiDetails } from "./api-client"; import { getApiClient, GitHubApiDetails } from "./api-client";
import * as apiCompatibility from "./api-compatibility.json"; import * as apiCompatibility from "./api-compatibility.json";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql"; import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import { Config } from "./config-utils"; import { Config, PackWithVersion } from "./config-utils";
import { Language } from "./languages"; import { Language } from "./languages";
import { Logger } from "./logging"; import { Logger } from "./logging";
@ -628,35 +628,46 @@ export function checkNotWindows11() {
} }
} }
export const ML_POWERED_JS_QUERIES_PACK_NAME =
"codeql/javascript-experimental-atm-queries";
/** /**
* Set containing version strings of the ML-powered JS query pack that are reportable. * The ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* * queries beta.
* We restrict the set of version strings we report to limit the cardinality of the ML-powered JS
* queries status report field, since some platforms that ingest this status report charge based on
* the cardinality of the fields.
*/ */
export const ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS = new Set(["~0.0.2"]); export const ML_POWERED_JS_QUERIES_PACK: PackWithVersion = {
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
};
/** /**
* Get information about ML-powered JS queries to populate status reports with. * Get information about ML-powered JS queries to populate status reports with.
* *
* This will be: * This will be:
* *
* - The version string if the analysis will use a specific version of the pack and that version * - The version string if the analysis is using the ML-powered query pack that will be added to the
* string is within {@link ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS}. * analysis if the repo is opted into the ML-powered queries beta, i.e.
* {@link ML_POWERED_JS_QUERIES_PACK.version}. If the version string
* {@link ML_POWERED_JS_QUERIES_PACK.version} is undefined, then the status report string will be
* "latest", however this shouldn't occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries. * - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases. * - "other" in all other cases.
* *
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings.
*
* We restrict the set of strings we report here by excluding other version strings and combinations
* of version strings. We do this to limit the cardinality of the ML-powered JS queries status
* report field, since some platforms that ingest this status report bill based on the cardinality
* of its fields.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for * This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an * `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is. * explanation as to why this is.
*/ */
export function getMlPoweredJsQueriesStatus(config: Config): string { export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter( const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME (pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK.packName
); );
if (mlPoweredJsQueryPacks.length === 0) { if (mlPoweredJsQueryPacks.length === 0) {
return "false"; return "false";
@ -664,10 +675,13 @@ export function getMlPoweredJsQueriesStatus(config: Config): string {
const firstVersionString = mlPoweredJsQueryPacks[0].version; const firstVersionString = mlPoweredJsQueryPacks[0].version;
if ( if (
mlPoweredJsQueryPacks.length === 1 && mlPoweredJsQueryPacks.length === 1 &&
firstVersionString && ML_POWERED_JS_QUERIES_PACK.version === firstVersionString
ML_POWERED_JS_QUERIES_REPORTABLE_VERSIONS.has(firstVersionString)
) { ) {
return firstVersionString; // We should always specify an explicit version string in `ML_POWERED_JS_QUERIES_PACK`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice, we should never hit the
// `latest` case here.
return ML_POWERED_JS_QUERIES_PACK.version || "latest";
} }
return "other"; return "other";
} }