Remove ML-powered queries
This commit is contained in:
Henry Mercer
parent
a370ce344f
commit
ebbadee09e
33 changed files with 60 additions and 836 deletions
97
lib/config-utils.js
generated
97
lib/config-utils.js
generated
|
|
@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.wrapEnvironment = exports.generateRegistries = exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.getMlPoweredJsQueriesStatus = exports.parsePacks = exports.validatePackSpecification = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguageAliases = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
|
||||
exports.wrapEnvironment = exports.generateRegistries = exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguageAliases = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const perf_hooks_1 = require("perf_hooks");
|
||||
|
|
@ -45,13 +45,12 @@ const PATHS_IGNORE_PROPERTY = "paths-ignore";
|
|||
const PATHS_PROPERTY = "paths";
|
||||
const PACKS_PROPERTY = "packs";
|
||||
/**
|
||||
* The default, empty augmentation properties. This is most useeful
|
||||
* The default, empty augmentation properties. This is most useful
|
||||
* for tests.
|
||||
*/
|
||||
exports.defaultAugmentationProperties = {
|
||||
queriesInputCombines: false,
|
||||
packsInputCombines: false,
|
||||
injectedMlQueries: false,
|
||||
packsInput: undefined,
|
||||
queriesInput: undefined,
|
||||
};
|
||||
|
|
@ -139,10 +138,8 @@ const builtinSuites = [
|
|||
/**
|
||||
* Determine the set of queries associated with suiteName's suites and add them to resultMap.
|
||||
* Throws an error if suiteName is not a valid builtin suite.
|
||||
* May inject ML queries, and the return value will declare if this was done.
|
||||
*/
|
||||
async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suiteName, features, configFile) {
|
||||
let injectedMlQueries = false;
|
||||
async function addBuiltinSuiteQueries(languages, codeQL, resultMap, suiteName, configFile) {
|
||||
const found = builtinSuites.find((suite) => suite === suiteName);
|
||||
if (!found) {
|
||||
throw new util_1.UserError(getQueryUsesInvalid(configFile, suiteName));
|
||||
|
|
@ -153,27 +150,8 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
|
|||
${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
|
||||
${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`);
|
||||
}
|
||||
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
|
||||
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
|
||||
// pack, then add the ML-powered query pack so that we run ML-powered queries.
|
||||
if (languages.includes("javascript") &&
|
||||
(found === "security-experimental" ||
|
||||
found === "security-extended" ||
|
||||
found === "security-and-quality") &&
|
||||
!packs.javascript?.some(isMlPoweredJsQueriesPack) &&
|
||||
(await features.getValue(feature_flags_1.Feature.MlPoweredQueriesEnabled, codeQL))) {
|
||||
if (!packs.javascript) {
|
||||
packs.javascript = [];
|
||||
}
|
||||
packs.javascript.push(await (0, util_1.getMlPoweredJsQueriesPack)(codeQL));
|
||||
injectedMlQueries = true;
|
||||
}
|
||||
const suites = languages.map((l) => `${l}-${suiteName}.qls`);
|
||||
await runResolveQueries(codeQL, resultMap, suites, undefined);
|
||||
return injectedMlQueries;
|
||||
}
|
||||
function isMlPoweredJsQueriesPack(pack) {
|
||||
return parsePacksSpecification(pack).name === util_1.ML_POWERED_JS_QUERIES_PACK_NAME;
|
||||
}
|
||||
/**
|
||||
* Retrieve the set of queries at localQueryPath and add them to resultMap.
|
||||
|
|
@ -230,13 +208,8 @@ async function addRemoteQueries(codeQL, resultMap, queryUses, tempDir, apiDetail
|
|||
* parsing the 'uses' actions in the workflow file. So it can handle
|
||||
* local paths starting with './', or references to remote repos, or
|
||||
* a finite set of hardcoded terms for builtin suites.
|
||||
*
|
||||
* This may inject ML queries into the packs to use, and the return value will
|
||||
* declare if this was done.
|
||||
*
|
||||
* @returns whether or not we injected ML queries into the packs
|
||||
*/
|
||||
async function parseQueryUses(languages, codeQL, resultMap, packs, queryUses, tempDir, workspacePath, apiDetails, features, logger, configFile) {
|
||||
async function parseQueryUses(languages, codeQL, resultMap, queryUses, tempDir, workspacePath, apiDetails, features, logger, configFile) {
|
||||
queryUses = queryUses.trim();
|
||||
if (queryUses === "") {
|
||||
throw new util_1.UserError(getQueryUsesInvalid(configFile));
|
||||
|
|
@ -244,11 +217,12 @@ async function parseQueryUses(languages, codeQL, resultMap, packs, queryUses, te
|
|||
// Check for the local path case before we start trying to parse the repository name
|
||||
if (queryUses.startsWith("./")) {
|
||||
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), workspacePath, configFile);
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
// Check for one of the builtin suites
|
||||
if (queryUses.indexOf("/") === -1 && queryUses.indexOf("@") === -1) {
|
||||
return await addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, queryUses, features, configFile);
|
||||
await addBuiltinSuiteQueries(languages, codeQL, resultMap, queryUses, configFile);
|
||||
return;
|
||||
}
|
||||
// Otherwise, must be a reference to another repo.
|
||||
// If config parsing is handled in CLI, then this repo will be downloaded
|
||||
|
|
@ -256,7 +230,6 @@ async function parseQueryUses(languages, codeQL, resultMap, packs, queryUses, te
|
|||
if (!(await (0, feature_flags_1.useCodeScanningConfigInCli)(codeQL, features))) {
|
||||
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, apiDetails, logger, configFile);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
// Regex validating stars in paths or paths-ignore entries.
|
||||
// The intention is to only allow ** to appear when immediately
|
||||
|
|
@ -516,16 +489,13 @@ async function getRawLanguages(languagesInput, repository, logger) {
|
|||
return { rawLanguages, autodetected };
|
||||
}
|
||||
exports.getRawLanguages = getRawLanguages;
|
||||
async function addQueriesAndPacksFromWorkflow(codeQL, queriesInput, languages, resultMap, packs, tempDir, workspacePath, apiDetails, features, logger) {
|
||||
let injectedMlQueries = false;
|
||||
async function addQueriesAndPacksFromWorkflow(codeQL, queriesInput, languages, resultMap, tempDir, workspacePath, apiDetails, features, logger) {
|
||||
queriesInput = queriesInput.trim();
|
||||
// "+" means "don't override config file" - see shouldAddConfigFileQueries
|
||||
queriesInput = queriesInput.replace(/^\+/, "");
|
||||
for (const query of queriesInput.split(",")) {
|
||||
const didInject = await parseQueryUses(languages, codeQL, resultMap, packs, query, tempDir, workspacePath, apiDetails, features, logger);
|
||||
injectedMlQueries = injectedMlQueries || didInject;
|
||||
await parseQueryUses(languages, codeQL, resultMap, query, tempDir, workspacePath, apiDetails, features, logger);
|
||||
}
|
||||
return injectedMlQueries;
|
||||
}
|
||||
// Returns true if either no queries were provided in the workflow.
|
||||
// or if the queries in the workflow were provided in "additive" mode,
|
||||
|
|
@ -557,8 +527,7 @@ async function getDefaultConfig(languagesInput, rawQueriesInput, rawPacksInput,
|
|||
}
|
||||
: {};
|
||||
if (rawQueriesInput) {
|
||||
augmentationProperties.injectedMlQueries =
|
||||
await addQueriesAndPacksFromWorkflow(codeQL, rawQueriesInput, languages, queries, packs, tempDir, workspacePath, apiDetails, features, logger);
|
||||
await addQueriesAndPacksFromWorkflow(codeQL, rawQueriesInput, languages, queries, tempDir, workspacePath, apiDetails, features, logger);
|
||||
}
|
||||
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(trapCachingEnabled, codeQL, languages, logger);
|
||||
return {
|
||||
|
|
@ -641,8 +610,7 @@ async function loadConfig(languagesInput, rawQueriesInput, rawPacksInput, config
|
|||
// unless they're prefixed with "+", in which case they supplement those
|
||||
// in the config file.
|
||||
if (rawQueriesInput) {
|
||||
augmentationProperties.injectedMlQueries =
|
||||
await addQueriesAndPacksFromWorkflow(codeQL, rawQueriesInput, languages, queries, packs, tempDir, workspacePath, apiDetails, features, logger);
|
||||
await addQueriesAndPacksFromWorkflow(codeQL, rawQueriesInput, languages, queries, tempDir, workspacePath, apiDetails, features, logger);
|
||||
}
|
||||
if (shouldAddConfigFileQueries(rawQueriesInput) &&
|
||||
QUERIES_PROPERTY in parsedYAML) {
|
||||
|
|
@ -654,7 +622,7 @@ async function loadConfig(languagesInput, rawQueriesInput, rawPacksInput, config
|
|||
if (typeof query[QUERIES_USES_PROPERTY] !== "string") {
|
||||
throw new util_1.UserError(getQueriesMissingUses(configFile));
|
||||
}
|
||||
await parseQueryUses(languages, codeQL, queries, packs, query[QUERIES_USES_PROPERTY], tempDir, workspacePath, apiDetails, features, logger, configFile);
|
||||
await parseQueryUses(languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir, workspacePath, apiDetails, features, logger, configFile);
|
||||
}
|
||||
}
|
||||
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
|
||||
|
|
@ -724,7 +692,6 @@ function calculateAugmentation(rawPacksInput, rawQueriesInput, languages) {
|
|||
const queriesInputCombines = shouldCombine(rawQueriesInput);
|
||||
const queriesInput = parseQueriesFromInput(rawQueriesInput, queriesInputCombines);
|
||||
return {
|
||||
injectedMlQueries: false,
|
||||
packsInputCombines,
|
||||
packsInput: packsInput?.[languages[0]],
|
||||
queriesInput,
|
||||
|
|
@ -932,46 +899,6 @@ function combinePacks(packs1, packs2) {
|
|||
}
|
||||
return packs;
|
||||
}
|
||||
/**
|
||||
* Get information about ML-powered JS queries to populate status reports with.
|
||||
*
|
||||
* This will be:
|
||||
*
|
||||
* - The version string if the analysis is using a single version of the ML-powered query pack.
|
||||
* - "latest" if the version string of the ML-powered query pack is undefined. This is unlikely to
|
||||
* occur in practice (see comment below).
|
||||
* - "false" if the analysis won't run any ML-powered JS queries.
|
||||
* - "other" in all other cases.
|
||||
*
|
||||
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
|
||||
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
|
||||
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
|
||||
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
|
||||
* version strings.
|
||||
*
|
||||
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
|
||||
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
|
||||
* explanation as to why this is.
|
||||
*/
|
||||
function getMlPoweredJsQueriesStatus(config) {
|
||||
const mlPoweredJsQueryPacks = (config.packs.javascript || [])
|
||||
.map((p) => parsePacksSpecification(p))
|
||||
.filter((pack) => pack.name === util_1.ML_POWERED_JS_QUERIES_PACK_NAME && !pack.path);
|
||||
switch (mlPoweredJsQueryPacks.length) {
|
||||
case 1:
|
||||
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
|
||||
// otherwise we won't be able to make changes to the pack unless those changes are compatible
|
||||
// with each version of the CodeQL Action. Therefore in practice we should only hit the
|
||||
// `latest` case here when customers have explicitly added the ML-powered query pack to their
|
||||
// CodeQL config.
|
||||
return mlPoweredJsQueryPacks[0].version || "latest";
|
||||
case 0:
|
||||
return "false";
|
||||
default:
|
||||
return "other";
|
||||
}
|
||||
}
|
||||
exports.getMlPoweredJsQueriesStatus = getMlPoweredJsQueriesStatus;
|
||||
function dbLocationOrDefault(dbLocation, tempDir) {
|
||||
return dbLocation || path.resolve(tempDir, "codeql_databases");
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue