robojerk/codeql-action

Remove parsing of queries, packs, paths, and pathsIgnore

Browse source
This commit is contained in:
Henry Mercer 2024-01-04 16:27:05 +00:00
parent f934b28e51
commit f65fc6a926
27 changed files with 72 additions and 3373 deletions
Download patch file Download diff file Expand all files Collapse all files

192
src/analyze.test.ts
View file

@ -1,16 +1,10 @@
import * as fs from "fs";
import * as path from "path";
import test, { ExecutionContext } from "ava";
import test from "ava";
import * as sinon from "sinon";
import {
convertPackToQuerySuiteEntry,
createQuerySuiteContents,
runQueries,
validateQueryFilters,
QueriesStatusReport,
} from "./analyze";
import { runQueries, QueriesStatusReport } from "./analyze";
import { CodeQL, setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature } from "./feature-flags";
@ -42,11 +36,6 @@ test("status report fields and search path setting", async (t) => {
const memoryFlag = "";
const addSnippetsFlag = "";
const threadsFlag = "";
const packs = {
[Language.cpp]: ["a/b@1.0.0"],
[Language.java]: ["c/d@2.0.0"],
};
sinon.stub(uploadLib, "validateSarifFileSchema");
for (const language of Object.values(Language)) {
@ -108,9 +97,6 @@ test("status report fields and search path setting", async (t) => {
searchPathsUsed = [];
const config: Config = {
languages: [language],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
@ -118,7 +104,6 @@ test("status report fields and search path setting", async (t) => {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs,
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
@ -169,9 +154,6 @@ function mockCodeQL(): Partial<CodeQL> {
function createBaseConfig(tmpDir: string): Config {
return {
languages: [],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "tempDir",
codeQLCmd: "",
@ -179,7 +161,6 @@ function createBaseConfig(tmpDir: string): Config {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
@ -246,172 +227,3 @@ test("optimizeForLastQueryRun for two languages", async (t) => {
);
});
});
test("validateQueryFilters", (t) => {
t.notThrows(() => validateQueryFilters([]));
t.notThrows(() => validateQueryFilters(undefined));
t.notThrows(() => {
return validateQueryFilters([
{
exclude: {
"problem.severity": "recommendation",
},
},
{
exclude: {
"tags contain": ["foo", "bar"],
},
},
{
include: {
"problem.severity": "something-to-think-about",
},
},
{
include: {
"tags contain": ["baz", "bop"],
},
},
]);
});
t.throws(
() => {
return validateQueryFilters([
{
exclude: {
"tags contain": ["foo", "bar"],
},
include: {
"tags contain": ["baz", "bop"],
},
},
]);
},
{ message: /Query filter must have exactly one key/ },
);
t.throws(
() => {
return validateQueryFilters([{ xxx: "foo" } as any]);
},
{ message: /Only "include" or "exclude" filters are allowed/ },
);
t.throws(
() => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
return validateQueryFilters({ exclude: "foo" } as any);
},
{
message:
/Query filters must be an array of "include" or "exclude" entries/,
},
);
});
const convertPackToQuerySuiteEntryMacro = test.macro({
exec: (t: ExecutionContext<unknown>, packSpec: string, suiteEntry: any) =>
t.deepEqual(convertPackToQuerySuiteEntry(packSpec), suiteEntry),
title: (_providedTitle, packSpec: string) => `Query Suite Entry: ${packSpec}`,
});
test(convertPackToQuerySuiteEntryMacro, "a/b", {
qlpack: "a/b",
from: undefined,
version: undefined,
query: undefined,
queries: undefined,
apply: undefined,
});
test(convertPackToQuerySuiteEntryMacro, "a/b@~1.2.3", {
qlpack: "a/b",
from: undefined,
version: "~1.2.3",
query: undefined,
queries: undefined,
apply: undefined,
});
test(convertPackToQuerySuiteEntryMacro, "a/b:my/path", {
qlpack: undefined,
from: "a/b",
version: undefined,
query: undefined,
queries: "my/path",
apply: undefined,
});
test(convertPackToQuerySuiteEntryMacro, "a/b@~1.2.3:my/path", {
qlpack: undefined,
from: "a/b",
version: "~1.2.3",
query: undefined,
queries: "my/path",
apply: undefined,
});
test(convertPackToQuerySuiteEntryMacro, "a/b:my/path/query.ql", {
qlpack: undefined,
from: "a/b",
version: undefined,
query: "my/path/query.ql",
queries: undefined,
apply: undefined,
});
test(convertPackToQuerySuiteEntryMacro, "a/b@~1.2.3:my/path/query.ql", {
qlpack: undefined,
from: "a/b",
version: "~1.2.3",
query: "my/path/query.ql",
queries: undefined,
apply: undefined,
});
test(convertPackToQuerySuiteEntryMacro, "a/b:my/path/suite.qls", {
qlpack: undefined,
from: "a/b",
version: undefined,
query: undefined,
queries: undefined,
apply: "my/path/suite.qls",
});
test(convertPackToQuerySuiteEntryMacro, "a/b@~1.2.3:my/path/suite.qls", {
qlpack: undefined,
from: "a/b",
version: "~1.2.3",
query: undefined,
queries: undefined,
apply: "my/path/suite.qls",
});
test("convertPackToQuerySuiteEntry Failure", (t) => {
t.throws(() => convertPackToQuerySuiteEntry("this-is-not-a-pack"));
});
test("createQuerySuiteContents", (t) => {
const yamlResult = createQuerySuiteContents(
["query1.ql", "query2.ql"],
[
{
exclude: { "problem.severity": "recommendation" },
},
{
include: { "problem.severity": "recommendation" },
},
],
);
const expected = `- query: query1.ql
- query: query2.ql
- exclude:
problem.severity: recommendation
- include:
problem.severity: recommendation
`;
t.deepEqual(yamlResult, expected);
});

63
src/analyze.ts
View file

@ -19,7 +19,6 @@ import { DatabaseCreationTimings, EventReport } from "./status-report";
import { endTracingForCluster } from "./tracer-config";
import { validateSarifFileSchema } from "./upload-lib";
import * as util from "./util";
import { UserError } from "./util";
export class CodeQLAnalysisError extends Error {
queriesStatusReport: QueriesStatusReport;
@ -391,32 +390,6 @@ export async function runQueries(
}
}
export function convertPackToQuerySuiteEntry(
packStr: string,
): configUtils.QuerySuitePackEntry {
const pack = configUtils.parsePacksSpecification(packStr);
return {
qlpack: !pack.path ? pack.name : undefined,
from: pack.path ? pack.name : undefined,
version: pack.version,
query: pack.path?.endsWith(".ql") ? pack.path : undefined,
queries:
!pack.path?.endsWith(".ql") && !pack.path?.endsWith(".qls")
? pack.path
: undefined,
apply: pack.path?.endsWith(".qls") ? pack.path : undefined,
};
}
export function createQuerySuiteContents(
queries: string[],
queryFilters: configUtils.QueryFilter[],
) {
return yaml.dump(
queries.map((q: string) => ({ query: q })).concat(queryFilters as any[]),
);
}
export async function runFinalize(
outputDir: string,
threadsFlag: string,
@ -465,39 +438,3 @@ export async function runCleanup(
}
logger.endGroup();
}
// exported for testing
export function validateQueryFilters(queryFilters?: configUtils.QueryFilter[]) {
if (!queryFilters) {
return [];
}
if (!Array.isArray(queryFilters)) {
throw new UserError(
`Query filters must be an array of "include" or "exclude" entries. Found ${typeof queryFilters}`,
);
}
const errors: string[] = [];
for (const qf of queryFilters) {
const keys = Object.keys(qf);
if (keys.length !== 1) {
errors.push(
`Query filter must have exactly one key: ${JSON.stringify(qf)}`,
);
}
if (!["exclude", "include"].includes(keys[0])) {
errors.push(
`Only "include" or "exclude" filters are allowed:\n${JSON.stringify(
qf,
)}`,
);
}
}
if (errors.length) {
throw new UserError(`Invalid query filter.\n${errors.join("\n")}`);
}
return queryFilters;
}

4
src/codeql.test.ts
View file

@ -46,9 +46,6 @@ test.beforeEach(() => {
stubConfig = {
languages: [Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
@ -56,7 +53,6 @@ test.beforeEach(() => {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,

1146
src/config-utils.test.ts
View file

File diff suppressed because it is too large Load diff

799
src/config-utils.ts
View file

@ -6,12 +6,7 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import * as api from "./api-client";
import {
CodeQL,
CODEQL_VERSION_LANGUAGE_ALIASING,
CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE,
ResolveQueriesOutput,
} from "./codeql";
import { CodeQL, CODEQL_VERSION_LANGUAGE_ALIASING } from "./codeql";
import { Language, parseLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
@ -24,12 +19,7 @@ import {
} from "./util";
// Property names from the user-supplied config file.
const NAME_PROPERTY = "name";
const DISABLE_DEFAULT_QUERIES_PROPERTY = "disable-default-queries";
const QUERIES_PROPERTY = "queries";
const QUERIES_USES_PROPERTY = "uses";
const PATHS_IGNORE_PROPERTY = "paths-ignore";
const PATHS_PROPERTY = "paths";
const PACKS_PROPERTY = "packs";
/**
@ -82,52 +72,6 @@ interface IncludeQueryFilter {
include: Record<string, string[] | string>;
}
export type QuerySuitePackEntry = {
version?: string;
} & (
| {
qlpack: string;
}
| {
from?: string;
query?: string;
queries?: string;
apply?: string;
}
);
export type QuerySuiteEntry = QuerySuitePackEntry | QueryFilter;
/**
* Lists of query files for each language.
* Will only contain .ql files and not other kinds of files,
* and all file paths will be absolute.
*
* The queries are split between ones from a builtin suite
* and custom queries from unknown locations. This allows us to treat
* them separately if we want to, for example to measure performance.
*/
type Queries = {
[language: string]: {
/** Queries from one of the builtin suites */
builtin: string[];
/** Custom queries, from a non-standard location */
custom: QueriesWithSearchPath[];
};
};
/**
* Contains some information about a user-defined query.
*/
export interface QueriesWithSearchPath {
/** Additional search path to use when running these queries. */
searchPath: string;
/** Array of absolute paths to a .ql file containing the queries. */
queries: string[];
}
/**
* Format of the parsed config file.
*/
@ -136,18 +80,6 @@ export interface Config {
* Set of languages to run analysis for.
*/
languages: Language[];
/**
* Map from language to query files.
*/
queries: Queries;
/**
* List of paths to ignore from analysis.
*/
pathsIgnore: string[];
/**
* List of paths to include in analysis.
*/
paths: string[];
/**
* A unaltered copy of the original user input.
* Mainly intended to be used for status reporting.
@ -174,10 +106,6 @@ export interface Config {
* The location where CodeQL databases should be stored.
*/
dbLocation: string;
/**
* List of packages, separated by language to download before any analysis.
*/
packs: Packs;
/**
* Specifies whether we are debugging mode and should try to produce extra
* output for debugging purposes when possible.
@ -255,419 +183,6 @@ export interface Pack {
path?: string;
}
/**
* A list of queries from https://github.com/github/codeql that
* we don't want to run. Disabling them here is a quicker alternative to
* disabling them in the code scanning query suites. Queries should also
* be disabled in the suites, and removed from this list here once the
* bundle is updated to make those suite changes live.
*
* Format is a map from language to an array of path suffixes of .ql files.
*/
const DISABLED_BUILTIN_QUERIES: { [language: string]: string[] } = {
csharp: [
"ql/src/Security Features/CWE-937/VulnerablePackage.ql",
"ql/src/Security Features/CWE-451/MissingXFrameOptions.ql",
],
};
function queryIsDisabled(language, query): boolean {
return (DISABLED_BUILTIN_QUERIES[language] || []).some((disabledQuery) =>
query.endsWith(disabledQuery),
);
}
/**
* Asserts that the noDeclaredLanguage and multipleDeclaredLanguages fields are
* both empty and errors if they are not.
*/
function validateQueries(resolvedQueries: ResolveQueriesOutput) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage;
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new UserError(
`${
"The following queries do not declare a language. " +
"Their qlpack.yml files are either missing or is invalid.\n"
}${noDeclaredLanguageQueries.join("\n")}`,
);
}
const multipleDeclaredLanguages = resolvedQueries.multipleDeclaredLanguages;
const multipleDeclaredLanguagesQueries = Object.keys(
multipleDeclaredLanguages,
);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new UserError(
`${
"The following queries declare multiple languages. " +
"Their qlpack.yml files are either missing or is invalid.\n"
}${multipleDeclaredLanguagesQueries.join("\n")}`,
);
}
}
/**
* Run 'codeql resolve queries' and add the results to resultMap
*
* If a checkout path is given then the queries are assumed to be custom queries
* and an error will be thrown if there is anything invalid about the queries.
* If a checkout path is not given then the queries are assumed to be builtin
* queries, and error checking will be suppressed.
*/
async function runResolveQueries(
codeQL: CodeQL,
resultMap: Queries,
toResolve: string[],
extraSearchPath: string | undefined,
) {
const resolvedQueries = await codeQL.resolveQueries(
toResolve,
extraSearchPath,
);
if (extraSearchPath !== undefined) {
validateQueries(resolvedQueries);
}
for (const [language, queryPaths] of Object.entries(
resolvedQueries.byLanguage,
)) {
if (resultMap[language] === undefined) {
resultMap[language] = {
builtin: [],
custom: [],
};
}
const queries = Object.keys(queryPaths).filter(
(q) => !queryIsDisabled(language, q),
);
if (extraSearchPath !== undefined) {
resultMap[language].custom.push({
searchPath: extraSearchPath,
queries,
});
} else {
resultMap[language].builtin.push(...queries);
}
}
}
/**
* Get the set of queries included by default.
*/
async function addDefaultQueries(
codeQL: CodeQL,
languages: string[],
resultMap: Queries,
) {
const suites = languages.map((l) => `${l}-code-scanning.qls`);
await runResolveQueries(codeQL, resultMap, suites, undefined);
}
// The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = [
"security-experimental",
"security-extended",
"security-and-quality",
] as const;
/**
* Determine the set of queries associated with suiteName's suites and add them to resultMap.
* Throws an error if suiteName is not a valid builtin suite.
*/
async function addBuiltinSuiteQueries(
languages: string[],
codeQL: CodeQL,
resultMap: Queries,
suiteName: string,
configFile?: string,
): Promise<void> {
const found = builtinSuites.find((suite) => suite === suiteName);
if (!found) {
throw new UserError(getQueryUsesInvalid(configFile, suiteName));
}
if (
suiteName === "security-experimental" &&
!(await codeQlVersionAbove(
codeQL,
CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE,
))
) {
throw new UserError(
`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`,
);
}
const suites = languages.map((l) => `${l}-${suiteName}.qls`);
await runResolveQueries(codeQL, resultMap, suites, undefined);
}
/**
* Retrieve the set of queries at localQueryPath and add them to resultMap.
*/
async function addLocalQueries(
codeQL: CodeQL,
resultMap: Queries,
localQueryPath: string,
workspacePath: string,
configFile?: string,
) {
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
let absoluteQueryPath = path.join(workspacePath, localQueryPath);
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
throw new UserError(getLocalPathDoesNotExist(configFile, localQueryPath));
}
// Call this after checking file exists, because it'll fail if file doesn't exist
absoluteQueryPath = fs.realpathSync(absoluteQueryPath);
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (
!(absoluteQueryPath + path.sep).startsWith(
fs.realpathSync(workspacePath) + path.sep,
)
) {
throw new UserError(
getLocalPathOutsideOfRepository(configFile, localQueryPath),
);
}
const extraSearchPath = workspacePath;
await runResolveQueries(
codeQL,
resultMap,
[absoluteQueryPath],
extraSearchPath,
);
}
/**
* Parse a query 'uses' field to a discrete set of query files and update resultMap.
*
* The logic for parsing the string is based on what actions does for
* parsing the 'uses' actions in the workflow file. So it can handle
* local paths starting with './', or references to remote repos, or
* a finite set of hardcoded terms for builtin suites.
*/
async function parseQueryUses(
languages: string[],
codeQL: CodeQL,
resultMap: Queries,
queryUses: string,
workspacePath: string,
configFile?: string,
): Promise<void> {
queryUses = queryUses.trim();
if (queryUses === "") {
throw new UserError(getQueryUsesInvalid(configFile));
}
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
await addLocalQueries(
codeQL,
resultMap,
queryUses.slice(2),
workspacePath,
configFile,
);
return;
}
// Check for one of the builtin suites
if (queryUses.indexOf("/") === -1 && queryUses.indexOf("@") === -1) {
await addBuiltinSuiteQueries(
languages,
codeQL,
resultMap,
queryUses,
configFile,
);
return;
}
}
// Regex validating stars in paths or paths-ignore entries.
// The intention is to only allow ** to appear when immediately
// preceded and followed by a slash.
const pathStarsRegex = /.*(?:\*\*[^/].*|\*\*$|[^/]\*\*.*)/;
// Characters that are supported by filters in workflows, but not by us.
// See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
const filterPatternCharactersRegex = /.*[?+[\]!].*/;
// Checks that a paths of paths-ignore entry is valid, possibly modifying it
// to make it valid, or if not possible then throws an error.
export function validateAndSanitisePath(
originalPath: string,
propertyName: string,
configFile: string,
logger: Logger,
): string {
// Take a copy so we don't modify the original path, so we can still construct error messages
let newPath = originalPath;
// All paths are relative to the src root, so strip off leading slashes.
while (newPath.charAt(0) === "/") {
newPath = newPath.substring(1);
}
// Trailing ** are redundant, so strip them off
if (newPath.endsWith("/**")) {
newPath = newPath.substring(0, newPath.length - 2);
}
// An empty path is not allowed as it's meaningless
if (newPath === "") {
throw new UserError(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" is not an invalid path. ` +
`It is not necessary to include it, and it is not allowed to exclude it.`,
),
);
}
// Check for illegal uses of **
if (newPath.match(pathStarsRegex)) {
throw new UserError(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" contains an invalid "**" wildcard. ` +
`They must be immediately preceded and followed by a slash as in "/**/", or come at the start or end.`,
),
);
}
// Check for other regex characters that we don't support.
// Output a warning so the user knows, but otherwise continue normally.
if (newPath.match(filterPatternCharactersRegex)) {
logger.warning(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" contains an unsupported character. ` +
`The filter pattern characters ?, +, [, ], ! are not supported and will be matched literally.`,
),
);
}
// Ban any uses of backslash for now.
// This may not play nicely with project layouts.
// This restriction can be lifted later if we determine they are ok.
if (newPath.indexOf("\\") !== -1) {
throw new UserError(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" contains an "\\" character. These are not allowed in filters. ` +
`If running on windows we recommend using "/" instead for path filters.`,
),
);
}
return newPath;
}
// An undefined configFile in some of these functions indicates that
// the property was in a workflow file, not a config file
export function getNameInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
NAME_PROPERTY,
"must be a non-empty string",
);
}
export function getDisableDefaultQueriesInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
DISABLE_DEFAULT_QUERIES_PROPERTY,
"must be a boolean",
);
}
export function getQueriesInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY,
"must be an array",
);
}
export function getQueriesMissingUses(configFile: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY,
"must be an array, with each entry having a 'uses' property",
);
}
export function getQueryUsesInvalid(
configFile: string | undefined,
queryUses?: string,
): string {
return getConfigFilePropertyError(
configFile,
`${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`,
`must be a built-in suite (${builtinSuites.join(
" or ",
)}), a relative path, or be of the form "owner/repo[/path]@ref"${
queryUses !== undefined ? `\n Found: ${queryUses}` : ""
}`,
);
}
export function getPathsIgnoreInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
PATHS_IGNORE_PROPERTY,
"must be an array of non-empty strings",
);
}
export function getPathsInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
PATHS_PROPERTY,
"must be an array of non-empty strings",
);
}
function getPacksRequireLanguage(lang: string, configFile: string): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
`has "${lang}", but it is not a valid language.`,
);
}
export function getPacksInvalidSplit(configFile: string): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
"must split packages by language",
);
}
export function getPacksInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
"must be an array of non-empty strings",
);
}
export function getPacksStrInvalid(
packStr: string,
configFile?: string,
@ -681,28 +196,6 @@ export function getPacksStrInvalid(
: `"${packStr}" is not a valid pack`;
}
export function getLocalPathOutsideOfRepository(
configFile: string | undefined,
localPath: string,
): string {
return getConfigFilePropertyError(
configFile,
`${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`,
`is invalid as the local path "${localPath}" is outside of the repository`,
);
}
export function getLocalPathDoesNotExist(
configFile: string | undefined,
localPath: string,
): string {
return getConfigFilePropertyError(
configFile,
`${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`,
`is invalid as the local path "${localPath}" does not exist in the repository`,
);
}
export function getConfigFileOutsideWorkspaceErrorMessage(
configFile: string,
): string {
@ -899,34 +392,6 @@ export async function getRawLanguages(
return { rawLanguages, autodetected };
}
async function addQueriesFromWorkflow(
codeQL: CodeQL,
queriesInput: string,
languages: string[],
resultMap: Queries,
workspacePath: string,
): Promise<void> {
queriesInput = queriesInput.trim();
// "+" means "don't override config file" - see shouldAddConfigFileQueries
queriesInput = queriesInput.replace(/^\+/, "");
for (const query of queriesInput.split(",")) {
await parseQueryUses(languages, codeQL, resultMap, query, workspacePath);
}
}
// Returns true if either no queries were provided in the workflow.
// or if the queries in the workflow were provided in "additive" mode,
// indicating that they shouldn't override the config queries but
// should instead be added in addition
function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
if (queriesInput) {
return queriesInput.trimStart().slice(0, 1) === "+";
}
return true;
}
/**
* Get the default config for when the user has not supplied one.
*/
@ -942,7 +407,6 @@ export async function getDefaultConfig(
repository: RepositoryNwo,
tempDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: GitHubVersion,
logger: Logger,
): Promise<Config> {
@ -952,33 +416,11 @@ export async function getDefaultConfig(
repository,
logger,
);
const queries: Queries = {};
for (const language of languages) {
queries[language] = {
builtin: [],
custom: [],
};
}
await addDefaultQueries(codeQL, languages, queries);
const augmentationProperties = calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages,
);
const packs = augmentationProperties.packsInput
? {
[languages[0]]: augmentationProperties.packsInput,
}
: {};
if (rawQueriesInput) {
await addQueriesFromWorkflow(
codeQL,
rawQueriesInput,
languages,
queries,
workspacePath,
);
}
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
@ -989,10 +431,6 @@ export async function getDefaultConfig(
return {
languages,
queries,
pathsIgnore: [],
paths: [],
packs,
originalUserInput: {},
tempDir,
codeQLCmd: codeQL.getPath(),
@ -1057,17 +495,6 @@ async function loadConfig(
parsedYAML = await getRemoteConfig(configFile, apiDetails);
}
// Validate that the 'name' property is syntactically correct,
// even though we don't use the value yet.
if (NAME_PROPERTY in parsedYAML) {
if (typeof parsedYAML[NAME_PROPERTY] !== "string") {
throw new UserError(getNameInvalid(configFile));
}
if (parsedYAML[NAME_PROPERTY]!.length === 0) {
throw new UserError(getNameInvalid(configFile));
}
}
const languages = await getLanguages(
codeQL,
languagesInput,
@ -1075,113 +502,11 @@ async function loadConfig(
logger,
);
const queries: Queries = {};
for (const language of languages) {
queries[language] = {
builtin: [],
custom: [],
};
}
const pathsIgnore: string[] = [];
const paths: string[] = [];
let disableDefaultQueries = false;
if (DISABLE_DEFAULT_QUERIES_PROPERTY in parsedYAML) {
if (typeof parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY] !== "boolean") {
throw new UserError(getDisableDefaultQueriesInvalid(configFile));
}
disableDefaultQueries = parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY]!;
}
if (!disableDefaultQueries) {
await addDefaultQueries(codeQL, languages, queries);
}
const augmentationProperties = calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages,
);
const packs = parsePacks(
parsedYAML[PACKS_PROPERTY] ?? {},
rawPacksInput,
augmentationProperties.packsInputCombines,
languages,
configFile,
logger,
);
// If queries were provided using `with` in the action configuration,
// they should take precedence over the queries in the config file
// unless they're prefixed with "+", in which case they supplement those
// in the config file.
if (rawQueriesInput) {
await addQueriesFromWorkflow(
codeQL,
rawQueriesInput,
languages,
queries,
workspacePath,
);
}
if (
shouldAddConfigFileQueries(rawQueriesInput) &&
QUERIES_PROPERTY in parsedYAML
) {
const queriesArr = parsedYAML[QUERIES_PROPERTY];
if (!Array.isArray(queriesArr)) {
throw new UserError(getQueriesInvalid(configFile));
}
for (const query of queriesArr) {
if (typeof query[QUERIES_USES_PROPERTY] !== "string") {
throw new UserError(getQueriesMissingUses(configFile));
}
await parseQueryUses(
languages,
codeQL,
queries,
query[QUERIES_USES_PROPERTY],
workspacePath,
configFile,
);
}
}
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
throw new UserError(getPathsIgnoreInvalid(configFile));
}
for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]!) {
if (typeof ignorePath !== "string" || ignorePath === "") {
throw new UserError(getPathsIgnoreInvalid(configFile));
}
pathsIgnore.push(
validateAndSanitisePath(
ignorePath,
PATHS_IGNORE_PROPERTY,
configFile,
logger,
),
);
}
}
if (PATHS_PROPERTY in parsedYAML) {
if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
throw new UserError(getPathsInvalid(configFile));
}
for (const includePath of parsedYAML[PATHS_PROPERTY]!) {
if (typeof includePath !== "string" || includePath === "") {
throw new UserError(getPathsInvalid(configFile));
}
paths.push(
validateAndSanitisePath(
includePath,
PATHS_PROPERTY,
configFile,
logger,
),
);
}
}
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
@ -1192,10 +517,6 @@ async function loadConfig(
return {
languages,
queries,
pathsIgnore,
paths,
packs,
originalUserInput: parsedYAML,
tempDir,
codeQLCmd: codeQL.getPath(),
@ -1289,52 +610,7 @@ const PACK_IDENTIFIER_PATTERN = (function () {
})();
// Exported for testing
export function parsePacksFromConfig(
packsByLanguage: string[] | Record<string, string[]>,
languages: Language[],
configFile: string,
logger: Logger,
): Packs {
const packs = {};
if (Array.isArray(packsByLanguage)) {
if (languages.length === 1) {
// single language analysis, so language is implicit
packsByLanguage = {
[languages[0]]: packsByLanguage,
};
} else {
// this is an error since multi-language analysis requires
// packs split by language
throw new UserError(getPacksInvalidSplit(configFile));
}
}
for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
if (!Array.isArray(packsArr)) {
throw new UserError(getPacksInvalid(configFile));
}
if (!languages.includes(lang as Language)) {
// This particular language is not being analyzed in this run.
if (Language[lang as Language]) {
logger.info(
`Ignoring packs for ${lang} since this language is not being analyzed in this run.`,
);
continue;
} else {
// This language is invalid, probably a misspelling
throw new UserError(getPacksRequireLanguage(configFile, lang));
}
}
packs[lang] = packsArr.map((packStr) =>
validatePackSpecification(packStr, configFile),
);
}
return packs;
}
function parsePacksFromInput(
export function parsePacksFromInput(
rawPacksInput: string | undefined,
languages: Language[],
packsInputCombines: boolean,
@ -1369,7 +645,7 @@ function parsePacksFromInput(
return {
[languages[0]]: rawPacksInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack, ""));
packs.push(validatePackSpecification(pack));
return packs;
}, [] as string[]),
};
@ -1393,12 +669,9 @@ function parsePacksFromInput(
* @param packStr the package specification to verify.
* @param configFile Config file to use for error reporting
*/
export function parsePacksSpecification(
packStr: string,
configFile?: string,
): Pack {
export function parsePacksSpecification(packStr: string): Pack {
if (typeof packStr !== "string") {
throw new UserError(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr));
}
packStr = packStr.trim();
@ -1426,14 +699,14 @@ export function parsePacksSpecification(
: undefined;
if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
throw new UserError(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr));
}
if (version) {
try {
new semver.Range(version);
} catch (e) {
// The range string is invalid. OK to ignore the caught error
throw new UserError(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr));
}
}
@ -1447,12 +720,12 @@ export function parsePacksSpecification(
path.normalize(packPath).split(path.sep).join("/") !==
packPath.split(path.sep).join("/"))
) {
throw new UserError(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr));
}
if (!packPath && pathStart) {
// 0 length path
throw new UserError(getPacksStrInvalid(packStr, configFile));
throw new UserError(getPacksStrInvalid(packStr));
}
return {
@ -1462,42 +735,8 @@ export function parsePacksSpecification(
};
}
export function validatePackSpecification(pack: string, configFile?: string) {
return prettyPrintPack(parsePacksSpecification(pack, configFile));
}
// exported for testing
export function parsePacks(
rawPacksFromConfig: string[] | Record<string, string[]>,
rawPacksFromInput: string | undefined,
packsInputCombines: boolean,
languages: Language[],
configFile: string,
logger: Logger,
): Packs {
const packsFomConfig = parsePacksFromConfig(
rawPacksFromConfig,
languages,
configFile,
logger,
);
const packsFromInput = parsePacksFromInput(
rawPacksFromInput,
languages,
packsInputCombines,
);
if (!packsFromInput) {
return packsFomConfig;
}
if (!packsInputCombines) {
if (!packsFromInput) {
throw new UserError(getPacksInvalid(configFile));
}
return packsFromInput;
}
return combinePacks(packsFromInput, packsFomConfig);
export function validatePackSpecification(pack: string) {
return prettyPrintPack(parsePacksSpecification(pack));
}
/**
@ -1514,19 +753,6 @@ function shouldCombine(inputValue?: string): boolean {
return !!inputValue?.trim().startsWith("+");
}
function combinePacks(packs1: Packs, packs2: Packs): Packs {
const packs = {};
for (const lang of Object.keys(packs1)) {
packs[lang] = packs1[lang].concat(packs2[lang] || []);
}
for (const lang of Object.keys(packs2)) {
if (!packs[lang]) {
packs[lang] = packs2[lang];
}
}
return packs;
}
function dbLocationOrDefault(
dbLocation: string | undefined,
tempDir: string,
@ -1588,7 +814,6 @@ export async function initConfig(
repository,
tempDir,
codeQL,
workspacePath,
gitHubVersion,
logger,
);

4
src/database-upload.test.ts
View file

@ -43,15 +43,11 @@ const testApiDetails: GitHubApiDetails = {
function getTestConfig(tmpDir: string): Config {
return {
languages: [Language.javascript],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "foo",
gitHubVersion: { type: GitHubVariant.DOTCOM },
dbLocation: tmpDir,
packs: {},
debugMode: false,
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,

3
src/init.ts
View file

@ -131,7 +131,8 @@ function printPathFiltersWarning(config: configUtils.Config, logger: Logger) {
// Index include/exclude/filters only work in javascript/python/ruby.
// If any other languages are detected/configured then show a warning.
if (
(config.paths.length !== 0 || config.pathsIgnore.length !== 0) &&
(config.originalUserInput.paths?.length !== 0 ||
config.originalUserInput["paths-ignore"]?.length !== 0) &&
!config.languages.every(isScannedLanguage)
) {
logger.warning(

4
src/tracer-config.test.ts
View file

@ -14,15 +14,11 @@ setupTests(test);
function getTestConfig(tmpDir: string): configUtils.Config {
return {
languages: [Language.java],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,

8
src/trap-caching.test.ts
View file

@ -71,9 +71,6 @@ const stubCodeql = setCodeQL({
const testConfigWithoutTmpDir: Config = {
languages: [Language.javascript, Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
@ -81,7 +78,6 @@ const testConfigWithoutTmpDir: Config = {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
@ -98,15 +94,11 @@ const testConfigWithoutTmpDir: Config = {
function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
return {
languages: [Language.javascript, Language.ruby],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,