Merge branch 'main' into query-overriding
This commit is contained in:
Sam Partington
commit
bdfd48264f
3443 changed files with 451694 additions and 2619 deletions
156
lib/config-utils.js
generated
156
lib/config-utils.js
generated
|
|
@ -8,13 +8,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const core = __importStar(require("@actions/core"));
|
||||
const io = __importStar(require("@actions/io"));
|
||||
const fs = __importStar(require("fs"));
|
||||
const yaml = __importStar(require("js-yaml"));
|
||||
const path = __importStar(require("path"));
|
||||
const api = __importStar(require("./api-client"));
|
||||
const codeql_1 = require("./codeql");
|
||||
const externalQueries = __importStar(require("./external-queries"));
|
||||
const languages_1 = require("./languages");
|
||||
const util = __importStar(require("./util"));
|
||||
// Property names from the user-supplied config file.
|
||||
const NAME_PROPERTY = 'name';
|
||||
|
|
@ -65,9 +64,8 @@ function validateQueries(resolvedQueries) {
|
|||
/**
|
||||
* Run 'codeql resolve queries' and add the results to resultMap
|
||||
*/
|
||||
async function runResolveQueries(resultMap, toResolve, extraSearchPath, errorOnInvalidQueries) {
|
||||
const codeQl = codeql_1.getCodeQL();
|
||||
const resolvedQueries = await codeQl.resolveQueries(toResolve, extraSearchPath);
|
||||
async function runResolveQueries(codeQL, resultMap, toResolve, extraSearchPath, errorOnInvalidQueries) {
|
||||
const resolvedQueries = await codeQL.resolveQueries(toResolve, extraSearchPath);
|
||||
for (const [language, queries] of Object.entries(resolvedQueries.byLanguage)) {
|
||||
if (resultMap[language] === undefined) {
|
||||
resultMap[language] = [];
|
||||
|
|
@ -81,9 +79,9 @@ async function runResolveQueries(resultMap, toResolve, extraSearchPath, errorOnI
|
|||
/**
|
||||
* Get the set of queries included by default.
|
||||
*/
|
||||
async function addDefaultQueries(languages, resultMap) {
|
||||
async function addDefaultQueries(codeQL, languages, resultMap) {
|
||||
const suites = languages.map(l => l + '-code-scanning.qls');
|
||||
await runResolveQueries(resultMap, suites, undefined, false);
|
||||
await runResolveQueries(codeQL, resultMap, suites, undefined, false);
|
||||
}
|
||||
// The set of acceptable values for built-in suites from the codeql bundle
|
||||
const builtinSuites = ['security-extended', 'security-and-quality'];
|
||||
|
|
@ -91,18 +89,18 @@ const builtinSuites = ['security-extended', 'security-and-quality'];
|
|||
* Determine the set of queries associated with suiteName's suites and add them to resultMap.
|
||||
* Throws an error if suiteName is not a valid builtin suite.
|
||||
*/
|
||||
async function addBuiltinSuiteQueries(languages, resultMap, suiteName, configFile) {
|
||||
async function addBuiltinSuiteQueries(languages, codeQL, resultMap, suiteName, configFile) {
|
||||
const suite = builtinSuites.find((suite) => suite === suiteName);
|
||||
if (!suite) {
|
||||
throw new Error(getQueryUsesInvalid(configFile, suiteName));
|
||||
}
|
||||
const suites = languages.map(l => l + '-' + suiteName + '.qls');
|
||||
await runResolveQueries(resultMap, suites, undefined, false);
|
||||
await runResolveQueries(codeQL, resultMap, suites, undefined, false);
|
||||
}
|
||||
/**
|
||||
* Retrieve the set of queries at localQueryPath and add them to resultMap.
|
||||
*/
|
||||
async function addLocalQueries(resultMap, localQueryPath, configFile) {
|
||||
async function addLocalQueries(codeQL, resultMap, localQueryPath, configFile) {
|
||||
// Resolve the local path against the workspace so that when this is
|
||||
// passed to codeql it resolves to exactly the path we expect it to resolve to.
|
||||
const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE'));
|
||||
|
|
@ -119,12 +117,12 @@ async function addLocalQueries(resultMap, localQueryPath, configFile) {
|
|||
}
|
||||
// Get the root of the current repo to use when resolving query dependencies
|
||||
const rootOfRepo = util.getRequiredEnvParam('GITHUB_WORKSPACE');
|
||||
await runResolveQueries(resultMap, [absoluteQueryPath], rootOfRepo, true);
|
||||
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], rootOfRepo, true);
|
||||
}
|
||||
/**
|
||||
* Retrieve the set of queries at the referenced remote repo and add them to resultMap.
|
||||
*/
|
||||
async function addRemoteQueries(resultMap, queryUses, configFile) {
|
||||
async function addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile) {
|
||||
let tok = queryUses.split('@');
|
||||
if (tok.length !== 2) {
|
||||
throw new Error(getQueryUsesInvalid(configFile, queryUses));
|
||||
|
|
@ -143,11 +141,11 @@ async function addRemoteQueries(resultMap, queryUses, configFile) {
|
|||
}
|
||||
const nwo = tok[0] + '/' + tok[1];
|
||||
// Checkout the external repository
|
||||
const rootOfRepo = await externalQueries.checkoutExternalRepository(nwo, ref);
|
||||
const rootOfRepo = await externalQueries.checkoutExternalRepository(nwo, ref, tempDir);
|
||||
const queryPath = tok.length > 2
|
||||
? path.join(rootOfRepo, tok.slice(2).join('/'))
|
||||
: rootOfRepo;
|
||||
await runResolveQueries(resultMap, [queryPath], rootOfRepo, true);
|
||||
await runResolveQueries(codeQL, resultMap, [queryPath], rootOfRepo, true);
|
||||
}
|
||||
/**
|
||||
* Parse a query 'uses' field to a discrete set of query files and update resultMap.
|
||||
|
|
@ -157,23 +155,23 @@ async function addRemoteQueries(resultMap, queryUses, configFile) {
|
|||
* local paths starting with './', or references to remote repos, or
|
||||
* a finite set of hardcoded terms for builtin suites.
|
||||
*/
|
||||
async function parseQueryUses(languages, resultMap, queryUses, configFile) {
|
||||
async function parseQueryUses(languages, codeQL, resultMap, queryUses, tempDir, configFile) {
|
||||
queryUses = queryUses.trim();
|
||||
if (queryUses === "") {
|
||||
throw new Error(getQueryUsesInvalid(configFile));
|
||||
}
|
||||
// Check for the local path case before we start trying to parse the repository name
|
||||
if (queryUses.startsWith("./")) {
|
||||
await addLocalQueries(resultMap, queryUses.slice(2), configFile);
|
||||
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), configFile);
|
||||
return;
|
||||
}
|
||||
// Check for one of the builtin suites
|
||||
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
|
||||
await addBuiltinSuiteQueries(languages, resultMap, queryUses, configFile);
|
||||
await addBuiltinSuiteQueries(languages, codeQL, resultMap, queryUses, configFile);
|
||||
return;
|
||||
}
|
||||
// Otherwise, must be a reference to another repo
|
||||
await addRemoteQueries(resultMap, queryUses, configFile);
|
||||
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile);
|
||||
}
|
||||
// Regex validating stars in paths or paths-ignore entries.
|
||||
// The intention is to only allow ** to appear when immediately
|
||||
|
|
@ -287,40 +285,39 @@ function getConfigFilePropertyError(configFile, property, error) {
|
|||
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
|
||||
}
|
||||
}
|
||||
function getNoLanguagesError() {
|
||||
return "Did not detect any languages to analyze. " +
|
||||
"Please update input in workflow or check that GitHub detects the correct languages in your repository.";
|
||||
}
|
||||
exports.getNoLanguagesError = getNoLanguagesError;
|
||||
function getUnknownLanguagesError(languages) {
|
||||
return "Did not recognise the following languages: " + languages.join(', ');
|
||||
}
|
||||
exports.getUnknownLanguagesError = getUnknownLanguagesError;
|
||||
/**
|
||||
* Gets the set of languages in the current repository
|
||||
*/
|
||||
async function getLanguagesInRepo() {
|
||||
var _a;
|
||||
// Translate between GitHub's API names for languages and ours
|
||||
const codeqlLanguages = {
|
||||
'C': 'cpp',
|
||||
'C++': 'cpp',
|
||||
'C#': 'csharp',
|
||||
'Go': 'go',
|
||||
'Java': 'java',
|
||||
'JavaScript': 'javascript',
|
||||
'TypeScript': 'javascript',
|
||||
'Python': 'python',
|
||||
};
|
||||
let repo_nwo = (_a = process.env['GITHUB_REPOSITORY']) === null || _a === void 0 ? void 0 : _a.split("/");
|
||||
if (repo_nwo) {
|
||||
let owner = repo_nwo[0];
|
||||
let repo = repo_nwo[1];
|
||||
core.debug(`GitHub repo ${owner} ${repo}`);
|
||||
const response = await api.getApiClient().request("GET /repos/:owner/:repo/languages", ({
|
||||
const response = await api.getActionsApiClient(true).repos.listLanguages({
|
||||
owner,
|
||||
repo
|
||||
}));
|
||||
});
|
||||
core.debug("Languages API response: " + JSON.stringify(response));
|
||||
// The GitHub API is going to return languages in order of popularity,
|
||||
// When we pick a language to autobuild we want to pick the most popular traced language
|
||||
// Since sets in javascript maintain insertion order, using a set here and then splatting it
|
||||
// into an array gives us an array of languages ordered by popularity
|
||||
let languages = new Set();
|
||||
for (let lang in response.data) {
|
||||
if (lang in codeqlLanguages) {
|
||||
languages.add(codeqlLanguages[lang]);
|
||||
for (let lang of Object.keys(response.data)) {
|
||||
let parsedLang = languages_1.parseLanguage(lang);
|
||||
if (parsedLang !== undefined) {
|
||||
languages.add(parsedLang);
|
||||
}
|
||||
}
|
||||
return [...languages];
|
||||
|
|
@ -335,6 +332,9 @@ async function getLanguagesInRepo() {
|
|||
* The result is obtained from the action input parameter 'languages' if that
|
||||
* has been set, otherwise it is deduced as all languages in the repo that
|
||||
* can be analysed.
|
||||
*
|
||||
* If no languages could be detected from either the workflow or the repository
|
||||
* then throw an error.
|
||||
*/
|
||||
async function getLanguages() {
|
||||
// Obtain from action input 'languages' if set
|
||||
|
|
@ -348,17 +348,37 @@ async function getLanguages() {
|
|||
languages = await getLanguagesInRepo();
|
||||
core.info("Automatically detected languages: " + JSON.stringify(languages));
|
||||
}
|
||||
return languages;
|
||||
// If the languages parameter was not given and no languages were
|
||||
// detected then fail here as this is a workflow configuration error.
|
||||
if (languages.length === 0) {
|
||||
throw new Error(getNoLanguagesError());
|
||||
}
|
||||
// Make sure they are supported
|
||||
const parsedLanguages = [];
|
||||
const unknownLanguages = [];
|
||||
for (let language of languages) {
|
||||
const parsedLanguage = languages_1.parseLanguage(language);
|
||||
if (parsedLanguage === undefined) {
|
||||
unknownLanguages.push(language);
|
||||
}
|
||||
else if (parsedLanguages.indexOf(parsedLanguage) === -1) {
|
||||
parsedLanguages.push(parsedLanguage);
|
||||
}
|
||||
}
|
||||
if (unknownLanguages.length > 0) {
|
||||
throw new Error(getUnknownLanguagesError(unknownLanguages));
|
||||
}
|
||||
return parsedLanguages;
|
||||
}
|
||||
/**
|
||||
* Returns true if queries were provided in the workflow file
|
||||
* (and thus added), otherwise false
|
||||
*/
|
||||
async function addQueriesFromWorkflowIfRequired(languages, resultMap, configFile) {
|
||||
async function addQueriesFromWorkflowIfRequired(codeQL, languages, resultMap, tempDir, configFile) {
|
||||
const queryUses = core.getInput('queries');
|
||||
if (queryUses) {
|
||||
for (const query of queryUses.split(',')) {
|
||||
await parseQueryUses(languages, resultMap, query, configFile);
|
||||
await parseQueryUses(languages, codeQL, resultMap, query, tempDir, configFile);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -367,24 +387,27 @@ async function addQueriesFromWorkflowIfRequired(languages, resultMap, configFile
|
|||
/**
|
||||
* Get the default config for when the user has not supplied one.
|
||||
*/
|
||||
async function getDefaultConfig() {
|
||||
async function getDefaultConfig(tempDir, toolCacheDir, codeQL) {
|
||||
const languages = await getLanguages();
|
||||
const queries = {};
|
||||
await addDefaultQueries(languages, queries);
|
||||
await addQueriesFromWorkflowIfRequired(languages, queries);
|
||||
await addDefaultQueries(codeQL, languages, queries);
|
||||
await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
|
||||
return {
|
||||
languages: languages,
|
||||
queries: queries,
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir,
|
||||
toolCacheDir,
|
||||
codeQLCmd: codeQL.getPath(),
|
||||
};
|
||||
}
|
||||
exports.getDefaultConfig = getDefaultConfig;
|
||||
/**
|
||||
* Load the config from the given file.
|
||||
*/
|
||||
async function loadConfig(configFile) {
|
||||
async function loadConfig(configFile, tempDir, toolCacheDir, codeQL) {
|
||||
let parsedYAML;
|
||||
if (isLocal(configFile)) {
|
||||
// Treat the config file as relative to the workspace
|
||||
|
|
@ -406,11 +429,6 @@ async function loadConfig(configFile) {
|
|||
}
|
||||
}
|
||||
const languages = await getLanguages();
|
||||
// If the languages parameter was not given and no languages were
|
||||
// detected then fail here as this is a workflow configuration error.
|
||||
if (languages.length === 0) {
|
||||
throw new Error("Did not detect any languages to analyze. Please update input in workflow.");
|
||||
}
|
||||
const queries = {};
|
||||
const pathsIgnore = [];
|
||||
const paths = [];
|
||||
|
|
@ -422,11 +440,11 @@ async function loadConfig(configFile) {
|
|||
disableDefaultQueries = parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY];
|
||||
}
|
||||
if (!disableDefaultQueries) {
|
||||
await addDefaultQueries(languages, queries);
|
||||
await addDefaultQueries(codeQL, languages, queries);
|
||||
}
|
||||
// If queries were provided using `with` in the action configuration,
|
||||
// they should take precedence over the queries in the config file
|
||||
const addedQueriesFromAction = await addQueriesFromWorkflowIfRequired(languages, queries, configFile);
|
||||
const addedQueriesFromAction = await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir, configFile);
|
||||
if (!addedQueriesFromAction && QUERIES_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
|
||||
throw new Error(getQueriesInvalid(configFile));
|
||||
|
|
@ -435,7 +453,7 @@ async function loadConfig(configFile) {
|
|||
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
|
||||
throw new Error(getQueryUsesInvalid(configFile));
|
||||
}
|
||||
await parseQueryUses(languages, queries, query[QUERIES_USES_PROPERTY], configFile);
|
||||
await parseQueryUses(languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir, configFile);
|
||||
}
|
||||
}
|
||||
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
|
||||
|
|
@ -460,12 +478,23 @@ async function loadConfig(configFile) {
|
|||
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile));
|
||||
});
|
||||
}
|
||||
// The list of queries should not be empty for any language. If it is then
|
||||
// it is a user configuration error.
|
||||
for (const language of languages) {
|
||||
if (queries[language] === undefined || queries[language].length === 0) {
|
||||
throw new Error(`Did not detect any queries to run for ${language}. ` +
|
||||
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
|
||||
}
|
||||
}
|
||||
return {
|
||||
languages,
|
||||
queries,
|
||||
pathsIgnore,
|
||||
paths,
|
||||
originalUserInput: parsedYAML
|
||||
originalUserInput: parsedYAML,
|
||||
tempDir,
|
||||
toolCacheDir,
|
||||
codeQLCmd: codeQL.getPath(),
|
||||
};
|
||||
}
|
||||
/**
|
||||
|
|
@ -474,16 +503,16 @@ async function loadConfig(configFile) {
|
|||
* This will parse the config from the user input if present, or generate
|
||||
* a default config. The parsed config is then stored to a known location.
|
||||
*/
|
||||
async function initConfig() {
|
||||
async function initConfig(tempDir, toolCacheDir, codeQL) {
|
||||
const configFile = core.getInput('config-file');
|
||||
let config;
|
||||
// If no config file was provided create an empty one
|
||||
if (configFile === '') {
|
||||
core.debug('No configuration file was provided');
|
||||
config = await getDefaultConfig();
|
||||
config = await getDefaultConfig(tempDir, toolCacheDir, codeQL);
|
||||
}
|
||||
else {
|
||||
config = await loadConfig(configFile);
|
||||
config = await loadConfig(configFile, tempDir, toolCacheDir, codeQL);
|
||||
}
|
||||
// Save the config so we can easily access it again in the future
|
||||
await saveConfig(config);
|
||||
|
|
@ -516,7 +545,7 @@ async function getRemoteConfig(configFile) {
|
|||
if (pieces === null || pieces.groups === undefined || pieces.length < 5) {
|
||||
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
|
||||
}
|
||||
const response = await api.getApiClient().repos.getContents({
|
||||
const response = await api.getActionsApiClient(true).repos.getContents({
|
||||
owner: pieces.groups.owner,
|
||||
repo: pieces.groups.repo,
|
||||
path: pieces.groups.path,
|
||||
|
|
@ -534,17 +563,11 @@ async function getRemoteConfig(configFile) {
|
|||
}
|
||||
return yaml.safeLoad(Buffer.from(fileContents, 'base64').toString('binary'));
|
||||
}
|
||||
/**
|
||||
* Get the directory where the parsed config will be stored.
|
||||
*/
|
||||
function getPathToParsedConfigFolder() {
|
||||
return util.getRequiredEnvParam('RUNNER_TEMP');
|
||||
}
|
||||
/**
|
||||
* Get the file path where the parsed config will be stored.
|
||||
*/
|
||||
function getPathToParsedConfigFile() {
|
||||
return path.join(getPathToParsedConfigFolder(), 'config');
|
||||
function getPathToParsedConfigFile(tempDir) {
|
||||
return path.join(tempDir, 'config');
|
||||
}
|
||||
exports.getPathToParsedConfigFile = getPathToParsedConfigFile;
|
||||
/**
|
||||
|
|
@ -552,8 +575,9 @@ exports.getPathToParsedConfigFile = getPathToParsedConfigFile;
|
|||
*/
|
||||
async function saveConfig(config) {
|
||||
const configString = JSON.stringify(config);
|
||||
await io.mkdirP(getPathToParsedConfigFolder());
|
||||
fs.writeFileSync(getPathToParsedConfigFile(), configString, 'utf8');
|
||||
const configFile = getPathToParsedConfigFile(config.tempDir);
|
||||
fs.mkdirSync(path.dirname(configFile), { recursive: true });
|
||||
fs.writeFileSync(configFile, configString, 'utf8');
|
||||
core.debug('Saved config:');
|
||||
core.debug(configString);
|
||||
}
|
||||
|
|
@ -565,8 +589,8 @@ async function saveConfig(config) {
|
|||
* stored to a known location. On the second and further calls, this will
|
||||
* return the contents of the parsed config from the known location.
|
||||
*/
|
||||
async function getConfig() {
|
||||
const configFile = getPathToParsedConfigFile();
|
||||
async function getConfig(tempDir) {
|
||||
const configFile = getPathToParsedConfigFile(tempDir);
|
||||
if (!fs.existsSync(configFile)) {
|
||||
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue