address review comments

This commit is contained in:
Robert Brignull 2020-07-15 17:36:49 +01:00
parent 7bb6ac6c60
commit b86c3701ed
14 changed files with 149 additions and 94 deletions

8
lib/codeql.js generated
View file

@ -15,7 +15,7 @@ const path = __importStar(require("path"));
const semver = __importStar(require("semver")); const semver = __importStar(require("semver"));
const util = __importStar(require("./util")); const util = __importStar(require("./util"));
/** /**
* Stores the CodeQL object, and is populated by `setupCodeQL`. * Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
* Can be overridden in tests using `setCodeQL`. * Can be overridden in tests using `setCodeQL`.
*/ */
let cachedCodeQL = undefined; let cachedCodeQL = undefined;
@ -88,6 +88,12 @@ function resolveFunction(partialCodeql, methodName) {
} }
return partialCodeql[methodName]; return partialCodeql[methodName];
} }
/**
* Set the functionality for CodeQL methods. Only for use in tests.
*
* Accepts a partial object and any undefined methods will be implemented
* to immediately throw an exception indicating which method is missing.
*/
function setCodeQL(partialCodeql) { function setCodeQL(partialCodeql) {
cachedCodeQL = { cachedCodeQL = {
getDir: resolveFunction(partialCodeql, 'getDir'), getDir: resolveFunction(partialCodeql, 'getDir'),

File diff suppressed because one or more lines are too long

94
lib/config-utils.js generated
View file

@ -18,7 +18,7 @@ const externalQueries = __importStar(require("./external-queries"));
const util = __importStar(require("./util")); const util = __importStar(require("./util"));
// Property names from the user-supplied config file. // Property names from the user-supplied config file.
const NAME_PROPERTY = 'name'; const NAME_PROPERTY = 'name';
const DISPLAY_DEFAULT_QUERIES_PROPERTY = 'disable-default-queries'; const DISABLE_DEFAULT_QUERIES_PROPERTY = 'disable-default-queries';
const QUERIES_PROPERTY = 'queries'; const QUERIES_PROPERTY = 'queries';
const QUERIES_USES_PROPERTY = 'uses'; const QUERIES_USES_PROPERTY = 'uses';
const PATHS_IGNORE_PROPERTY = 'paths-ignore'; const PATHS_IGNORE_PROPERTY = 'paths-ignore';
@ -42,6 +42,26 @@ function queryIsDisabled(language, query) {
return (DISABLED_BUILTIN_QUERIES[language] || []) return (DISABLED_BUILTIN_QUERIES[language] || [])
.some(disabledQuery => query.endsWith(disabledQuery)); .some(disabledQuery => query.endsWith(disabledQuery));
} }
/**
* Asserts that the noDeclaredLanguage and multipleDeclaredLanguages fields are
* both empty and errors if they are not.
*/
function validateQueries(resolvedQueries) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage;
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new Error('The following queries do not declare a language. ' +
'Their qlpack.yml files are either missing or is invalid.\n' +
noDeclaredLanguageQueries.join('\n'));
}
const multipleDeclaredLanguages = resolvedQueries.multipleDeclaredLanguages;
const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new Error('The following queries declare multiple languages. ' +
'Their qlpack.yml files are either missing or is invalid.\n' +
multipleDeclaredLanguagesQueries.join('\n'));
}
}
/** /**
* Run 'codeql resolve queries' and add the results to resultMap * Run 'codeql resolve queries' and add the results to resultMap
*/ */
@ -55,16 +75,7 @@ async function runResolveQueries(resultMap, toResolve, extraSearchPath, errorOnI
resultMap[language].push(...Object.keys(queries).filter(q => !queryIsDisabled(language, q))); resultMap[language].push(...Object.keys(queries).filter(q => !queryIsDisabled(language, q)));
} }
if (errorOnInvalidQueries) { if (errorOnInvalidQueries) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage; validateQueries(resolvedQueries);
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new Error('Some queries do not declare a language, their qlpack.yml file is missing or is invalid');
}
const multipleDeclaredLanguages = resolvedQueries.multipleDeclaredLanguages;
const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new Error('Some queries declare multiple languages, their qlpack.yml file is missing or is invalid');
}
} }
} }
/** /**
@ -77,9 +88,10 @@ async function addDefaultQueries(languages, resultMap) {
// The set of acceptable values for built-in suites from the codeql bundle // The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = ['security-extended', 'security-and-quality']; const builtinSuites = ['security-extended', 'security-and-quality'];
/** /**
* Parse the suitename to a set of queries and update resultMap. * Determine the set of queries associated with suiteName's suites and add them to resultMap.
* Throws an error if suiteName is not a valid builtin suite.
*/ */
async function parseBuiltinSuite(configFile, languages, resultMap, suiteName) { async function addBuiltinSuiteQueries(configFile, languages, resultMap, suiteName) {
const suite = builtinSuites.find((suite) => suite === suiteName); const suite = builtinSuites.find((suite) => suite === suiteName);
if (!suite) { if (!suite) {
throw new Error(getQueryUsesInvalid(configFile, suiteName)); throw new Error(getQueryUsesInvalid(configFile, suiteName));
@ -88,9 +100,9 @@ async function parseBuiltinSuite(configFile, languages, resultMap, suiteName) {
await runResolveQueries(resultMap, suites, undefined, false); await runResolveQueries(resultMap, suites, undefined, false);
} }
/** /**
* Parse the local path to a set of queries and update resultMap. * Retrieve the set of queries at localQueryPath and add them to resultMap.
*/ */
async function parseLocalQueryPath(configFile, resultMap, localQueryPath) { async function addLocalQueries(configFile, resultMap, localQueryPath) {
// Resolve the local path against the workspace so that when this is // Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to. // passed to codeql it resolves to exactly the path we expect it to resolve to.
const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE')); const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE'));
@ -110,9 +122,9 @@ async function parseLocalQueryPath(configFile, resultMap, localQueryPath) {
await runResolveQueries(resultMap, [absoluteQueryPath], rootOfRepo, true); await runResolveQueries(resultMap, [absoluteQueryPath], rootOfRepo, true);
} }
/** /**
* Parse the remote repo reference to a set of queries and update resultMap. * Retrieve the set of queries at the referenced remote repo and add them to resultMap.
*/ */
async function parseRemoteQuery(configFile, resultMap, queryUses) { async function addRemoteQueries(configFile, resultMap, queryUses) {
let tok = queryUses.split('@'); let tok = queryUses.split('@');
if (tok.length !== 2) { if (tok.length !== 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses)); throw new Error(getQueryUsesInvalid(configFile, queryUses));
@ -139,26 +151,29 @@ async function parseRemoteQuery(configFile, resultMap, queryUses) {
} }
/** /**
* Parse a query 'uses' field to a discrete set of query files and update resultMap. * Parse a query 'uses' field to a discrete set of query files and update resultMap.
*
* The logic for parsing the string is based on what actions does for
* parsing the 'uses' actions in the workflow file. So it can handle
* local paths starting with './', or references to remote repos, or
* a finite set of hardcoded terms for builtin suites.
*/ */
async function parseQueryUses(configFile, languages, resultMap, queryUses) { async function parseQueryUses(configFile, languages, resultMap, queryUses) {
// The logic for parsing the string is based on what actions does for
// parsing the 'uses' actions in the workflow file
queryUses = queryUses.trim(); queryUses = queryUses.trim();
if (queryUses === "") { if (queryUses === "") {
throw new Error(getQueryUsesInvalid(configFile)); throw new Error(getQueryUsesInvalid(configFile));
} }
// Check for the local path case before we start trying to parse the repository name // Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) { if (queryUses.startsWith("./")) {
await parseLocalQueryPath(configFile, resultMap, queryUses.slice(2)); await addLocalQueries(configFile, resultMap, queryUses.slice(2));
return; return;
} }
// Check for one of the builtin suites // Check for one of the builtin suites
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) { if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
await parseBuiltinSuite(configFile, languages, resultMap, queryUses); await addBuiltinSuiteQueries(configFile, languages, resultMap, queryUses);
return; return;
} }
// Otherwise, must be a reference to another repo // Otherwise, must be a reference to another repo
await parseRemoteQuery(configFile, resultMap, queryUses); await addRemoteQueries(configFile, resultMap, queryUses);
} }
// Regex validating stars in paths or paths-ignore entries. // Regex validating stars in paths or paths-ignore entries.
// The intention is to only allow ** to appear when immediately // The intention is to only allow ** to appear when immediately
@ -211,7 +226,7 @@ function getNameInvalid(configFile) {
} }
exports.getNameInvalid = getNameInvalid; exports.getNameInvalid = getNameInvalid;
function getDisableDefaultQueriesInvalid(configFile) { function getDisableDefaultQueriesInvalid(configFile) {
return getConfigFilePropertyError(configFile, DISPLAY_DEFAULT_QUERIES_PROPERTY, 'must be a boolean'); return getConfigFilePropertyError(configFile, DISABLE_DEFAULT_QUERIES_PROPERTY, 'must be a boolean');
} }
exports.getDisableDefaultQueriesInvalid = getDisableDefaultQueriesInvalid; exports.getDisableDefaultQueriesInvalid = getDisableDefaultQueriesInvalid;
function getQueriesInvalid(configFile) { function getQueriesInvalid(configFile) {
@ -328,7 +343,10 @@ async function getLanguages() {
} }
return languages; return languages;
} }
async function getBlankConfig() { /**
* Get the default config for when the user has not supplied one.
*/
async function getDefaultConfig() {
const languages = await getLanguages(); const languages = await getLanguages();
const queries = {}; const queries = {};
await addDefaultQueries(languages, queries); await addDefaultQueries(languages, queries);
@ -339,7 +357,7 @@ async function getBlankConfig() {
paths: [] paths: []
}; };
} }
exports.getBlankConfig = getBlankConfig; exports.getDefaultConfig = getDefaultConfig;
/** /**
* Load the config from the given file. * Load the config from the given file.
*/ */
@ -373,11 +391,11 @@ async function loadConfig(configFile) {
const queries = {}; const queries = {};
const pathsIgnore = []; const pathsIgnore = [];
const paths = []; const paths = [];
if (DISPLAY_DEFAULT_QUERIES_PROPERTY in parsedYAML) { if (DISABLE_DEFAULT_QUERIES_PROPERTY in parsedYAML) {
if (typeof parsedYAML[DISPLAY_DEFAULT_QUERIES_PROPERTY] !== "boolean") { if (typeof parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY] !== "boolean") {
throw new Error(getDisableDefaultQueriesInvalid(configFile)); throw new Error(getDisableDefaultQueriesInvalid(configFile));
} }
if (!parsedYAML[DISPLAY_DEFAULT_QUERIES_PROPERTY]) { if (!parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY]) {
await addDefaultQueries(languages, queries); await addDefaultQueries(languages, queries);
} }
} }
@ -423,12 +441,12 @@ async function loadConfig(configFile) {
* a default config. The parsed config is then stored to a known location. * a default config. The parsed config is then stored to a known location.
*/ */
async function initConfig() { async function initConfig() {
let configFile = core.getInput('config-file'); const configFile = core.getInput('config-file');
let config; let config;
// If no config file was provided create an empty one // If no config file was provided create an empty one
if (configFile === '') { if (configFile === '') {
core.debug('No configuration file was provided'); core.debug('No configuration file was provided');
config = await getBlankConfig(); config = await getDefaultConfig();
} }
else { else {
config = await loadConfig(configFile); config = await loadConfig(configFile);
@ -485,23 +503,23 @@ async function getRemoteConfig(configFile) {
/** /**
* Get the directory where the parsed config will be stored. * Get the directory where the parsed config will be stored.
*/ */
function getParsedConfigFolder() { function getPathToParsedConfigFolder() {
return util.getRequiredEnvParam('RUNNER_TEMP'); return util.getRequiredEnvParam('RUNNER_TEMP');
} }
/** /**
* Get the file path where the parsed config will be stored. * Get the file path where the parsed config will be stored.
*/ */
function getParsedConfigFile() { function getPathToParsedConfigFile() {
return path.join(getParsedConfigFolder(), 'config'); return path.join(getPathToParsedConfigFolder(), 'config');
} }
exports.getParsedConfigFile = getParsedConfigFile; exports.getPathToParsedConfigFile = getPathToParsedConfigFile;
/** /**
* Store the given config to the path returned from getParsedConfigFile. * Store the given config to the path returned from getPathToParsedConfigFile.
*/ */
async function saveConfig(config) { async function saveConfig(config) {
const configString = JSON.stringify(config); const configString = JSON.stringify(config);
await io.mkdirP(getParsedConfigFolder()); await io.mkdirP(getPathToParsedConfigFolder());
fs.writeFileSync(getParsedConfigFile(), configString, 'utf8'); fs.writeFileSync(getPathToParsedConfigFile(), configString, 'utf8');
core.debug('Saved config:'); core.debug('Saved config:');
core.debug(configString); core.debug(configString);
} }
@ -514,7 +532,7 @@ async function saveConfig(config) {
* return the contents of the parsed config from the known location. * return the contents of the parsed config from the known location.
*/ */
async function getConfig() { async function getConfig() {
const configFile = getParsedConfigFile(); const configFile = getPathToParsedConfigFile();
if (!fs.existsSync(configFile)) { if (!fs.existsSync(configFile)) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?"); throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
} }

File diff suppressed because one or more lines are too long

View file

@ -58,7 +58,7 @@ ava_1.default("load empty config", async (t) => {
}, },
}); });
const config = await configUtils.initConfig(); const config = await configUtils.initConfig();
t.deepEqual(config, await configUtils.getBlankConfig()); t.deepEqual(config, await configUtils.getDefaultConfig());
}); });
}); });
ava_1.default("loading config saves config", async (t) => { ava_1.default("loading config saves config", async (t) => {
@ -77,11 +77,13 @@ ava_1.default("loading config saves config", async (t) => {
}, },
}); });
// Sanity check the saved config file does not already exist // Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getParsedConfigFile())); t.false(fs.existsSync(configUtils.getPathToParsedConfigFile()));
// Sanity check that getConfig throws before we have called initConfig
t.throwsAsync(configUtils.getConfig);
const config1 = await configUtils.initConfig(); const config1 = await configUtils.initConfig();
// The saved config file should now exist // The saved config file should now exist
t.true(fs.existsSync(configUtils.getParsedConfigFile())); t.true(fs.existsSync(configUtils.getPathToParsedConfigFile()));
// And the same config should be returned again // And that same newly-initialised config should now be returned by getConfig
const config2 = await configUtils.getConfig(); const config2 = await configUtils.getConfig();
t.deepEqual(config1, config2); t.deepEqual(config1, config2);
}); });

File diff suppressed because one or more lines are too long

View file

@ -13,7 +13,7 @@ const fs = __importStar(require("fs"));
const path = __importStar(require("path")); const path = __importStar(require("path"));
const util = __importStar(require("./util")); const util = __importStar(require("./util"));
/** /**
* Checkout a repository at the given ref, and return the directory of the checkout. * Check out repository at the given ref, and return the directory of the checkout.
*/ */
async function checkoutExternalRepository(repository, ref) { async function checkoutExternalRepository(repository, ref) {
const folder = util.getRequiredEnvParam('RUNNER_TEMP'); const folder = util.getRequiredEnvParam('RUNNER_TEMP');

View file

@ -21,7 +21,7 @@ ava_1.default("checkoutExternalQueries", async (t) => {
await util.withTmpDir(async (tmpDir) => { await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir; process.env["RUNNER_TEMP"] = tmpDir;
await externalQueries.checkoutExternalRepository("github/codeql-go", "df4c6869212341b601005567381944ed90906b6b"); await externalQueries.checkoutExternalRepository("github/codeql-go", "df4c6869212341b601005567381944ed90906b6b");
// COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in master // COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in the default branch
t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", "COPYRIGHT"))); t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", "COPYRIGHT")));
}); });
}); });

View file

@ -1 +1 @@
{"version":3,"file":"external-queries.test.js","sourceRoot":"","sources":["../src/external-queries.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAuB;AACvB,uCAAyB;AACzB,2CAA6B;AAE7B,oEAAsD;AACtD,mDAA2C;AAC3C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,yBAAyB,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IACxC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;QACnC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC;QACpC,MAAM,eAAe,CAAC,0BAA0B,CAAC,kBAAkB,EAAE,0CAA0C,CAAC,CAAC;QAEjH,uFAAuF;QACvF,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"} {"version":3,"file":"external-queries.test.js","sourceRoot":"","sources":["../src/external-queries.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAuB;AACvB,uCAAyB;AACzB,2CAA6B;AAE7B,oEAAsD;AACtD,mDAA2C;AAC3C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,yBAAyB,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IACxC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;QACnC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC;QACpC,MAAM,eAAe,CAAC,0BAA0B,CAAC,kBAAkB,EAAE,0CAA0C,CAAC,CAAC;QAEjH,mGAAmG;QACnG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

View file

@ -63,7 +63,7 @@ export interface ResolveQueriesOutput {
} }
/** /**
* Stores the CodeQL object, and is populated by `setupCodeQL`. * Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
* Can be overridden in tests using `setCodeQL`. * Can be overridden in tests using `setCodeQL`.
*/ */
let cachedCodeQL: CodeQL | undefined = undefined; let cachedCodeQL: CodeQL | undefined = undefined;
@ -145,6 +145,12 @@ function resolveFunction<T>(partialCodeql: Partial<CodeQL>, methodName: string):
return partialCodeql[methodName]; return partialCodeql[methodName];
} }
/**
* Set the functionality for CodeQL methods. Only for use in tests.
*
* Accepts a partial object and any undefined methods will be implemented
* to immediately throw an exception indicating which method is missing.
*/
export function setCodeQL(partialCodeql: Partial<CodeQL>) { export function setCodeQL(partialCodeql: Partial<CodeQL>) {
cachedCodeQL = { cachedCodeQL = {
getDir: resolveFunction(partialCodeql, 'getDir'), getDir: resolveFunction(partialCodeql, 'getDir'),

View file

@ -56,7 +56,7 @@ test("load empty config", async t => {
const config = await configUtils.initConfig(); const config = await configUtils.initConfig();
t.deepEqual(config, await configUtils.getBlankConfig()); t.deepEqual(config, await configUtils.getDefaultConfig());
}); });
}); });
@ -78,15 +78,19 @@ test("loading config saves config", async t => {
}, },
}); });
// Sanity check the saved config file does not already exist // Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getParsedConfigFile())); t.false(fs.existsSync(configUtils.getPathToParsedConfigFile()));
// Sanity check that getConfig throws before we have called initConfig
t.throwsAsync(configUtils.getConfig);
const config1 = await configUtils.initConfig(); const config1 = await configUtils.initConfig();
// The saved config file should now exist // The saved config file should now exist
t.true(fs.existsSync(configUtils.getParsedConfigFile())); t.true(fs.existsSync(configUtils.getPathToParsedConfigFile()));
// And the same config should be returned again // And that same newly-initialised config should now be returned by getConfig
const config2 = await configUtils.getConfig(); const config2 = await configUtils.getConfig();
t.deepEqual(config1, config2); t.deepEqual(config1, config2);
}); });

View file

@ -5,13 +5,13 @@ import * as yaml from 'js-yaml';
import * as path from 'path'; import * as path from 'path';
import * as api from './api-client'; import * as api from './api-client';
import { getCodeQL } from './codeql'; import { getCodeQL, ResolveQueriesOutput } from './codeql';
import * as externalQueries from "./external-queries"; import * as externalQueries from "./external-queries";
import * as util from './util'; import * as util from './util';
// Property names from the user-supplied config file. // Property names from the user-supplied config file.
const NAME_PROPERTY = 'name'; const NAME_PROPERTY = 'name';
const DISPLAY_DEFAULT_QUERIES_PROPERTY = 'disable-default-queries'; const DISABLE_DEFAULT_QUERIES_PROPERTY = 'disable-default-queries';
const QUERIES_PROPERTY = 'queries'; const QUERIES_PROPERTY = 'queries';
const QUERIES_USES_PROPERTY = 'uses'; const QUERIES_USES_PROPERTY = 'uses';
const PATHS_IGNORE_PROPERTY = 'paths-ignore'; const PATHS_IGNORE_PROPERTY = 'paths-ignore';
@ -62,6 +62,28 @@ function queryIsDisabled(language, query): boolean {
.some(disabledQuery => query.endsWith(disabledQuery)); .some(disabledQuery => query.endsWith(disabledQuery));
} }
/**
* Asserts that the noDeclaredLanguage and multipleDeclaredLanguages fields are
* both empty and errors if they are not.
*/
function validateQueries(resolvedQueries: ResolveQueriesOutput) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage;
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new Error('The following queries do not declare a language. ' +
'Their qlpack.yml files are either missing or is invalid.\n' +
noDeclaredLanguageQueries.join('\n'));
}
const multipleDeclaredLanguages = resolvedQueries.multipleDeclaredLanguages;
const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new Error('The following queries declare multiple languages. ' +
'Their qlpack.yml files are either missing or is invalid.\n' +
multipleDeclaredLanguagesQueries.join('\n'));
}
}
/** /**
* Run 'codeql resolve queries' and add the results to resultMap * Run 'codeql resolve queries' and add the results to resultMap
*/ */
@ -82,17 +104,7 @@ async function runResolveQueries(
} }
if (errorOnInvalidQueries) { if (errorOnInvalidQueries) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage; validateQueries(resolvedQueries);
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new Error('Some queries do not declare a language, their qlpack.yml file is missing or is invalid');
}
const multipleDeclaredLanguages = resolvedQueries.multipleDeclaredLanguages;
const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new Error('Some queries declare multiple languages, their qlpack.yml file is missing or is invalid');
}
} }
} }
@ -108,9 +120,10 @@ async function addDefaultQueries(languages: string[], resultMap: { [language: st
const builtinSuites = ['security-extended', 'security-and-quality'] as const; const builtinSuites = ['security-extended', 'security-and-quality'] as const;
/** /**
* Parse the suitename to a set of queries and update resultMap. * Determine the set of queries associated with suiteName's suites and add them to resultMap.
* Throws an error if suiteName is not a valid builtin suite.
*/ */
async function parseBuiltinSuite( async function addBuiltinSuiteQueries(
configFile: string, configFile: string,
languages: string[], languages: string[],
resultMap: { [language: string]: string[] }, resultMap: { [language: string]: string[] },
@ -126,9 +139,9 @@ async function parseBuiltinSuite(
} }
/** /**
* Parse the local path to a set of queries and update resultMap. * Retrieve the set of queries at localQueryPath and add them to resultMap.
*/ */
async function parseLocalQueryPath( async function addLocalQueries(
configFile: string, configFile: string,
resultMap: { [language: string]: string[] }, resultMap: { [language: string]: string[] },
localQueryPath: string) { localQueryPath: string) {
@ -158,9 +171,9 @@ async function parseLocalQueryPath(
} }
/** /**
* Parse the remote repo reference to a set of queries and update resultMap. * Retrieve the set of queries at the referenced remote repo and add them to resultMap.
*/ */
async function parseRemoteQuery(configFile: string, resultMap: { [language: string]: string[] }, queryUses: string) { async function addRemoteQueries(configFile: string, resultMap: { [language: string]: string[] }, queryUses: string) {
let tok = queryUses.split('@'); let tok = queryUses.split('@');
if (tok.length !== 2) { if (tok.length !== 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses)); throw new Error(getQueryUsesInvalid(configFile, queryUses));
@ -193,6 +206,11 @@ async function parseRemoteQuery(configFile: string, resultMap: { [language: stri
/** /**
* Parse a query 'uses' field to a discrete set of query files and update resultMap. * Parse a query 'uses' field to a discrete set of query files and update resultMap.
*
* The logic for parsing the string is based on what actions does for
* parsing the 'uses' actions in the workflow file. So it can handle
* local paths starting with './', or references to remote repos, or
* a finite set of hardcoded terms for builtin suites.
*/ */
async function parseQueryUses( async function parseQueryUses(
configFile: string, configFile: string,
@ -200,8 +218,6 @@ async function parseQueryUses(
resultMap: { [language: string]: string[] }, resultMap: { [language: string]: string[] },
queryUses: string) { queryUses: string) {
// The logic for parsing the string is based on what actions does for
// parsing the 'uses' actions in the workflow file
queryUses = queryUses.trim(); queryUses = queryUses.trim();
if (queryUses === "") { if (queryUses === "") {
throw new Error(getQueryUsesInvalid(configFile)); throw new Error(getQueryUsesInvalid(configFile));
@ -209,18 +225,18 @@ async function parseQueryUses(
// Check for the local path case before we start trying to parse the repository name // Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) { if (queryUses.startsWith("./")) {
await parseLocalQueryPath(configFile, resultMap, queryUses.slice(2)); await addLocalQueries(configFile, resultMap, queryUses.slice(2));
return; return;
} }
// Check for one of the builtin suites // Check for one of the builtin suites
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) { if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
await parseBuiltinSuite(configFile, languages, resultMap, queryUses); await addBuiltinSuiteQueries(configFile, languages, resultMap, queryUses);
return; return;
} }
// Otherwise, must be a reference to another repo // Otherwise, must be a reference to another repo
await parseRemoteQuery(configFile, resultMap, queryUses); await addRemoteQueries(configFile, resultMap, queryUses);
} }
// Regex validating stars in paths or paths-ignore entries. // Regex validating stars in paths or paths-ignore entries.
@ -299,7 +315,7 @@ export function getNameInvalid(configFile: string): string {
} }
export function getDisableDefaultQueriesInvalid(configFile: string): string { export function getDisableDefaultQueriesInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, DISPLAY_DEFAULT_QUERIES_PROPERTY, 'must be a boolean'); return getConfigFilePropertyError(configFile, DISABLE_DEFAULT_QUERIES_PROPERTY, 'must be a boolean');
} }
export function getQueriesInvalid(configFile: string): string { export function getQueriesInvalid(configFile: string): string {
@ -433,7 +449,10 @@ async function getLanguages(): Promise<string[]> {
return languages; return languages;
} }
export async function getBlankConfig(): Promise<Config> { /**
* Get the default config for when the user has not supplied one.
*/
export async function getDefaultConfig(): Promise<Config> {
const languages = await getLanguages(); const languages = await getLanguages();
const queries = {}; const queries = {};
await addDefaultQueries(languages, queries); await addDefaultQueries(languages, queries);
@ -483,11 +502,11 @@ async function loadConfig(configFile: string): Promise<Config> {
const pathsIgnore: string[] = []; const pathsIgnore: string[] = [];
const paths: string[] = []; const paths: string[] = [];
if (DISPLAY_DEFAULT_QUERIES_PROPERTY in parsedYAML) { if (DISABLE_DEFAULT_QUERIES_PROPERTY in parsedYAML) {
if (typeof parsedYAML[DISPLAY_DEFAULT_QUERIES_PROPERTY] !== "boolean") { if (typeof parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY] !== "boolean") {
throw new Error(getDisableDefaultQueriesInvalid(configFile)); throw new Error(getDisableDefaultQueriesInvalid(configFile));
} }
if (!parsedYAML[DISPLAY_DEFAULT_QUERIES_PROPERTY]) { if (!parsedYAML[DISABLE_DEFAULT_QUERIES_PROPERTY]) {
await addDefaultQueries(languages, queries); await addDefaultQueries(languages, queries);
} }
} }
@ -538,13 +557,13 @@ async function loadConfig(configFile: string): Promise<Config> {
* a default config. The parsed config is then stored to a known location. * a default config. The parsed config is then stored to a known location.
*/ */
export async function initConfig(): Promise<Config> { export async function initConfig(): Promise<Config> {
let configFile = core.getInput('config-file'); const configFile = core.getInput('config-file');
let config: Config; let config: Config;
// If no config file was provided create an empty one // If no config file was provided create an empty one
if (configFile === '') { if (configFile === '') {
core.debug('No configuration file was provided'); core.debug('No configuration file was provided');
config = await getBlankConfig(); config = await getDefaultConfig();
} else { } else {
config = await loadConfig(configFile); config = await loadConfig(configFile);
} }
@ -608,24 +627,24 @@ async function getRemoteConfig(configFile: string): Promise<any> {
/** /**
* Get the directory where the parsed config will be stored. * Get the directory where the parsed config will be stored.
*/ */
function getParsedConfigFolder(): string { function getPathToParsedConfigFolder(): string {
return util.getRequiredEnvParam('RUNNER_TEMP'); return util.getRequiredEnvParam('RUNNER_TEMP');
} }
/** /**
* Get the file path where the parsed config will be stored. * Get the file path where the parsed config will be stored.
*/ */
export function getParsedConfigFile(): string { export function getPathToParsedConfigFile(): string {
return path.join(getParsedConfigFolder(), 'config'); return path.join(getPathToParsedConfigFolder(), 'config');
} }
/** /**
* Store the given config to the path returned from getParsedConfigFile. * Store the given config to the path returned from getPathToParsedConfigFile.
*/ */
async function saveConfig(config: Config) { async function saveConfig(config: Config) {
const configString = JSON.stringify(config); const configString = JSON.stringify(config);
await io.mkdirP(getParsedConfigFolder()); await io.mkdirP(getPathToParsedConfigFolder());
fs.writeFileSync(getParsedConfigFile(), configString, 'utf8'); fs.writeFileSync(getPathToParsedConfigFile(), configString, 'utf8');
core.debug('Saved config:'); core.debug('Saved config:');
core.debug(configString); core.debug(configString);
} }
@ -639,7 +658,7 @@ async function saveConfig(config: Config) {
* return the contents of the parsed config from the known location. * return the contents of the parsed config from the known location.
*/ */
export async function getConfig(): Promise<Config> { export async function getConfig(): Promise<Config> {
const configFile = getParsedConfigFile(); const configFile = getPathToParsedConfigFile();
if (!fs.existsSync(configFile)) { if (!fs.existsSync(configFile)) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?"); throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
} }

View file

@ -13,7 +13,7 @@ test("checkoutExternalQueries", async t => {
process.env["RUNNER_TEMP"] = tmpDir; process.env["RUNNER_TEMP"] = tmpDir;
await externalQueries.checkoutExternalRepository("github/codeql-go", "df4c6869212341b601005567381944ed90906b6b"); await externalQueries.checkoutExternalRepository("github/codeql-go", "df4c6869212341b601005567381944ed90906b6b");
// COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in master // COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in the default branch
t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", "COPYRIGHT"))); t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", "COPYRIGHT")));
}); });
}); });

View file

@ -6,7 +6,7 @@ import * as path from 'path';
import * as util from './util'; import * as util from './util';
/** /**
* Checkout a repository at the given ref, and return the directory of the checkout. * Check out repository at the given ref, and return the directory of the checkout.
*/ */
export async function checkoutExternalRepository(repository: string, ref: string): Promise<string> { export async function checkoutExternalRepository(repository: string, ref: string): Promise<string> {
const folder = util.getRequiredEnvParam('RUNNER_TEMP'); const folder = util.getRequiredEnvParam('RUNNER_TEMP');