robojerk/codeql-action

Merge branch 'master' into safe-config-parsing

Browse source
This commit is contained in:
Robert Brignull 2020-06-08 13:40:17 +01:00
commit e35c90f53d
21 changed files with 388 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View file

@ -0,0 +1,5 @@
blank_issues_enabled: true
contact_links:
- name: Contact GitHub Support
url: https://support.github.com/contact?subject=Code+Scanning+Beta+Support&tags=code-scanning-support
about: Contact Support about code scanning

7
.github/codeql/codeql-config.yml vendored
View file

@ -2,5 +2,12 @@ name: "CodeQL config"
queries:
- name: Run custom queries
uses: ./queries
# Run all extra query suites, both because we want to
# and because it'll act as extra testing. This is why
# we include both even though one is a superset of the
# other, because we're testing the parsing logic and
# that the suites exist in the codeql bundle.
- uses: security-extended
- uses: security-and-quality
paths-ignore:
- tests

12
.github/workflows/codeql.yml vendored
View file

@ -1,6 +1,6 @@
name: "CodeQL action"
on: [push]
on: [push, pull_request]
jobs:
build:
@ -11,6 +11,16 @@ jobs:
steps:
- uses: actions/checkout@v1
with:
# Must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head of the pull request.
fetch-depth: 2
# If this run was triggered by a pull request event then checkout
# the head of the pull request instead of the merge commit.
- run: git checkout HEAD^2
if: ${{ github.event_name == 'pull_request' }}
- uses: ./init
with:
languages: javascript

44
.github/workflows/integration-testing.yml vendored
View file

@ -1,14 +1,10 @@
name: "Integration Testing"
on: [push]
on: [push, pull_request]
jobs:
multi-language-repo_test-autodetect-languages:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
@ -16,9 +12,8 @@ jobs:
shell: bash
run: |
mkdir ../action
shopt -s dotglob
mv * ../action/
mv ../action/tests/multi-language-repo/* .
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
- uses: ./../action/init
- name: Build code
shell: bash
@ -26,6 +21,20 @@ jobs:
- uses: ./../action/analyze
env:
TEST_MODE: true
- run: |
cd "$CODEQL_ACTION_DATABASE_DIR"
# List all directories as there will be precisely one directory per database
# but there may be other files in this directory such as query suites.
if [ "$(ls -d */ | wc -l)" != 6 ] || \
[[ ! -d cpp ]] || \
[[ ! -d csharp ]] || \
[[ ! -d go ]] || \
[[ ! -d java ]] || \
[[ ! -d javascript ]] || \
[[ ! -d python ]]; then
echo "Did not find expected number of databases. Database dir contains: $(ls)"
exit 1
fi
multi-language-repo_test-custom-queries:
strategy:
@ -40,9 +49,8 @@ jobs:
shell: bash
run: |
mkdir ../action
shopt -s dotglob
mv * ../action/
mv ../action/tests/multi-language-repo/* .
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
- uses: ./../action/init
with:
languages: cpp,csharp,java,javascript,python
@ -72,9 +80,8 @@ jobs:
shell: bash
run: |
mkdir ../action
shopt -s dotglob
mv * ../action/
mv ../action/tests/multi-language-repo/* .
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
- uses: ./../action/init
with:
languages: go
@ -96,9 +103,8 @@ jobs:
shell: bash
run: |
mkdir ../action
shopt -s dotglob
mv * ../action/
mv ../action/tests/multi-language-repo/* .
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
@ -117,4 +123,4 @@ jobs:
with:
sarif_file: rubocop.sarif
env:
TEST_MODE: true
TEST_MODE: true

101
README.md
View file

@ -10,6 +10,8 @@ The underlying CodeQL CLI, used in this action, is licensed under the [GitHub Co
## Usage
This is a short walkthrough, but for more information read [configuring code scanning](https://help.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning).
To get code scanning results from CodeQL analysis on your repo you can use the following workflow as a template:
```yaml
@ -18,6 +20,7 @@ name: "Code Scanning - Action"
on:
push:
pull_request:
schedule:
- cron: '0 0 * * 0'
@ -33,6 +36,17 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
# Must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head of the pull request.
# Only include this option if you are running this workflow on pull requests.
fetch-depth: 2
# If this run was triggered by a pull request event then checkout
# the head of the pull request instead of the merge commit.
# Only include this step if you are running this workflow on pull requests.
- run: git checkout HEAD^2
if: ${{ github.event_name == 'pull_request' }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
@ -78,24 +92,9 @@ If you prefer to integrate this within an existing CI workflow, it should end up
uses: github/codeql-action/analyze@v1
```
### Actions triggers
### Configuration file
The CodeQL action should be run on `push` events, and on a `schedule`. `Push` events allow us to do a detailed analysis of the delta in a pull request, while the `schedule` event ensures that GitHub regularly scans the repository for the latest vulnerabilities, even if the repository becomes inactive. This action does not support the `pull_request` event.
### Configuration
You may optionally specify additional queries for CodeQL to execute by using a config file. The queries must belong to a [QL pack](https://help.semmle.com/codeql/codeql-cli/reference/qlpack-overview.html) and can be in your repository or any public repository. You can choose a single .ql file, a folder containing multiple .ql files, a .qls [query suite](https://help.semmle.com/codeql/codeql-cli/procedures/query-suites.html) file, or any combination of the above. To use queries stored in your repository or from other repositories use the same syntax as when [using an action](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsuses). Note that when using local queries starting with `./`, the path is relative to the root of the repository and not to the location of the config file.
You can disable the default queries using `disable-default-queries: true`.
You can choose to ignore some files or folders from the analysis, or include additional files/folders for analysis. This *only* works for Javascript and Python analysis.
Identifying potential files for extraction:
- Scans each folder that's defined as `paths` in turn, traversing subfolders, and looking for relevant files.
- If it finds a subfolder that's defined as `paths-ignore`, stop traversing.
- If a file or folder is both in `paths` and `paths-ignore`, the `paths-ignore` is ignored.
Use the `config-file` parameter of the init action to enable the configuration file. For example:
Use the `config-file` parameter of the `init` action to enable the configuration file. The value of `config-file` is the path to the configuration file you want to use. This example loads the configuration file `./.github/codeql/codeql-config.yml`.
```yaml
- uses: github/codeql-action/init@v1
@ -103,72 +102,8 @@ Use the `config-file` parameter of the init action to enable the configuration f
config-file: ./.github/codeql/codeql-config.yml
```
A config file looks like this:
```yaml
name: "My CodeQL config"
disable-default-queries: true
queries:
- name: In-repo queries (Runs the queries located in the my-queries folder of the repo)
uses: ./my-queries
- name: External Javascript QL pack (Runs a QL pack located in an external repo)
uses: /Semmle/ql/javascript/ql/src/Electron@master
- name: External query (Runs a single query located in an external QL pack)
uses: Semmle/ql/javascript/ql/src/AngularJS/DeadAngularJSEventListener.ql@master
- name: Select query suite (Runs a query suites)
uses: ./codeql-querypacks/complex-python-querypack/rootAndBar.qls
paths:
- src/util.ts
paths-ignore:
- src
- lib
```
The configuration file must be located within the local repository. For information on how to write a configuration file, see "[Using a custom configuration](https://help.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration)."
## Troubleshooting
### Trouble with Go dependencies
#### If you use a vendor directory
Try passing
```yaml
env:
GOFLAGS: "-mod=vendor"
```
to `github/codeql-action/analyze`.
#### If you do not use a vendor directory
Dependencies on public repositories should just work. If you have dependencies on private repositories, one option is to use `git config` and a [personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) to authenticate when downloading dependencies. Add a section like
```yaml
steps:
- name: Configure git private repo access
env:
TOKEN: ${{ secrets.GITHUB_PAT }}
run: |
git config --global url."https://${TOKEN}@github.com/foo/bar".insteadOf "https://github.com/foo/bar"
git config --global url."https://${TOKEN}@github.com/foo/baz".insteadOf "https://github.com/foo/baz"
```
before any codeql actions. A similar thing can also be done with an SSH key or deploy key.
### C# using dotnet version 2 on linux
This currently requires invoking `dotnet` with the `/p:UseSharedCompilation=false` flag. For example:
```shell
dotnet build /p:UseSharedCompilation=false
```
Version 3 does not require the additional flag.
### Analysing Go together with other languages on `macos-latest`
When running on macos it is currently not possible to analyze Go in conjunction with any of Java, C/C++, or C#. Each language can still be analyzed separately.
Read about [troubleshooting code scanning](https://help.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning).

2
init/action.yml
View file

@ -5,7 +5,7 @@ inputs:
tools:
description: URL of CodeQL tools
required: false
default: https://github.com/github/codeql-action/releases/download/codeql-bundle-20200427/codeql-bundle.tar.gz
default: https://github.com/github/codeql-action/releases/download/codeql-bundle-20200601/codeql-bundle.tar.gz
languages:
description: The languages to be analysed
required: false

40
lib/config-utils.js generated
View file

@ -27,12 +27,15 @@ class ExternalQuery {
}
}
exports.ExternalQuery = ExternalQuery;
// The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = ['security-extended', 'security-and-quality'];
class Config {
constructor() {
this.name = "";
this.disableDefaultQueries = false;
this.additionalQueries = [];
this.externalQueries = [];
this.additionalSuites = [];
this.pathsIgnore = [];
this.paths = [];
}
@ -45,9 +48,33 @@ class Config {
}
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
this.additionalQueries.push(queryUses.slice(2));
const localQueryPath = queryUses.slice(2);
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
const workspacePath = util.getRequiredEnvParam('GITHUB_WORKSPACE');
const absoluteQueryPath = path.join(workspacePath, localQueryPath);
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
throw new Error(getLocalPathDoesNotExist(configFile, localQueryPath));
}
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (!(fs.realpathSync(absoluteQueryPath) + path.sep).startsWith(workspacePath + path.sep)) {
throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
}
this.additionalQueries.push(absoluteQueryPath);
return;
}
// Check for one of the builtin suites
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
const suite = builtinSuites.find((suite) => suite === queryUses);
if (suite) {
this.additionalSuites.push(suite);
return;
}
else {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
}
}
let tok = queryUses.split('@');
if (tok.length !== 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
@ -88,7 +115,8 @@ function getQueriesInvalid(configFile) {
}
exports.getQueriesInvalid = getQueriesInvalid;
function getQueryUsesInvalid(configFile, queryUses) {
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY, 'must be non-empty string containing either a local path starting with "./", or be of the form "owner/repo[/path]@ref"' +
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY, 'must be a built-in suite (' + builtinSuites.join(' or ') +
'), a relative path, or be of the form "owner/repo[/path]@ref"' +
(queryUses !== undefined ? '\n Found: ' + queryUses : ''));
}
exports.getQueryUsesInvalid = getQueryUsesInvalid;
@ -100,6 +128,14 @@ function getPathsInvalid(configFile) {
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, 'must be an array of non-empty string');
}
exports.getPathsInvalid = getPathsInvalid;
function getLocalPathOutsideOfRepository(configFile, localPath) {
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY, 'is invalid as the local path "' + localPath + '" is output of the repository');
}
exports.getLocalPathOutsideOfRepository = getLocalPathOutsideOfRepository;
function getLocalPathDoesNotExist(configFile, localPath) {
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY, 'is invalid as the local path "' + localPath + '" does not exist in the repository');
}
exports.getLocalPathDoesNotExist = getLocalPathDoesNotExist;
function getConfigFileOutsideWorkspaceErrorMessage(configFile) {
return 'The configuration file "' + configFile + '" is outside of the workspace';
}

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

21
lib/config-utils.test.js generated
View file

@ -87,6 +87,7 @@ ava_1.default("load non-empty input", async (t) => {
name: my config
disable-default-queries: true
queries:
- uses: ./
- uses: ./foo
- uses: foo/bar@dev
paths-ignore:
@ -98,12 +99,14 @@ ava_1.default("load non-empty input", async (t) => {
const expectedConfig = new configUtils.Config();
expectedConfig.name = 'my config';
expectedConfig.disableDefaultQueries = true;
expectedConfig.additionalQueries.push('foo');
expectedConfig.additionalQueries.push(tmpDir);
expectedConfig.additionalQueries.push(path.join(tmpDir, 'foo'));
expectedConfig.externalQueries = [new configUtils.ExternalQuery('foo/bar', 'dev')];
expectedConfig.pathsIgnore = ['a', 'b'];
expectedConfig.paths = ['c/d'];
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
fs.mkdirSync(path.join(tmpDir, 'foo'));
const actualConfig = await configUtils.loadConfig();
// Should exactly equal the object we constructed earlier
t.deepEqual(actualConfig, expectedConfig);
@ -138,7 +141,7 @@ doInvalidInputTest('queries uses invalid type', `
queries:
- uses:
- hello: world`, configUtils.getQueryUsesInvalid);
function doInvalidQueryUsesTest(input, inputInErrorMessage) {
function doInvalidQueryUsesTest(input, expectedErrorMessageGenerator) {
// Invalid contents of a "queries.uses" field.
// Should fail with the expected error message
const inputFileContents = `
@ -146,12 +149,14 @@ function doInvalidQueryUsesTest(input, inputInErrorMessage) {
queries:
- name: foo
uses: ` + input;
doInvalidInputTest("queries uses \"" + input + "\"", inputFileContents, configFile => configUtils.getQueryUsesInvalid(configFile, inputInErrorMessage ? input : undefined));
doInvalidInputTest("queries uses \"" + input + "\"", inputFileContents, expectedErrorMessageGenerator);
}
// Various "uses" fields, and the errors they should produce
doInvalidQueryUsesTest("''", false);
doInvalidQueryUsesTest("foo/bar", true);
doInvalidQueryUsesTest("foo/bar@v1@v2", true);
doInvalidQueryUsesTest("foo@master", true);
doInvalidQueryUsesTest("https://github.com/foo/bar@master", true);
doInvalidQueryUsesTest("''", c => configUtils.getQueryUsesInvalid(c, undefined));
doInvalidQueryUsesTest("foo/bar", c => configUtils.getQueryUsesInvalid(c, "foo/bar"));
doInvalidQueryUsesTest("foo/bar@v1@v2", c => configUtils.getQueryUsesInvalid(c, "foo/bar@v1@v2"));
doInvalidQueryUsesTest("foo@master", c => configUtils.getQueryUsesInvalid(c, "foo@master"));
doInvalidQueryUsesTest("https://github.com/foo/bar@master", c => configUtils.getQueryUsesInvalid(c, "https://github.com/foo/bar@master"));
doInvalidQueryUsesTest("./foo", c => configUtils.getLocalPathDoesNotExist(c, "foo"));
doInvalidQueryUsesTest("./..", c => configUtils.getLocalPathOutsideOfRepository(c, ".."));
//# sourceMappingURL=config-utils.test.js.map

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

71
lib/finalize-db.js generated
View file

@ -67,26 +67,50 @@ async function finalizeDatabaseCreation(codeqlCmd, databaseFolder) {
core.endGroup();
}
}
async function runResolveQueries(codeqlCmd, queries) {
let output = '';
const options = {
listeners: {
stdout: (data) => {
output += data.toString();
}
}
};
await exec.exec(codeqlCmd, [
'resolve',
'queries',
...queries,
'--format=bylanguage'
], options);
return JSON.parse(output);
}
async function resolveQueryLanguages(codeqlCmd, config) {
let res = new Map();
if (config.additionalQueries.length !== 0) {
let resolveQueriesOutput = '';
const options = {
listeners: {
stdout: (data) => {
resolveQueriesOutput += data.toString();
}
if (!config.disableDefaultQueries || config.additionalSuites.length !== 0) {
const suites = [];
for (const language of await util.getLanguages()) {
if (!config.disableDefaultQueries) {
suites.push(language + '-code-scanning.qls');
}
};
await exec.exec(codeqlCmd, [
'resolve',
'queries',
...config.additionalQueries,
'--format=bylanguage'
], options);
const resolveQueriesOutputObject = JSON.parse(resolveQueriesOutput);
for (const additionalSuite of config.additionalSuites) {
suites.push(language + '-' + additionalSuite + '.qls');
}
}
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, suites);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
res[language] = Object.keys(queries);
if (res[language] === undefined) {
res[language] = [];
}
res[language].push(...Object.keys(queries));
}
}
if (config.additionalQueries.length !== 0) {
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, config.additionalQueries);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
if (res[language] === undefined) {
res[language] = [];
}
res[language].push(...Object.keys(queries));
}
const noDeclaredLanguage = resolveQueriesOutputObject.noDeclaredLanguage;
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
@ -106,11 +130,16 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
const queriesPerLanguage = await resolveQueryLanguages(codeqlCmd, config);
for (let database of fs.readdirSync(databaseFolder)) {
core.startGroup('Analyzing ' + database);
const queries = [];
if (!config.disableDefaultQueries) {
queries.push(database + '-code-scanning.qls');
const queries = queriesPerLanguage[database] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + database + ' as no queries were selected for this language');
}
queries.push(...(queriesPerLanguage[database] || []));
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = path.join(databaseFolder, database + '-queries.qls');
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + database + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, database + '.sarif');
await exec.exec(codeqlCmd, [
'database',
@ -120,7 +149,7 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
...queries
querySuite
]);
core.debug('SARIF results for database ' + database + ' created at "' + sarifFile + '"');
core.endGroup();

2
lib/finalize-db.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/upload-lib.js generated
View file

@ -133,7 +133,7 @@ async function uploadFiles(sarifFiles) {
return false;
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
const commitOid = util.getRequiredEnvParam('GITHUB_SHA');
const commitOid = await util.getCommitOid();
const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID');
const ref = util.getRef();
const analysisKey = await util.getAnalysisKey();

2
lib/upload-lib.js.map
View file

File diff suppressed because one or more lines are too long

38
lib/util.js generated
View file

@ -11,6 +11,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(require("@actions/core"));
const exec = __importStar(require("@actions/exec"));
const http = __importStar(require("@actions/http-client"));
const auth = __importStar(require("@actions/http-client/auth"));
const octokit = __importStar(require("@octokit/rest"));
@ -33,12 +34,6 @@ function should_abort(actionName, requireInitActionHasRun) {
core.setFailed('GITHUB_REF must be set.');
return true;
}
// Should abort if called on a merge commit for a pull request.
if (ref.startsWith('refs/pull/')) {
core.warning('The CodeQL ' + actionName + ' action is intended for workflows triggered on `push` events, '
+ 'but the current workflow is running on a pull request. Aborting.');
return true;
}
// If the init action is required, then check the it completed successfully.
if (requireInitActionHasRun && process.env[sharedEnv.CODEQL_ACTION_INIT_COMPLETED] === undefined) {
core.setFailed('The CodeQL ' + actionName + ' action cannot be used unless the CodeQL init action is run first. Aborting.');
@ -151,6 +146,21 @@ async function getLanguages() {
return languages;
}
exports.getLanguages = getLanguages;
/**
* Gets the SHA of the commit that is currently checked out.
*/
async function getCommitOid() {
let commitOid = '';
await exec.exec('git', ['rev-parse', 'HEAD'], {
silent: true,
listeners: {
stdout: (data) => { commitOid += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
return commitOid.trim();
}
exports.getCommitOid = getCommitOid;
/**
* Get the path of the currently executing workflow.
*/
@ -196,8 +206,20 @@ exports.getAnalysisKey = getAnalysisKey;
* Get the ref currently being analyzed.
*/
function getRef() {
// it's in the form "refs/heads/master"
return getRequiredEnvParam('GITHUB_REF');
// Will be in the form "refs/heads/master" on a push event
// or in the form "refs/pull/N/merge" on a pull_request event
const ref = getRequiredEnvParam('GITHUB_REF');
// For pull request refs we want to convert from the 'merge' ref
// to the 'head' ref, as that is what we want to analyse.
// There should have been some code earlier in the workflow to do
// the checkout, but we have no way of verifying that here.
const pull_ref_regex = /refs\/pull\/(\d+)\/merge/;
if (pull_ref_regex.test(ref)) {
return ref.replace(pull_ref_regex, 'refs/pull/$1/head');
}
else {
return ref;
}
}
exports.getRef = getRef;
/**

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

41
src/config-utils.test.ts
View file

@ -91,6 +91,7 @@ test("load non-empty input", async t => {
name: my config
disable-default-queries: true
queries:
- uses: ./
- uses: ./foo
- uses: foo/bar@dev
paths-ignore:
@ -103,7 +104,8 @@ test("load non-empty input", async t => {
const expectedConfig = new configUtils.Config();
expectedConfig.name = 'my config';
expectedConfig.disableDefaultQueries = true;
expectedConfig.additionalQueries.push('foo');
expectedConfig.additionalQueries.push(tmpDir);
expectedConfig.additionalQueries.push(path.join(tmpDir, 'foo'));
expectedConfig.externalQueries = [new configUtils.ExternalQuery('foo/bar', 'dev')];
expectedConfig.pathsIgnore = ['a', 'b'];
expectedConfig.paths = ['c/d'];
@ -111,6 +113,8 @@ test("load non-empty input", async t => {
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
fs.mkdirSync(path.join(tmpDir, 'foo'));
const actualConfig = await configUtils.loadConfig();
// Should exactly equal the object we constructed earlier
@ -177,7 +181,10 @@ doInvalidInputTest(
- hello: world`,
configUtils.getQueryUsesInvalid);
function doInvalidQueryUsesTest(input: string, inputInErrorMessage: boolean) {
function doInvalidQueryUsesTest(
input: string,
expectedErrorMessageGenerator: (configFile: string) => string) {
// Invalid contents of a "queries.uses" field.
// Should fail with the expected error message
const inputFileContents = `
@ -189,14 +196,28 @@ function doInvalidQueryUsesTest(input: string, inputInErrorMessage: boolean) {
doInvalidInputTest(
"queries uses \"" + input + "\"",
inputFileContents,
configFile => configUtils.getQueryUsesInvalid(
configFile,
inputInErrorMessage ? input : undefined));
expectedErrorMessageGenerator);
}
// Various "uses" fields, and the errors they should produce
doInvalidQueryUsesTest("''", false);
doInvalidQueryUsesTest("foo/bar", true);
doInvalidQueryUsesTest("foo/bar@v1@v2", true);
doInvalidQueryUsesTest("foo@master", true);
doInvalidQueryUsesTest("https://github.com/foo/bar@master", true);
doInvalidQueryUsesTest(
"''",
c => configUtils.getQueryUsesInvalid(c, undefined));
doInvalidQueryUsesTest(
"foo/bar",
c => configUtils.getQueryUsesInvalid(c, "foo/bar"));
doInvalidQueryUsesTest(
"foo/bar@v1@v2",
c => configUtils.getQueryUsesInvalid(c, "foo/bar@v1@v2"));
doInvalidQueryUsesTest(
"foo@master",
c => configUtils.getQueryUsesInvalid(c, "foo@master"));
doInvalidQueryUsesTest(
"https://github.com/foo/bar@master",
c => configUtils.getQueryUsesInvalid(c, "https://github.com/foo/bar@master"));
doInvalidQueryUsesTest(
"./foo",
c => configUtils.getLocalPathDoesNotExist(c, "foo"));
doInvalidQueryUsesTest(
"./..",
c => configUtils.getLocalPathOutsideOfRepository(c, ".."));

52
src/config-utils.ts
View file

@ -24,11 +24,17 @@ export class ExternalQuery {
}
}
// The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = ['security-extended', 'security-and-quality'] as const;
// Derive the union type from the array values
type BuiltInSuite = typeof builtinSuites[number];
export class Config {
public name = "";
public disableDefaultQueries = false;
public additionalQueries: string[] = [];
public externalQueries: ExternalQuery[] = [];
public additionalSuites: BuiltInSuite[] = [];
public pathsIgnore: string[] = [];
public paths: string[] = [];
@ -42,10 +48,37 @@ export class Config {
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
this.additionalQueries.push(queryUses.slice(2));
const localQueryPath = queryUses.slice(2);
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
const workspacePath = util.getRequiredEnvParam('GITHUB_WORKSPACE');
const absoluteQueryPath = path.join(workspacePath, localQueryPath);
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
throw new Error(getLocalPathDoesNotExist(configFile, localQueryPath));
}
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (!(fs.realpathSync(absoluteQueryPath) + path.sep).startsWith(workspacePath + path.sep)) {
throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
}
this.additionalQueries.push(absoluteQueryPath);
return;
}
// Check for one of the builtin suites
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
const suite = builtinSuites.find((suite) => suite === queryUses);
if (suite) {
this.additionalSuites.push(suite);
return;
} else {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
}
}
let tok = queryUses.split('@');
if (tok.length !== 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
@ -92,7 +125,8 @@ export function getQueryUsesInvalid(configFile: string, queryUses?: string): str
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'must be non-empty string containing either a local path starting with "./", or be of the form "owner/repo[/path]@ref"' +
'must be a built-in suite (' + builtinSuites.join(' or ') +
'), a relative path, or be of the form "owner/repo[/path]@ref"' +
(queryUses !== undefined ? '\n Found: ' + queryUses : ''));
}
@ -104,6 +138,20 @@ export function getPathsInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, 'must be an array of non-empty string');
}
export function getLocalPathOutsideOfRepository(configFile: string, localPath: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'is invalid as the local path "' + localPath + '" is output of the repository');
}
export function getLocalPathDoesNotExist(configFile: string, localPath: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'is invalid as the local path "' + localPath + '" does not exist in the repository');
}
export function getConfigFileOutsideWorkspaceErrorMessage(configFile: string): string {
return 'The configuration file "' + configFile + '" is outside of the workspace';
}

96
src/finalize-db.ts
View file

@ -69,32 +69,74 @@ async function finalizeDatabaseCreation(codeqlCmd: string, databaseFolder: strin
}
}
interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
[queryPath: string]: {}
}
};
noDeclaredLanguage: {
[queryPath: string]: {}
};
multipleDeclaredLanguages: {
[queryPath: string]: {}
};
}
async function runResolveQueries(codeqlCmd: string, queries: string[]): Promise<ResolveQueriesOutput> {
let output = '';
const options = {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
}
}
};
await exec.exec(
codeqlCmd, [
'resolve',
'queries',
...queries,
'--format=bylanguage'
],
options);
return JSON.parse(output);
}
async function resolveQueryLanguages(codeqlCmd: string, config: configUtils.Config): Promise<Map<string, string[]>> {
let res = new Map();
if (config.additionalQueries.length !== 0) {
let resolveQueriesOutput = '';
const options = {
listeners: {
stdout: (data: Buffer) => {
resolveQueriesOutput += data.toString();
}
if (!config.disableDefaultQueries || config.additionalSuites.length !== 0) {
const suites: string[] = [];
for (const language of await util.getLanguages()) {
if (!config.disableDefaultQueries) {
suites.push(language + '-code-scanning.qls');
}
};
for (const additionalSuite of config.additionalSuites) {
suites.push(language + '-' + additionalSuite + '.qls');
}
}
await exec.exec(
codeqlCmd, [
'resolve',
'queries',
...config.additionalQueries,
'--format=bylanguage'
],
options);
const resolveQueriesOutputObject = JSON.parse(resolveQueriesOutput);
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, suites);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
res[language] = Object.keys(<any>queries);
if (res[language] === undefined) {
res[language] = [];
}
res[language].push(...Object.keys(<any>queries));
}
}
if (config.additionalQueries.length !== 0) {
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, config.additionalQueries);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
if (res[language] === undefined) {
res[language] = [];
}
res[language].push(...Object.keys(<any>queries));
}
const noDeclaredLanguage = resolveQueriesOutputObject.noDeclaredLanguage;
@ -120,11 +162,17 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
for (let database of fs.readdirSync(databaseFolder)) {
core.startGroup('Analyzing ' + database);
const queries: string[] = [];
if (!config.disableDefaultQueries) {
queries.push(database + '-code-scanning.qls');
const queries = queriesPerLanguage[database] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + database + ' as no queries were selected for this language');
}
queries.push(...(queriesPerLanguage[database] || []));
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = path.join(databaseFolder, database + '-queries.qls');
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + database + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, database + '.sarif');
@ -136,7 +184,7 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
...queries
querySuite
]);
core.debug('SARIF results for database ' + database + ' created at "' + sarifFile + '"');

2
src/upload-lib.ts
View file

@ -136,7 +136,7 @@ async function uploadFiles(sarifFiles: string[]): Promise<boolean> {
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
const commitOid = util.getRequiredEnvParam('GITHUB_SHA');
const commitOid = await util.getCommitOid();
const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID');
const ref = util.getRef();
const analysisKey = await util.getAnalysisKey();

39
src/util.ts
View file

@ -1,4 +1,5 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as http from '@actions/http-client';
import * as auth from '@actions/http-client/auth';
import * as octokit from '@octokit/rest';
@ -25,13 +26,6 @@ export function should_abort(actionName: string, requireInitActionHasRun: boolea
return true;
}
// Should abort if called on a merge commit for a pull request.
if (ref.startsWith('refs/pull/')) {
core.warning('The CodeQL ' + actionName + ' action is intended for workflows triggered on `push` events, '
+ 'but the current workflow is running on a pull request. Aborting.');
return true;
}
// If the init action is required, then check the it completed successfully.
if (requireInitActionHasRun && process.env[sharedEnv.CODEQL_ACTION_INIT_COMPLETED] === undefined) {
core.setFailed('The CodeQL ' + actionName + ' action cannot be used unless the CodeQL init action is run first. Aborting.');
@ -152,6 +146,21 @@ export async function getLanguages(): Promise<string[]> {
return languages;
}
/**
* Gets the SHA of the commit that is currently checked out.
*/
export async function getCommitOid(): Promise<string> {
let commitOid = '';
await exec.exec('git', ['rev-parse', 'HEAD'], {
silent: true,
listeners: {
stdout: (data) => { commitOid += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
return commitOid.trim();
}
/**
* Get the path of the currently executing workflow.
*/
@ -204,8 +213,20 @@ export async function getAnalysisKey(): Promise<string> {
* Get the ref currently being analyzed.
*/
export function getRef(): string {
// it's in the form "refs/heads/master"
return getRequiredEnvParam('GITHUB_REF');
// Will be in the form "refs/heads/master" on a push event
// or in the form "refs/pull/N/merge" on a pull_request event
const ref = getRequiredEnvParam('GITHUB_REF');
// For pull request refs we want to convert from the 'merge' ref
// to the 'head' ref, as that is what we want to analyse.
// There should have been some code earlier in the workflow to do
// the checkout, but we have no way of verifying that here.
const pull_ref_regex = /refs\/pull\/(\d+)\/merge/;
if (pull_ref_regex.test(ref)) {
return ref.replace(pull_ref_regex, 'refs/pull/$1/head');
} else {
return ref;
}
}
interface StatusReport {