robojerk/codeql-action

Merge pull request #616 from github/update-v1.0.5-1c26d408

Browse source 
Merge main into v1
This commit is contained in:
Aditya Sharad 2021-07-12 16:03:11 -07:00 • committed by GitHub
commit 500dad96d7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 403 additions and 220 deletions
Download patch file Download diff file Expand all files Collapse all files

115
.github/workflows/pr-checks.yml vendored
View file

@ -7,6 +7,7 @@ on:
push:
branches: [main, v1]
pull_request:
workflow_dispatch:
jobs:
lint-js:
@ -119,9 +120,6 @@ jobs:
with:
config-file: ".github/codeql/codeql-config-packaging.yml"
languages: javascript
# TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
- name: Build code
shell: bash
run: ./build.sh
@ -164,8 +162,6 @@ jobs:
config-file: ".github/codeql/codeql-config-packaging2.yml"
languages: javascript
packs: dsp-testing/codeql-pack1@0.0.4, dsp-testing/codeql-pack2
# TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
- name: Build code
shell: bash
@ -209,8 +205,6 @@ jobs:
config-file: ".github/codeql/codeql-config-packaging3.yml"
packs: +dsp-testing/codeql-pack1@0.0.4
languages: javascript
# TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
- name: Build code
shell: bash
@ -234,6 +228,59 @@ jobs:
exit 1
fi
# Tests a split workflow where database construction and query execution happen in different steps
test-split-workflow:
needs: [check-js, check-node-modules]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Move codeql-action
shell: bash
run: |
mkdir ../action
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
mv ../action/.github/workflows .github
- uses: ./../action/init
with:
config-file: ".github/codeql/codeql-config-packaging3.yml"
packs: +dsp-testing/codeql-pack1@0.0.4
languages: javascript
tools: latest
- name: Build code
shell: bash
run: ./build.sh
- uses: ./../action/analyze
with:
skip-queries: true
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Assert No Results
run: |
if [ "$(ls -A $RUNNER_TEMP/results)" ]; then
echo "Expected results directory to be empty after skipping query execution!"
exit 1
fi
- uses: ./../action/analyze
with:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Assert Results
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."
exit 1
fi
# Identify the CodeQL tool versions to integration test against.
check-codeql-versions:
@ -241,6 +288,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
versions: ${{ steps.compare.outputs.versions }}
nightly-url: ${{ steps.get-url.outputs.nightly-url }}
steps:
- uses: actions/checkout@v2
@ -266,22 +314,45 @@ jobs:
with:
tools: latest
languages: javascript
- name: Compare default and latest CodeQL bundle versions
- name: Remove empty database
# allows us to run init a third time
run: |
rm -rf "$RUNNER_TEMP/codeql_databases"
- name: Get nightly release URL
id: get-url
env:
GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
shell: bash
# This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
run: |
export LATEST=`gh release list --repo dsp-testing/codeql-cli-nightlies -L 1 | cut -f 3`
echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST/codeql-bundle.tar.gz"
- name: Init with a CodeQL bundle from a nightly release
id: init-nightly
uses: ./../action/init
with:
tools: ${{ steps.get-url.outputs.nightly-url }}
languages: javascript
- name: Compare CodeQL bundle versions
id: compare
env:
CODEQL_DEFAULT: ${{ steps.init-default.outputs.codeql-path }}
CODEQL_LATEST: ${{ steps.init-latest.outputs.codeql-path }}
CODEQL_NIGHTLY: ${{ steps.init-nightly.outputs.codeql-path }}
NIGHTLY_URL: ${{ steps.get-url.outputs.nightly-url }}
run: |
CODEQL_VERSION_DEFAULT="$("$CODEQL_DEFAULT" version --format terse)"
CODEQL_VERSION_LATEST="$("$CODEQL_LATEST" version --format terse)"
CODEQL_VERSION_NIGHTLY="$("$CODEQL_NIGHTLY" version --format terse)"
echo "Default CodeQL bundle version is $CODEQL_VERSION_DEFAULT"
echo "Latest CodeQL bundle version is $CODEQL_VERSION_LATEST"
echo "Nightly CodeQL bundle version is $CODEQL_VERSION_NIGHTLY"
if [[ "$CODEQL_VERSION_DEFAULT" == "$CODEQL_VERSION_LATEST" ]]; then
# Just use `tools: null` to avoid duplication in the integration tests.
VERSIONS_JSON='[null]'
# Skip `tools: latest` since it would be the same as `tools: null`
VERSIONS_JSON="[null, \"$NIGHTLY_URL\"]"
else
# Use both `tools: null` and `tools: latest` in the integration tests.
VERSIONS_JSON='[null, "latest"]'
# Run integration tests with all three bundles.
VERSIONS_JSON="[null, \"$NIGHTLY_URL\", \"latest\"]"
fi
# Output a JSON-encoded list with the distinct versions to test against.
echo "Suggested matrix config for integration tests: $VERSIONS_JSON"
@ -803,3 +874,23 @@ jobs:
# Deliberately don't use TEST_MODE here. This is specifically testing
# the compatibility with the API.
runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
multi-language-repo_test-local-codeql:
needs: [check-js, check-node-modules, check-codeql-versions]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Move codeql-action
run: |
wget ${{ needs.check-codeql-versions.outputs.nightly-url }}
mkdir ../action
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
mv ../action/.github/workflows .github
- uses: ./../action/init
with:
tools: ../action/codeql-bundle.tar.gz
- name: Build code
run: ./build.sh
- uses: ./../action/analyze

5
CHANGELOG.md
View file

@ -1,5 +1,10 @@
# CodeQL Action and CodeQL Runner Changelog
## 1.0.5 - 12 Jul 2021
- The `analyze` step of the Action now supports a `skip-queries` option to merely build the CodeQL database without analyzing. This functionality is not present in the runner. Additionally, the step will no longer fail if it encounters a finalized database, and will instead continue with query execution. [#602](https://github.com/github/codeql-action/pull/602)
- Update the warning message when the baseline lines of code count is unavailable. [#608](https://github.com/github/codeql-action/pull/608)
## 1.0.4 - 28 Jun 2021
- Fix `RUNNER_TEMP environment variable must be set` when using runner. [#594](https://github.com/github/codeql-action/pull/594)

8
README.md
View file

@ -128,3 +128,11 @@ By default, this will override any queries specified in a config file. If you wi
## Troubleshooting
Read about [troubleshooting code scanning](https://help.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning).
### Note on "missing analysis" message
The very first time code scanning is run and if it is on a pull request, you will probably get a message mentioning a "missing analysis". This is expected.
After code scanning has analyzed the code in a pull request, it needs to compare the analysis of the topic branch (the merge commit of the branch you used to create the pull request) with the analysis of the base branch (the branch into which you want to merge the pull request). This allows code scanning to compute which alerts are newly introduced by the pull request, which alerts were already present in the base branch, and whether any existing alerts are fixed by the changes in the pull request. Initially, if you use a pull request to add code scanning to a repository, the base branch has not yet been analyzed, so it's not possible to compute these details. In this case, when you click through from the results check on the pull request you will see the "Missing analysis for base commit SHA-HASH" message.
For more information and other causes of this message, see [Reasons for the "missing analysis" message](https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/setting-up-code-scanning-for-a-repository#reasons-for-the-missing-analysis-message)

4
analyze/action.yml
View file

@ -24,6 +24,10 @@ inputs:
description: Specify whether or not to add code snippets to the output sarif file.
required: false
default: "false"
skip-queries:
description: If this option is set, the CodeQL database will be built but no queries will be run on it. Thus, no results will be produced.
required: false
default: "false"
threads:
description: The number of threads to be used by CodeQL.
required: false

32
lib/analyze-action.js generated
View file

@ -34,7 +34,8 @@ async function sendStatusReport(startedAt, stats, error) {
}
async function run() {
const startedAt = new Date();
let stats = undefined;
let uploadStats = undefined;
let runStats = undefined;
let config = undefined;
util.initializeEnvironment(util.Mode.actions, pkg.version);
try {
@ -51,7 +52,11 @@ async function run() {
url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
};
const outputDir = actionsUtil.getRequiredInput("output");
const queriesStats = await analyze_1.runAnalyze(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger), actionsUtil.getOptionalInput("category"), config, logger);
const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger);
await analyze_1.runFinalize(outputDir, threads, config, logger);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await analyze_1.runQueries(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await analyze_1.runCleanup(config, actionsUtil.getOptionalInput("cleanup-level") || "brutal", logger);
}
@ -60,13 +65,11 @@ async function run() {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
}
core.setOutput("db-locations", dbLocations);
if (actionsUtil.getRequiredInput("upload") === "true") {
const uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
stats = { ...queriesStats, ...uploadStats };
if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
}
else {
logger.info("Not uploading results");
stats = { ...queriesStats };
}
const repositoryNwo = repository_1.parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
await database_upload_1.uploadDatabases(repositoryNwo, config, apiDetails, logger);
@ -75,9 +78,12 @@ async function run() {
core.setFailed(error.message);
console.log(error);
if (error instanceof analyze_1.CodeQLAnalysisError) {
stats = { ...error.queriesStatusReport };
const stats = { ...error.queriesStatusReport };
await sendStatusReport(startedAt, stats, error);
}
else {
await sendStatusReport(startedAt, undefined, error);
}
await sendStatusReport(startedAt, stats, error);
return;
}
finally {
@ -103,7 +109,15 @@ async function run() {
}
}
}
await sendStatusReport(startedAt, stats);
if (runStats && uploadStats) {
await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
}
else if (runStats) {
await sendStatusReport(startedAt, { ...runStats });
}
else {
await sendStatusReport(startedAt, undefined);
}
}
async function runWrapper() {
try {

2
lib/analyze-action.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAKmB;AACnB,iDAAmD;AACnD,uDAAoD;AACpD,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;QAED,MAAM,aAAa,GAAG,+BAAkB,CACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;QACF,MAAM,iCAAe,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;KAClE;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAMmB;AACnB,iDAAmD;AACnD,uDAAoD;AACpD,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAE3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,WAAW,GAAmC,SAAS,CAAC;IAC5D,IAAI,QAAQ,GAAoC,SAAS,CAAC;IAC1D,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CACjC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EACvC,MAAM,CACP,CAAC;QACF,MAAM,qBAAW,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,KAAK,MAAM,EAAE;YAC3D,QAAQ,GAAG,MAAM,oBAAU,CACzB,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,OAAO,EACP,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;SACH;QAED,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,QAAQ,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACjE,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAC9C,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;SACH;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;SACtC;QAED,MAAM,aAAa,GAAG,+BAAkB,CACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;QACF,MAAM,iCAAe,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;KAClE;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,MAAM,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;YAC/C,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SACjD;aAAM;YACL,MAAM,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;SACrD;QAED,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,IAAI,QAAQ,IAAI,WAAW,EAAE;QAC3B,MAAM,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;KACpE;SAAM,IAAI,QAAQ,EAAE;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;KACpD;SAAM;QACL,MAAM,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;KAC9C;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

32
lib/analyze.js generated
View file

@ -10,6 +10,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const yaml = __importStar(require("js-yaml"));
const analysisPaths = __importStar(require("./analysis-paths"));
const codeql_1 = require("./codeql");
const count_loc_1 = require("./count-loc");
@ -55,7 +56,8 @@ async function createdDBForScannedLanguages(config, logger) {
analysisPaths.includeAndExcludeAnalysisPaths(config);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (languages_1.isScannedLanguage(language)) {
if (languages_1.isScannedLanguage(language) &&
!dbIsFinalized(config, language, logger)) {
logger.startGroup(`Extracting ${language}`);
if (language === languages_1.Language.python) {
await setupPythonExtractor(logger);
@ -65,13 +67,29 @@ async function createdDBForScannedLanguages(config, logger) {
}
}
}
function dbIsFinalized(config, language, logger) {
const dbPath = util.getCodeQLDatabasePath(config, language);
try {
const dbInfo = yaml.load(fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8"));
return !("inProgress" in dbInfo);
}
catch (e) {
logger.warning(`Could not check whether database for ${language} was finalized. Assuming it is not.`);
return false;
}
}
async function finalizeDatabaseCreation(config, threadsFlag, logger) {
await createdDBForScannedLanguages(config, logger);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config, language), threadsFlag);
logger.endGroup();
if (dbIsFinalized(config, language, logger)) {
logger.info(`There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`);
}
else {
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config, language), threadsFlag);
logger.endGroup();
}
}
}
// Runs queries and creates sarif files in the given folder
@ -183,15 +201,13 @@ function packWithVersionToQuerySuiteEntry(pack) {
}
return text;
}
async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
async function runFinalize(outputDir, threadsFlag, config, logger) {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
await finalizeDatabaseCreation(config, threadsFlag, logger);
const queriesStats = await runQueries(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger);
return { ...queriesStats };
}
exports.runAnalyze = runAnalyze;
exports.runFinalize = runFinalize;
async function runCleanup(config, cleanupLevel, logger) {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

84
lib/codeql.js generated
View file

@ -154,50 +154,58 @@ async function setupCodeQL(codeqlURL, apiDetails, tempDir, toolCacheDir, variant
if (forceLatest) {
codeqlURL = undefined;
}
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
// If we find the specified version, we always use that.
let codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
// If we don't find the requested version, in some cases we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (!codeqlFolder && !codeqlURL && !forceLatest) {
const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
if (codeqlVersions.length === 1) {
const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
codeqlFolder = tmpCodeqlFolder;
}
}
}
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
let codeqlFolder;
let codeqlURLVersion;
if (codeqlURL && !codeqlURL.startsWith("http")) {
codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
codeqlURLVersion = "local";
}
else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
// If we find the specified version, we always use that.
codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
// If we don't find the requested version, in some cases we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (!codeqlFolder && !codeqlURL && !forceLatest) {
const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
if (codeqlVersions.length === 1) {
const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
codeqlFolder = tmpCodeqlFolder;
}
}
}
const parsedCodeQLURL = new URL(codeqlURL);
const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
const headers = { accept: "application/octet-stream" };
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL.
if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
parsedQueryString["token"] === undefined) {
logger.debug("Downloading CodeQL bundle with token.");
headers.authorization = `token ${apiDetails.auth}`;
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
}
else {
logger.debug("Downloading CodeQL bundle without token.");
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
}
const parsedCodeQLURL = new URL(codeqlURL);
const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
const headers = { accept: "application/octet-stream" };
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL.
if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
parsedQueryString["token"] === undefined) {
logger.debug("Downloading CodeQL bundle with token.");
headers.authorization = `token ${apiDetails.auth}`;
}
else {
logger.debug("Downloading CodeQL bundle without token.");
}
logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
}
logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
}
let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
if (process.platform === "win32") {

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

15
lib/count-loc.js generated
View file

@ -77,14 +77,13 @@ async function countLoc(cwd, include, exclude, dbLanguages, logger) {
}
}
else {
logger.info("Could not determine the total number of lines of code in this repository. " +
"Because of this, it will not be possible to compare the number of lines " +
"of code analyzed by code scanning with the total number of lines of " +
"code in the repository. This will not affect the results produced by code " +
"scanning. If you have any questions, you can raise an issue at " +
"https://github.com/github/codeql-action/issues. Please include a link " +
"to the repository if public, or otherwise information about the code scanning " +
"workflow you are using.");
logger.info("Could not determine the baseline lines of code count in this repository. " +
"Because of this, it will not be possible to compare the lines " +
"of code analyzed by code scanning with the baseline. This will not affect " +
"the results produced by code scanning. If you have any questions, you can " +
"raise an issue at https://github.com/github/codeql-action/issues. Please " +
"include a link to the repository if public, or otherwise information about " +
"the code scanning workflow you are using.");
}
return lineCounts;
}

2
lib/count-loc.js.map
View file

@ -1 +1 @@
{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,4EAA4E;YAC1E,0EAA0E;YAC1E,sEAAsE;YACtE,4EAA4E;YAC5E,iEAAiE;YACjE,wEAAwE;YACxE,gFAAgF;YAChF,yBAAyB,CAC5B,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA/CD,4BA+CC"}
{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,2EAA2E;YACzE,gEAAgE;YAChE,4EAA4E;YAC5E,4EAA4E;YAC5E,2EAA2E;YAC3E,6EAA6E;YAC7E,2CAA2C,CAC9C,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA9CD,4BA8CC"}

2
lib/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20210622"
"bundleVersion": "codeql-bundle-20210702"
}

4
lib/runner.js generated
View file

@ -243,7 +243,9 @@ program
url: util_1.parseGitHubUrl(cmd.githubUrl),
};
const outputDir = cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
await analyze_1.runAnalyze(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), util_1.getThreadsFlag(cmd.threads, logger), cmd.category, config, logger);
const threads = util_1.getThreadsFlag(cmd.threads, logger);
await analyze_1.runFinalize(outputDir, threads, config, logger);
await analyze_1.runQueries(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), threads, cmd.category, config, logger);
if (!cmd.upload) {
logger.info("Not uploading results");
return;

2
lib/runner.js.map
View file

File diff suppressed because one or more lines are too long

2
package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "1.0.4",
"version": "1.0.5",
"lockfileVersion": 2,
"requires": true,
"packages": {

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "1.0.4",
"version": "1.0.5",
"private": true,
"description": "CodeQL action",
"scripts": {

2
runner/package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "1.0.4",
"version": "1.0.5",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
runner/package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "1.0.4",
"version": "1.0.5",
"private": true,
"description": "CodeQL runner",
"scripts": {

48
src/analyze-action.ts
View file

@ -5,16 +5,18 @@ import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import {
runAnalyze,
CodeQLAnalysisError,
QueriesStatusReport,
runCleanup,
runQueries,
runFinalize,
} from "./analyze";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
import { UploadStatusReport } from "./upload-lib";
import * as util from "./util";
// eslint-disable-next-line import/no-commonjs
@ -53,7 +55,8 @@ async function sendStatusReport(
async function run() {
const startedAt = new Date();
let stats: AnalysisStatusReport | undefined = undefined;
let uploadStats: UploadStatusReport | undefined = undefined;
let runStats: QueriesStatusReport | undefined = undefined;
let config: Config | undefined = undefined;
util.initializeEnvironment(util.Mode.actions, pkg.version);
@ -82,15 +85,22 @@ async function run() {
url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
};
const outputDir = actionsUtil.getRequiredInput("output");
const queriesStats = await runAnalyze(
outputDir,
util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger),
actionsUtil.getOptionalInput("category"),
config,
const threads = util.getThreadsFlag(
actionsUtil.getOptionalInput("threads"),
logger
);
await runFinalize(outputDir, threads, config, logger);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await runQueries(
outputDir,
util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
threads,
actionsUtil.getOptionalInput("category"),
config,
logger
);
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await runCleanup(
@ -106,17 +116,15 @@ async function run() {
}
core.setOutput("db-locations", dbLocations);
if (actionsUtil.getRequiredInput("upload") === "true") {
const uploadStats = await upload_lib.uploadFromActions(
if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
uploadStats = await upload_lib.uploadFromActions(
outputDir,
config.gitHubVersion,
apiDetails,
logger
);
stats = { ...queriesStats, ...uploadStats };
} else {
logger.info("Not uploading results");
stats = { ...queriesStats };
}
const repositoryNwo = parseRepositoryNwo(
@ -128,10 +136,12 @@ async function run() {
console.log(error);
if (error instanceof CodeQLAnalysisError) {
stats = { ...error.queriesStatusReport };
const stats = { ...error.queriesStatusReport };
await sendStatusReport(startedAt, stats, error);
} else {
await sendStatusReport(startedAt, undefined, error);
}
await sendStatusReport(startedAt, stats, error);
return;
} finally {
if (core.isDebug() && config !== undefined) {
@ -161,7 +171,13 @@ async function run() {
}
}
await sendStatusReport(startedAt, stats);
if (runStats && uploadStats) {
await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
} else if (runStats) {
await sendStatusReport(startedAt, { ...runStats });
} else {
await sendStatusReport(startedAt, undefined);
}
}
async function runWrapper() {

62
src/analyze.ts
View file

@ -2,6 +2,7 @@ import * as fs from "fs";
import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as yaml from "js-yaml";
import * as analysisPaths from "./analysis-paths";
import { getCodeQL } from "./codeql";
@ -117,7 +118,10 @@ async function createdDBForScannedLanguages(
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
if (
isScannedLanguage(language) &&
!dbIsFinalized(config, language, logger)
) {
logger.startGroup(`Extracting ${language}`);
if (language === Language.python) {
@ -133,6 +137,25 @@ async function createdDBForScannedLanguages(
}
}
function dbIsFinalized(
config: configUtils.Config,
language: Language,
logger: Logger
) {
const dbPath = util.getCodeQLDatabasePath(config, language);
try {
const dbInfo = yaml.load(
fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8")
);
return !("inProgress" in dbInfo);
} catch (e) {
logger.warning(
`Could not check whether database for ${language} was finalized. Assuming it is not.`
);
return false;
}
}
async function finalizeDatabaseCreation(
config: configUtils.Config,
threadsFlag: string,
@ -142,12 +165,18 @@ async function finalizeDatabaseCreation(
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(
util.getCodeQLDatabasePath(config, language),
threadsFlag
);
logger.endGroup();
if (dbIsFinalized(config, language, logger)) {
logger.info(
`There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`
);
} else {
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(
util.getCodeQLDatabasePath(config, language),
threadsFlag
);
logger.endGroup();
}
}
}
@ -349,33 +378,18 @@ function packWithVersionToQuerySuiteEntry(
return text;
}
export async function runAnalyze(
export async function runFinalize(
outputDir: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string,
automationDetailsId: string | undefined,
config: configUtils.Config,
logger: Logger
): Promise<QueriesStatusReport> {
) {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
await finalizeDatabaseCreation(config, threadsFlag, logger);
const queriesStats = await runQueries(
outputDir,
memoryFlag,
addSnippetsFlag,
threadsFlag,
automationDetailsId,
config,
logger
);
return { ...queriesStats };
}
export async function runCleanup(

169
src/codeql.ts
View file

@ -318,96 +318,101 @@ export async function setupCodeQL(
if (forceLatest) {
codeqlURL = undefined;
}
const codeqlURLVersion = getCodeQLURLVersion(
codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
);
const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
// If we find the specified version, we always use that.
let codeqlFolder = toolcache.find(
"CodeQL",
codeqlURLSemVer,
toolCacheDir,
logger
);
// If we don't find the requested version, in some cases we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (!codeqlFolder && !codeqlURL && !forceLatest) {
const codeqlVersions = toolcache.findAllVersions(
"CodeQL",
toolCacheDir,
logger
);
if (codeqlVersions.length === 1) {
const tmpCodeqlFolder = toolcache.find(
"CodeQL",
codeqlVersions[0],
toolCacheDir,
logger
);
if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
logger.debug(
`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
);
codeqlFolder = tmpCodeqlFolder;
}
}
}
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
let codeqlFolder: string;
let codeqlURLVersion: string;
if (codeqlURL && !codeqlURL.startsWith("http")) {
codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
codeqlURLVersion = "local";
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL(
apiDetails,
variant,
logger
);
}
codeqlURLVersion = getCodeQLURLVersion(
codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
);
const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
const parsedCodeQLURL = new URL(codeqlURL);
const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
const headers: IHeaders = { accept: "application/octet-stream" };
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL.
if (
codeqlURL.startsWith(`${apiDetails.url}/`) &&
parsedQueryString["token"] === undefined
) {
logger.debug("Downloading CodeQL bundle with token.");
headers.authorization = `token ${apiDetails.auth}`;
} else {
logger.debug("Downloading CodeQL bundle without token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const codeqlPath = await toolcache.downloadTool(
codeqlURL,
tempDir,
headers
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(
codeqlPath,
tempDir,
logger
);
codeqlFolder = await toolcache.cacheDir(
codeqlExtracted,
// If we find the specified version, we always use that.
codeqlFolder = toolcache.find(
"CodeQL",
codeqlURLSemVer,
toolCacheDir,
logger
);
}
// If we don't find the requested version, in some cases we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (!codeqlFolder && !codeqlURL && !forceLatest) {
const codeqlVersions = toolcache.findAllVersions(
"CodeQL",
toolCacheDir,
logger
);
if (codeqlVersions.length === 1) {
const tmpCodeqlFolder = toolcache.find(
"CodeQL",
codeqlVersions[0],
toolCacheDir,
logger
);
if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
logger.debug(
`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
);
codeqlFolder = tmpCodeqlFolder;
}
}
}
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL(
apiDetails,
variant,
logger
);
}
const parsedCodeQLURL = new URL(codeqlURL);
const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
const headers: IHeaders = { accept: "application/octet-stream" };
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL.
if (
codeqlURL.startsWith(`${apiDetails.url}/`) &&
parsedQueryString["token"] === undefined
) {
logger.debug("Downloading CodeQL bundle with token.");
headers.authorization = `token ${apiDetails.auth}`;
} else {
logger.debug("Downloading CodeQL bundle without token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const codeqlPath = await toolcache.downloadTool(
codeqlURL,
tempDir,
headers
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(
codeqlPath,
tempDir,
logger
);
codeqlFolder = await toolcache.cacheDir(
codeqlExtracted,
"CodeQL",
codeqlURLSemVer,
toolCacheDir,
logger
);
}
}
let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
if (process.platform === "win32") {
codeqlCmd += ".exe";

15
src/count-loc.ts
View file

@ -98,14 +98,13 @@ export async function countLoc(
}
} else {
logger.info(
"Could not determine the total number of lines of code in this repository. " +
"Because of this, it will not be possible to compare the number of lines " +
"of code analyzed by code scanning with the total number of lines of " +
"code in the repository. This will not affect the results produced by code " +
"scanning. If you have any questions, you can raise an issue at " +
"https://github.com/github/codeql-action/issues. Please include a link " +
"to the repository if public, or otherwise information about the code scanning " +
"workflow you are using."
"Could not determine the baseline lines of code count in this repository. " +
"Because of this, it will not be possible to compare the lines " +
"of code analyzed by code scanning with the baseline. This will not affect " +
"the results produced by code scanning. If you have any questions, you can " +
"raise an issue at https://github.com/github/codeql-action/issues. Please " +
"include a link to the repository if public, or otherwise information about " +
"the code scanning workflow you are using."
);
}

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20210622"
"bundleVersion": "codeql-bundle-20210702"
}

8
src/runner.ts
View file

@ -4,7 +4,7 @@ import * as path from "path";
import { Command } from "commander";
import { runAnalyze } from "./analyze";
import { runFinalize, runQueries } from "./analyze";
import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
import { CodeQL, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
@ -431,11 +431,13 @@ program
const outputDir =
cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
await runAnalyze(
const threads = getThreadsFlag(cmd.threads, logger);
await runFinalize(outputDir, threads, config, logger);
await runQueries(
outputDir,
getMemoryFlag(cmd.ram),
getAddSnippetsFlag(cmd.addSnippets),
getThreadsFlag(cmd.threads, logger),
threads,
cmd.category,
config,
logger