robojerk/codeql-action

Merge remote-tracking branch 'origin/main' into henrymercer/run-atm-on-windows

Browse source
This commit is contained in:
Henry Mercer 2022-05-11 19:32:14 +01:00
commit 533ce91971
59 changed files with 630 additions and 465 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/update-release-branch.py vendored
View file

@ -292,7 +292,7 @@ def main():
conflicted_files = run_git('diff', '--name-only', '--diff-filter', 'U').splitlines()
if len(conflicted_files) > 0:
run_git('add', '.')
run_git('commit', '--no-edit')
run_git('commit', '--no-edit')
# Migrate the package version number from a v2 version number to a v1 version number
print(f'Setting version number to {version}')

30
.github/workflows/__packaging-config-inputs-js.yml generated vendored
View file

@ -26,9 +26,27 @@ jobs:
matrix:
include:
- os: ubuntu-latest
version: nightly-20210831
version: latest
- os: macos-latest
version: nightly-20210831
version: latest
- os: windows-2019
version: latest
- os: windows-2022
version: latest
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-2019
version: cached
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
- os: windows-2019
version: nightly-latest
- os: windows-2022
version: nightly-latest
name: 'Packaging: Config and input'
timeout-minutes: 45
runs-on: ${{ matrix.os }}
@ -43,7 +61,7 @@ jobs:
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml
packs: +dsp-testing/codeql-pack1@0.1.0
packs: +dsp-testing/codeql-pack1@1.0.0
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
@ -58,11 +76,11 @@ jobs:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

28
.github/workflows/__packaging-config-js.yml generated vendored
View file

@ -26,9 +26,27 @@ jobs:
matrix:
include:
- os: ubuntu-latest
version: nightly-20210831
version: latest
- os: macos-latest
version: nightly-20210831
version: latest
- os: windows-2019
version: latest
- os: windows-2022
version: latest
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-2019
version: cached
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
- os: windows-2019
version: nightly-latest
- os: windows-2022
version: nightly-latest
name: 'Packaging: Config file'
timeout-minutes: 45
runs-on: ${{ matrix.os }}
@ -57,11 +75,11 @@ jobs:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

30
.github/workflows/__packaging-inputs-js.yml generated vendored
View file

@ -26,9 +26,27 @@ jobs:
matrix:
include:
- os: ubuntu-latest
version: nightly-20210831
version: latest
- os: macos-latest
version: nightly-20210831
version: latest
- os: windows-2019
version: latest
- os: windows-2022
version: latest
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-2019
version: cached
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
- os: windows-2019
version: nightly-latest
- os: windows-2022
version: nightly-latest
name: 'Packaging: Action input'
timeout-minutes: 45
runs-on: ${{ matrix.os }}
@ -44,7 +62,7 @@ jobs:
with:
config-file: .github/codeql/codeql-config-packaging2.yml
languages: javascript
packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
packs: dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2, dsp-testing/codeql-pack3:other-query.ql
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
shell: bash
@ -58,11 +76,11 @@ jobs:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

20
.github/workflows/__split-workflow.yml generated vendored
View file

@ -26,9 +26,17 @@ jobs:
matrix:
include:
- os: ubuntu-latest
version: nightly-20210831
version: latest
- os: macos-latest
version: nightly-20210831
version: latest
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
name: Split workflow
timeout-minutes: 45
runs-on: ${{ matrix.os }}
@ -43,7 +51,7 @@ jobs:
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml
packs: +dsp-testing/codeql-pack1@0.1.0
packs: +dsp-testing/codeql-pack1@1.0.0
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
@ -72,11 +80,11 @@ jobs:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

35
.github/workflows/script/update-required-checks.sh vendored Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Update the required checks based on the current branch.
# Typically, this will be main.
if [ -z "$GITHUB_TOKEN" ]; then
echo "Failed: No GitHub token found. This script requires admin access to `github/codeql-action`."
exit 1
fi
if [ "$#" -eq 1 ]; then
# If we were passed an argument, pass it as a query to fzf
GITHUB_SHA="$@"
elif [ "$#" -gt 1 ]; then
echo "Usage: $0 [SHA]"
echo "Update the required checks based on the SHA, or main."
elif [ -z "$GITHUB_SHA" ]; then
# If we don't have a SHA, use main
GITHUB_SHA="$(git rev-parse main)"
fi
echo "Getting checks for $GITHUB_SHA"
# Ignore any checks with "https://", CodeQL, LGTM, and Update checks.
CHECKS="$(gh api repos/github/codeql-action/commits/${GITHUB_SHA}/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs | .[].name | select(contains("https://") or . == "CodeQL" or . == "LGTM.com" or contains("Update") | not)] | sort')"
echo "$CHECKS" | jq
echo "{\"contexts\": ${CHECKS}}" > checks.json
for BRANCH in main releases/v2 releases/v1; do
echo "Updating $BRANCH"
gh api --silent -X "PATCH" "repos/github/codeql-action/branches/$BRANCH/protection/required_status_checks" --input checks.json
done
rm checks.json

5
CHANGELOG.md
View file

@ -4,6 +4,11 @@
No user facing changes.
## 2.1.10 - 10 May 2022
- Update default CodeQL bundle version to 2.9.5. [#1056](https://github.com/github/codeql-action/pull/1056)
- When `wait-for-processing` is enabled, the workflow will now fail if there were any errors that occurred during processing of the analysis results.
## 2.1.9 - 27 Apr 2022
- Add `working-directory` input to the `autobuild` action. [#1024](https://github.com/github/codeql-action/pull/1024)

19
CONTRIBUTING.md
View file

@ -80,22 +80,11 @@ Here are a few things you can do that will increase the likelihood of your pull
## Keeping the PR checks up to date (admin access required)
Since the `codeql-action` runs most of its testing through individual Actions workflows, there are over two hundred jobs that need to pass in order for a PR to turn green. Managing these PR checks manually is time consuming and complex. Here is a semi-automated approach.
Since the `codeql-action` runs most of its testing through individual Actions workflows, there are over two hundred jobs that need to pass in order for a PR to turn green. You can regenerate the checks automatically by running the [update-required-checks.sh](.github/workflows/script/update-required-checks.sh) script:
To regenerate the PR jobs for the action:
1. From a terminal, run the following commands (replace `SHA` with the sha of the commit whose checks you want to use, typically this should be the latest from `main`):
```sh
SHA= ####
CHECKS="$(gh api repos/github/codeql-action/commits/${SHA}/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs | .[].name | select(contains("https://") or . == "CodeQL" or . == "LGTM.com" or . == "Update dependencies" or . == "Update Supported Enterprise Server Versions" | not)]')"
echo "{\"contexts\": ${CHECKS}}" > checks.json
gh api -X "PATCH" repos/github/codeql-action/branches/main/protection/required_status_checks --input checks.json
gh api -X "PATCH" repos/github/codeql-action/branches/releases/v2/protection/required_status_checks --input checks.json
gh api -X "PATCH" repos/github/codeql-action/branches/releases/v1/protection/required_status_checks --input checks.json
````
2. Go to the [branch protection rules settings page](https://github.com/github/codeql-action/settings/branches) and validate that the rules have been updated.
1. By default, this script retrieves the checks from the latest SHA on `main`, so make sure that your `main` branch is up to date.
2. Run the script. If there's a reason to, you can pass in a different SHA as a CLI argument.
3. After running, go to the [branch protection rules settings page](https://github.com/github/codeql-action/settings/branches) and validate that the rules for `main`, `v1`, and `v2` have been updated.
## Resources

3
lib/actions-util.js generated
View file

@ -584,8 +584,7 @@ async function sendStatusReport(statusReport) {
const statusReportJSON = JSON.stringify(statusReport);
core.debug(`Sending status report: ${statusReportJSON}`);
// If in test mode we don't want to upload the results
const testMode = process.env["TEST_MODE"] === "true" || false;
if (testMode) {
if ((0, util_1.isInTestMode)()) {
core.debug("In test mode. Status reports are not uploaded.");
return true;
}

2
lib/actions-util.js.map
View file

File diff suppressed because one or more lines are too long

10
lib/analyze-action-env.test.js generated
View file

@ -38,14 +38,17 @@ const util = __importStar(require("./util"));
// but the first test would fail.
(0, ava_1.default)("analyze action with RAM & threads from environment variables", async (t) => {
await util.withTmpDir(async (tmpDir) => {
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
process.env["GITHUB_REPOSITORY"] = "fake/repository";
process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
sinon
.stub(actionsUtil, "createStatusReportBase")
.resolves({});
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
const gitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
gitHubVersion,
languages: [],
packs: [],
});
@ -54,6 +57,7 @@ const util = __importStar(require("./util"));
requiredInputStub.withArgs("upload-database").returns("false");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
(0, testing_utils_1.mockFeatureFlagApiEndpoint)(200, {});
// When there are no action inputs for RAM and threads, the action uses

2
lib/analyze-action-env.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YAClD,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,sCAAsC,CAAC;QAC1E,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAuB;YACxC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;SAChC,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa;YACb,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC7D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

10
lib/analyze-action-input.test.js generated
View file

@ -38,14 +38,17 @@ const util = __importStar(require("./util"));
// but the first test would fail.
(0, ava_1.default)("analyze action with RAM & threads from action inputs", async (t) => {
await util.withTmpDir(async (tmpDir) => {
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
process.env["GITHUB_REPOSITORY"] = "fake/repository";
process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
sinon
.stub(actionsUtil, "createStatusReportBase")
.resolves({});
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
const gitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
gitHubVersion,
languages: [],
packs: [],
});
@ -54,6 +57,7 @@ const util = __importStar(require("./util"));
requiredInputStub.withArgs("upload-database").returns("false");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
(0, testing_utils_1.mockFeatureFlagApiEndpoint)(200, {});
process.env["CODEQL_THREADS"] = "1";

2
lib/analyze-action-input.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YAClD,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,sCAAsC,CAAC;QAC1E,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAuB;YACxC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;SAChC,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa;YACb,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC7D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

6
lib/analyze-action.js generated
View file

@ -118,7 +118,11 @@ async function run() {
}
// Possibly upload the database bundles for remote queries
await (0, database_upload_1.uploadDatabases)(repositoryNwo, config, apiDetails, logger);
if (uploadResult !== undefined &&
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
core.debug("In test mode. Waiting for processing is disabled.");
}
else if (uploadResult !== undefined &&
actionsUtil.getRequiredInput("wait-for-processing") === "true") {
await upload_lib.waitForProcessing((0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY")), uploadResult.sarifID, apiDetails, (0, logging_1.getActionsLogger)());
}

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

24
lib/analyze.js generated
View file

@ -159,7 +159,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
}
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), undefined));
querySuitePaths.push(...(await runQueryPacks(language, "packs", packsWithVersion, undefined)));
ranCustom = true;
}
if (ranCustom) {
@ -217,21 +217,23 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
async function runQueryPacks(language, type, packs, searchPath) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Run the queries individually instead of all at once to avoid command
// line length restrictions, particularly on windows.
for (const pack of packs) {
logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
await codeql.databaseRunQueries(databasePath, searchPath, pack, memoryFlag, threadsFlag);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
}
return packs;
}
}
exports.runQueries = runQueries;
function createQuerySuiteContents(queries) {
return queries.map((q) => `- query: ${q}`).join("\n");
}
function createPackSuiteContents(packsWithVersion) {
return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
}
function packWithVersionToQuerySuiteEntry(pack) {
let text = `- qlpack: ${pack.packName}`;
if (pack.version) {
text += `\n version: ${pack.version}`;
}
return text;
}
async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
if (await util.codeQlVersionAbove(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

37
lib/analyze.test.js generated
View file

@ -26,7 +26,6 @@ const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const ava_1 = __importDefault(require("ava"));
const yaml = __importStar(require("js-yaml"));
const semver_1 = require("semver");
const sinon = __importStar(require("sinon"));
const analyze_1 = require("./analyze");
const codeql_1 = require("./codeql");
@ -53,18 +52,8 @@ const util = __importStar(require("./util"));
const addSnippetsFlag = "";
const threadsFlag = "";
const packs = {
[languages_1.Language.cpp]: [
{
packName: "a/b",
version: (0, semver_1.clean)("1.0.0"),
},
],
[languages_1.Language.java]: [
{
packName: "c/d",
version: (0, semver_1.clean)("2.0.0"),
},
],
[languages_1.Language.cpp]: ["a/b@1.0.0"],
[languages_1.Language.java]: ["c/d@2.0.0"],
};
for (const language of Object.values(languages_1.Language)) {
(0, codeql_1.setCodeQL)({
@ -209,32 +198,10 @@ const util = __importStar(require("./util"));
query: "bar.ql",
},
];
const qlsPackContentCpp = [
{
qlpack: "a/b",
version: "1.0.0",
},
];
const qlsPackContentJava = [
{
qlpack: "c/d",
version: "2.0.0",
},
];
for (const lang of Object.values(languages_1.Language)) {
t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
const packSuiteName = `${lang}-queries-packs.qls`;
if (lang === languages_1.Language.cpp) {
t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
}
else if (lang === languages_1.Language.java) {
t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
}
else {
t.false(fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName)));
}
}
function readContents(name) {
const x = fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8");

2
lib/analyze.test.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/codeql.js generated
View file

@ -647,8 +647,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
"pack",
"download",
"--format=json",
"--resolve-query-specs",
...getExtraOptionsFromEnv(["pack", "download"]),
...packs.map(packWithVersionToString),
...packs,
];
const output = await runTool(cmd, codeqlArgs);
try {
@ -704,9 +705,6 @@ async function getCodeQLForCmd(cmd, checkVersion) {
}
return codeql;
}
function packWithVersionToString(pack) {
return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*/

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

79
lib/config-utils.js generated
View file

@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPacksRequireLanguage = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePacksSpecification = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPacksRequireLanguage = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const yaml = __importStar(require("js-yaml"));
@ -136,7 +136,7 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS))) &&
languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") &&
!((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK_NAME)) &&
!((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some(isMlPoweredJsQueriesPack)) &&
(await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) &&
(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) {
if (!packs.javascript) {
@ -149,6 +149,11 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
await runResolveQueries(codeQL, resultMap, suites, undefined);
return injectedMlQueries;
}
function isMlPoweredJsQueriesPack(pack) {
return (pack === util_1.ML_POWERED_JS_QUERIES_PACK_NAME ||
pack.startsWith(`${util_1.ML_POWERED_JS_QUERIES_PACK_NAME}@`) ||
pack.startsWith(`${util_1.ML_POWERED_JS_QUERIES_PACK_NAME}:`));
}
/**
* Retrieve the set of queries at localQueryPath and add them to resultMap.
*/
@ -635,7 +640,7 @@ function parsePacksFromConfig(packsByLanguage, languages, configFile) {
}
packs[lang] = [];
for (const packStr of packsArr) {
packs[lang].push(toPackWithVersion(packStr, configFile));
packs[lang].push(validatePacksSpecification(packStr, configFile));
}
}
return packs;
@ -660,32 +665,74 @@ function parsePacksFromInput(packsInput, languages) {
}
return {
[languages[0]]: packsInput.split(",").reduce((packs, pack) => {
packs.push(toPackWithVersion(pack, ""));
packs.push(validatePacksSpecification(pack, ""));
return packs;
}, []),
};
}
function toPackWithVersion(packStr, configFile) {
/**
* Validates that this package specification is syntactically correct.
* It may not point to any real package, but after this function returns
* without throwing, we are guaranteed that the package specification
* is roughly correct.
*
* The CLI itself will do a more thorough validation of the package
* specification.
*
* A package specification looks like this:
*
* `scope/name@version:path`
*
* Version and path are optional.
*
* @param packStr the package specification to verify.
* @param configFile Config file to use for error reporting
*/
function validatePacksSpecification(packStr, configFile) {
if (typeof packStr !== "string") {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
const nameWithVersion = packStr.trim().split("@");
let version;
if (nameWithVersion.length > 2 ||
!PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])) {
packStr = packStr.trim();
const atIndex = packStr.indexOf("@");
const colonIndex = packStr.indexOf(":", atIndex);
const packStart = 0;
const versionStart = atIndex + 1 || undefined;
const pathStart = colonIndex + 1 || undefined;
const packEnd = Math.min(atIndex > 0 ? atIndex : Infinity, colonIndex > 0 ? colonIndex : Infinity, packStr.length);
const versionEnd = versionStart
? Math.min(colonIndex > 0 ? colonIndex : Infinity, packStr.length)
: undefined;
const pathEnd = pathStart ? packStr.length : undefined;
const packName = packStr.slice(packStart, packEnd).trim();
const version = versionStart
? packStr.slice(versionStart, versionEnd).trim()
: undefined;
const packPath = pathStart
? packStr.slice(pathStart, pathEnd).trim()
: undefined;
if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
else if (nameWithVersion.length === 2) {
version = semver.clean(nameWithVersion[1]) || undefined;
if (!version) {
if (version) {
try {
new semver.Range(version);
}
catch (e) {
// The range string is invalid. OK to ignore the caught error
throw new Error(getPacksStrInvalid(packStr, configFile));
}
}
return {
packName: nameWithVersion[0].trim(),
version,
};
if (packPath &&
(path.isAbsolute(packPath) || path.normalize(packPath) !== packPath)) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
if (!packPath && pathStart) {
// 0 length path
throw new Error(getPacksStrInvalid(packStr, configFile));
}
return (packName + (version ? `@${version}` : "") + (packPath ? `:${packPath}` : ""));
}
exports.validatePacksSpecification = validatePacksSpecification;
// exported for testing
function parsePacks(rawPacksFromConfig, rawPacksInput, languages, configFile) {
const packsFromInput = parsePacksFromInput(rawPacksInput, languages);

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

112
lib/config-utils.test.js generated
View file

@ -26,7 +26,6 @@ const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const github = __importStar(require("@actions/github"));
const ava_1 = __importDefault(require("ava"));
const semver_1 = require("semver");
const sinon = __importStar(require("sinon"));
const api = __importStar(require("./api-client"));
const codeql_1 = require("./codeql");
@ -601,12 +600,7 @@ function queriesToResolvedQueryForm(queries) {
const languages = "javascript";
const { packs } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
t.deepEqual(packs, {
[languages_1.Language.javascript]: [
{
packName: "a/b",
version: (0, semver_1.clean)("1.2.3"),
},
],
[languages_1.Language.javascript]: ["a/b@1.2.3"],
});
});
});
@ -640,18 +634,8 @@ function queriesToResolvedQueryForm(queries) {
const languages = "javascript,python,cpp";
const { packs, queries } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example" }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
t.deepEqual(packs, {
[languages_1.Language.javascript]: [
{
packName: "a/b",
version: (0, semver_1.clean)("1.2.3"),
},
],
[languages_1.Language.python]: [
{
packName: "c/d",
version: (0, semver_1.clean)("1.2.3"),
},
],
[languages_1.Language.javascript]: ["a/b@1.2.3"],
[languages_1.Language.python]: ["c/d@1.2.3"],
});
t.deepEqual(queries, {
cpp: {
@ -786,28 +770,47 @@ const invalidPackNameMacro = ava_1.default.macro({
});
(0, ava_1.default)("no packs", parsePacksMacro, {}, [], {});
(0, ava_1.default)("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: (0, semver_1.clean)("1.2.3") },
],
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
});
(0, ava_1.default)("two packs with spaces", parsePacksMacro, [" a/b ", " c/d@1.2.3 "], [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: (0, semver_1.clean)("1.2.3") },
],
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
});
(0, ava_1.default)("two packs with language", parsePacksMacro, {
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
[languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
}, [languages_1.Language.cpp, languages_1.Language.java, languages_1.Language.csharp], {
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
[languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
});
(0, ava_1.default)("packs with other valid names", parsePacksMacro, [
// ranges are ok
"c/d@1.0",
"c/d@~1.0.0",
"c/d@~1.0.0:a/b",
"c/d@~1.0.0+abc:a/b",
"c/d@~1.0.0-abc:a/b",
"c/d:a/b",
// whitespace is removed
" c/d @ ~1.0.0 : b.qls ",
// and it is retained within a path
" c/d @ ~1.0.0 : b/a path with/spaces.qls ",
// this is valid. the path is '@'. It will probably fail when passed to the CLI
"c/d@1.2.3:@",
// this is valid, too. It will fail if it doesn't match a path
// (globbing is not done)
"c/d@1.2.3:+*)_(",
], [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: (0, semver_1.clean)("1.2.3") },
],
[languages_1.Language.java]: [
{ packName: "d/e", version: undefined },
{ packName: "f/g", version: (0, semver_1.clean)("1.2.3") },
"c/d@1.0",
"c/d@~1.0.0",
"c/d@~1.0.0:a/b",
"c/d@~1.0.0+abc:a/b",
"c/d@~1.0.0-abc:a/b",
"c/d:a/b",
"c/d@~1.0.0:b.qls",
"c/d@~1.0.0:b/a path with/spaces.qls",
"c/d@1.2.3:@",
"c/d@1.2.3:+*)_(",
],
});
(0, ava_1.default)("no language", parsePacksErrorMacro, ["a/b@1.2.3"], [languages_1.Language.java, languages_1.Language.python], /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/);
@ -817,7 +820,14 @@ const invalidPackNameMacro = ava_1.default.macro({
(0, ava_1.default)(invalidPackNameMacro, "c-/d");
(0, ava_1.default)(invalidPackNameMacro, "-c/d");
(0, ava_1.default)(invalidPackNameMacro, "c/d_d");
(0, ava_1.default)(invalidPackNameMacro, "c/d@x");
(0, ava_1.default)(invalidPackNameMacro, "c/d@@");
(0, ava_1.default)(invalidPackNameMacro, "c/d@1.0.0:");
(0, ava_1.default)(invalidPackNameMacro, "c/d:");
(0, ava_1.default)(invalidPackNameMacro, "c/d:/a");
(0, ava_1.default)(invalidPackNameMacro, "@1.0.0:a");
(0, ava_1.default)(invalidPackNameMacro, "c/d@../a");
(0, ava_1.default)(invalidPackNameMacro, "c/d@b/../a");
(0, ava_1.default)(invalidPackNameMacro, "c/d:z@1");
/**
* Test macro for testing the packs block and the packs input
*/
@ -834,39 +844,22 @@ function parseInputAndConfigErrorMacro(t, packsFromConfig, packsFromInput, langu
}
parseInputAndConfigErrorMacro.title = (providedTitle) => `Parse Packs input and config Error: ${providedTitle}`;
(0, ava_1.default)("input only", parseInputAndConfigMacro, {}, " c/d ", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [{ packName: "c/d", version: undefined }],
[languages_1.Language.cpp]: ["c/d"],
});
(0, ava_1.default)("input only with multiple", parseInputAndConfigMacro, {}, "a/b , c/d@1.2.3", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: "1.2.3" },
],
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
});
(0, ava_1.default)("input only with +", parseInputAndConfigMacro, {}, " + a/b , c/d@1.2.3 ", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: "1.2.3" },
],
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
});
(0, ava_1.default)("config only", parseInputAndConfigMacro, ["a/b", "c/d"], " ", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: undefined },
],
[languages_1.Language.cpp]: ["a/b", "c/d"],
});
(0, ava_1.default)("input overrides", parseInputAndConfigMacro, ["a/b", "c/d"], " e/f, g/h@1.2.3 ", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "e/f", version: undefined },
{ packName: "g/h", version: "1.2.3" },
],
[languages_1.Language.cpp]: ["e/f", "g/h@1.2.3"],
});
(0, ava_1.default)("input and config", parseInputAndConfigMacro, ["a/b", "c/d"], " +e/f, g/h@1.2.3 ", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "e/f", version: undefined },
{ packName: "g/h", version: "1.2.3" },
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: undefined },
],
[languages_1.Language.cpp]: ["e/f", "g/h@1.2.3", "a/b", "c/d"],
});
(0, ava_1.default)("input with no language", parseInputAndConfigErrorMacro, {}, "c/d", [], /No languages specified/);
(0, ava_1.default)("input with two languages", parseInputAndConfigErrorMacro, {}, "c/d", [languages_1.Language.cpp, languages_1.Language.csharp], /multi-language analysis/);
@ -895,10 +888,7 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
if (expectedVersionString !== undefined) {
t.deepEqual(packs, {
[languages_1.Language.javascript]: [
{
packName: "codeql/javascript-experimental-atm-queries",
version: expectedVersionString,
},
`codeql/javascript-experimental-atm-queries@${expectedVersionString}`,
],
});
}

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20220421"
"bundleVersion": "codeql-bundle-20220428"
}

29
lib/upload-lib.js generated
View file

@ -93,8 +93,7 @@ function getAutomationID(category, analysis_key, environment) {
async function uploadPayload(payload, repositoryNwo, apiDetails, logger) {
logger.info("Uploading results");
// If in test mode we don't want to upload the results
const testMode = process.env["TEST_MODE"] === "true" || false;
if (testMode) {
if (util.isInTestMode()) {
const payloadSaveFile = path.join(actionsUtil.getTemporaryDirectory(), "payload.json");
logger.info(`In test mode. Results are not uploaded. Saving to ${payloadSaveFile}`);
logger.info(`Payload: ${JSON.stringify(payload, null, 2)}`);
@ -311,26 +310,28 @@ async function waitForProcessing(repositoryNwo, sarifID, apiDetails, logger) {
logger.warning("Timed out waiting for analysis to finish processing. Continuing.");
break;
}
let response = undefined;
try {
const response = await client.request("GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id", {
response = await client.request("GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id", {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
sarif_id: sarifID,
});
const status = response.data.processing_status;
logger.info(`Analysis upload status is ${status}.`);
if (status === "complete") {
break;
}
else if (status === "pending") {
logger.debug("Analysis processing is still pending...");
}
else if (status === "failed") {
throw new Error(`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`);
}
}
catch (e) {
logger.warning(`An error occurred checking the status of the delivery. ${e} It should still be processed in the background, but errors that occur during processing may not be reported.`);
break;
}
const status = response.data.processing_status;
logger.info(`Analysis upload status is ${status}.`);
if (status === "complete") {
break;
}
else if (status === "pending") {
logger.debug("Analysis processing is still pending...");
}
else if (status === "failed") {
throw new Error(`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`);
}
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
}

2
lib/upload-lib.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/upload-sarif-action.js generated
View file

@ -51,7 +51,11 @@ async function run() {
const gitHubVersion = await (0, api_client_1.getGitHubVersionActionsOnly)();
const uploadResult = await upload_lib.uploadFromActions(actionsUtil.getRequiredInput("sarif_file"), gitHubVersion, apiDetails, (0, logging_1.getActionsLogger)());
core.setOutput("sarif-id", uploadResult.sarifID);
if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
// We don't upload results in test mode, so don't wait for processing
if ((0, util_1.isInTestMode)()) {
core.debug("In test mode. Waiting for processing is disabled.");
}
else if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
await upload_lib.waitForProcessing((0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY")), uploadResult.sarifID, apiDetails, (0, logging_1.getActionsLogger)());
}
await sendSuccessStatusReport(startedAt, uploadResult.statusReport);

2
lib/upload-sarif-action.js.map
View file

@ -1 +1 @@
{"version":3,"file":"upload-sarif-action.js","sourceRoot":"","sources":["../src/upload-sarif-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,4DAA8C;AAC9C,6CAA2D;AAC3D,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,iCAKgB;AAEhB,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAMvC,KAAK,UAAU,uBAAuB,CACpC,SAAe,EACf,WAA0C;IAE1C,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,cAAc,EACd,SAAS,EACT,SAAS,CACV,CAAC;IACF,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,GAAG,WAAW;KACf,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,IAAA,yBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;QACA,OAAO;KACR;IAED,IAAI;QACF,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;SAC9C,CAAC;QAEF,MAAM,aAAa,GAAG,MAAM,IAAA,wCAA2B,GAAE,CAAC;QAE1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACrD,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAC1C,aAAa,EACb,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,WAAW,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,KAAK,MAAM,EAAE;YAClE,MAAM,UAAU,CAAC,iBAAiB,CAChC,IAAA,+BAAkB,EAAC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,EAC5D,YAAY,CAAC,OAAO,EACpB,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;SACH;QACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC;KACrE;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,EACnC,SAAS,EACT,OAAO,EACP,KAAK,CACN,CACF,CAAC;QACF,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,sCAAsC,KAAK,EAAE,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
{"version":3,"file":"upload-sarif-action.js","sourceRoot":"","sources":["../src/upload-sarif-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,4DAA8C;AAC9C,6CAA2D;AAC3D,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,iCAMgB;AAEhB,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAMvC,KAAK,UAAU,uBAAuB,CACpC,SAAe,EACf,WAA0C;IAE1C,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,cAAc,EACd,SAAS,EACT,SAAS,CACV,CAAC;IACF,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,GAAG,WAAW;KACf,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,IAAA,yBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;QACA,OAAO;KACR;IAED,IAAI;QACF,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;SAC9C,CAAC;QAEF,MAAM,aAAa,GAAG,MAAM,IAAA,wCAA2B,GAAE,CAAC;QAE1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACrD,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAC1C,aAAa,EACb,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QAEjD,qEAAqE;QACrE,IAAI,IAAA,mBAAY,GAAE,EAAE;YAClB,IAAI,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;SACjE;aAAM,IAAI,WAAW,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,KAAK,MAAM,EAAE;YACzE,MAAM,UAAU,CAAC,iBAAiB,CAChC,IAAA,+BAAkB,EAAC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,EAC5D,YAAY,CAAC,OAAO,EACpB,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;SACH;QACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC;KACrE;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,EACnC,SAAS,EACT,OAAO,EACP,KAAK,CACN,CACF,CAAC;QACF,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,sCAAsC,KAAK,EAAE,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

22
lib/util.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@ -553,9 +553,9 @@ exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-qu
*/
async function getMlPoweredJsQueriesPack(codeQL) {
if (await codeQlVersionAbove(codeQL, "2.8.4")) {
return { packName: exports.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.2.0" };
return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.2.0`;
}
return { packName: exports.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" };
return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`;
}
exports.getMlPoweredJsQueriesPack = getMlPoweredJsQueriesPack;
/**
@ -580,7 +580,10 @@ exports.getMlPoweredJsQueriesPack = getMlPoweredJsQueriesPack;
* explanation as to why this is.
*/
function getMlPoweredJsQueriesStatus(config) {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK_NAME);
const mlPoweredJsQueryPacks = (config.packs.javascript || [])
.map((pack) => pack.split("@"))
.filter((packNameVersion) => packNameVersion[0] === "codeql/javascript-experimental-atm-queries" &&
packNameVersion.length <= 2);
switch (mlPoweredJsQueryPacks.length) {
case 1:
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
@ -588,7 +591,7 @@ function getMlPoweredJsQueriesStatus(config) {
// with each version of the CodeQL Action. Therefore in practice we should only hit the
// `latest` case here when customers have explicitly added the ML-powered query pack to their
// CodeQL config.
return mlPoweredJsQueryPacks[0].version || "latest";
return mlPoweredJsQueryPacks[0][1] || "latest";
case 0:
return "false";
default:
@ -623,4 +626,13 @@ async function checkActionVersion(version) {
}
}
exports.checkActionVersion = checkActionVersion;
/*
* Returns whether we are in test mode.
*
* In test mode, we don't upload SARIF results or status reports to the GitHub API.
*/
function isInTestMode() {
return process.env["TEST_MODE"] === "true" || false;
}
exports.isInTestMode = isInTestMode;
//# sourceMappingURL=util.js.map

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

26
lib/util.test.js generated
View file

@ -209,40 +209,28 @@ const ML_POWERED_JS_STATUS_TESTS = [
// If no packs are loaded, status is false.
[[], "false"],
// If another pack is loaded but not the ML-powered query pack, status is false.
[[{ packName: "someOtherPack" }], "false"],
[["someOtherPack"], "false"],
// If the ML-powered query pack is loaded with a specific version, status is that version.
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" }],
"~0.1.0",
],
[[`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
// If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
// status is the version of the ML-powered query pack.
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" },
],
["someOtherPack", `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`],
"~0.1.0",
],
// If the ML-powered query pack is loaded without a version, the status is "latest".
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
[[util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
// If the ML-powered query pack is loaded with two different versions, the status is "other".
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.1`,
`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.2`,
],
"other",
],
// If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
// the status is "latest".
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME },
],
"latest",
],
[["someOtherPack", util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
const packDescriptions = `[${packs

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long

2
node_modules/.package-lock.json generated vendored
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "2.1.10",
"version": "2.1.11",
"lockfileVersion": 2,
"requires": true,
"packages": {

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "codeql",
"version": "2.1.10",
"version": "2.1.11",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "codeql",
"version": "2.1.10",
"version": "2.1.11",
"license": "MIT",
"dependencies": {
"@actions/artifact": "^1.0.0",

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "2.1.10",
"version": "2.1.11",
"private": true,
"description": "CodeQL action",
"scripts": {

11
pr-checks/checks/packaging-config-inputs-js.yml
View file

@ -1,12 +1,11 @@
name: "Packaging: Config and input"
description: "Checks that specifying packages using a combination of a config file and input to the Action works"
versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
os: ["ubuntu-latest", "macos-latest"]
versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
steps:
- uses: ./../action/init
with:
config-file: ".github/codeql/codeql-config-packaging3.yml"
packs: +dsp-testing/codeql-pack1@0.1.0
packs: +dsp-testing/codeql-pack1@1.0.0
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
@ -21,11 +20,11 @@ steps:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

9
pr-checks/checks/packaging-config-js.yml
View file

@ -1,7 +1,6 @@
name: "Packaging: Config file"
description: "Checks that specifying packages using only a config file works"
versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
os: ["ubuntu-latest", "macos-latest"]
versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
steps:
- uses: ./../action/init
with:
@ -20,11 +19,11 @@ steps:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

11
pr-checks/checks/packaging-inputs-js.yml
View file

@ -1,13 +1,12 @@
name: "Packaging: Action input"
description: "Checks that specifying packages using the input to the Action works"
versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
os: ["ubuntu-latest", "macos-latest"]
versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
steps:
- uses: ./../action/init
with:
config-file: ".github/codeql/codeql-config-packaging2.yml"
languages: javascript
packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
packs: dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2, dsp-testing/codeql-pack3:other-query.ql
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
shell: bash
@ -21,11 +20,11 @@ steps:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

10
pr-checks/checks/split-workflow.yml
View file

@ -1,12 +1,12 @@
name: "Split workflow"
description: "Tests a split-up workflow in which we first build a database and later analyze it"
versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
os: ["ubuntu-latest", "macos-latest"]
versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
steps:
- uses: ./../action/init
with:
config-file: ".github/codeql/codeql-config-packaging3.yml"
packs: +dsp-testing/codeql-pack1@0.1.0
packs: +dsp-testing/codeql-pack1@1.0.0
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
@ -35,11 +35,11 @@ steps:
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."

13
python-setup/install_tools.sh
View file

@ -29,9 +29,16 @@ python3 -m pip install --user poetry!=1.0.10
python3 -m pip install --user pipenv
if command -v python2 >/dev/null 2>&1; then
# Setup Python 2 dependency installation tools.
# The Ubuntu 20.04 GHA environment does not come with a Python 2 pip
curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
# Setup Python 2 dependency installation tools. The Ubuntu 20.04 GHA environment
# does not come with a Python 2 pip, but if it is already installed, don't try to
# install it again (since that causes problems).
#
# This might seem like a hypothetical situation, but it happens all the time in our
# internal testing where we run the action twice in a row.
if ! python2 -m pip --version; then
echo "Will install pip for python2"
curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
fi
python2 -m pip install --user --upgrade pip setuptools wheel

4
src/actions-util.ts
View file

@ -15,6 +15,7 @@ import {
GITHUB_DOTCOM_URL,
isGitHubGhesVersionBelow,
isHTTPError,
isInTestMode,
UserError,
} from "./util";
@ -763,8 +764,7 @@ export async function sendStatusReport<S extends StatusReportBase>(
const statusReportJSON = JSON.stringify(statusReport);
core.debug(`Sending status report: ${statusReportJSON}`);
// If in test mode we don't want to upload the results
const testMode = process.env["TEST_MODE"] === "true" || false;
if (testMode) {
if (isInTestMode()) {
core.debug("In test mode. Status reports are not uploaded.");
return true;
}

10
src/analyze-action-env.test.ts
View file

@ -22,14 +22,17 @@ setupTests(test);
test("analyze action with RAM & threads from environment variables", async (t) => {
await util.withTmpDir(async (tmpDir) => {
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
process.env["GITHUB_REPOSITORY"] = "fake/repository";
process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
sinon
.stub(actionsUtil, "createStatusReportBase")
.resolves({} as actionsUtil.StatusReportBase);
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
@ -38,6 +41,7 @@ test("analyze action with RAM & threads from environment variables", async (t) =
requiredInputStub.withArgs("upload-database").returns("false");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
setupActionsVars(tmpDir, tmpDir);
mockFeatureFlagApiEndpoint(200, {});

10
src/analyze-action-input.test.ts
View file

@ -22,14 +22,17 @@ setupTests(test);
test("analyze action with RAM & threads from action inputs", async (t) => {
await util.withTmpDir(async (tmpDir) => {
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
process.env["GITHUB_REPOSITORY"] = "fake/repository";
process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
sinon
.stub(actionsUtil, "createStatusReportBase")
.resolves({} as actionsUtil.StatusReportBase);
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
@ -38,6 +41,7 @@ test("analyze action with RAM & threads from action inputs", async (t) => {
requiredInputStub.withArgs("upload-database").returns("false");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
setupActionsVars(tmpDir, tmpDir);
mockFeatureFlagApiEndpoint(200, {});

5
src/analyze-action.ts
View file

@ -196,7 +196,10 @@ async function run() {
// Possibly upload the database bundles for remote queries
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
if (
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
core.debug("In test mode. Waiting for processing is disabled.");
} else if (
uploadResult !== undefined &&
actionsUtil.getRequiredInput("wait-for-processing") === "true"
) {

37
src/analyze.test.ts
View file

@ -3,7 +3,6 @@ import * as path from "path";
import test from "ava";
import * as yaml from "js-yaml";
import { clean } from "semver";
import * as sinon from "sinon";
import { runQueries } from "./analyze";
@ -35,18 +34,8 @@ test("status report fields and search path setting", async (t) => {
const addSnippetsFlag = "";
const threadsFlag = "";
const packs = {
[Language.cpp]: [
{
packName: "a/b",
version: clean("1.0.0")!,
},
],
[Language.java]: [
{
packName: "c/d",
version: clean("2.0.0")!,
},
],
[Language.cpp]: ["a/b@1.0.0"],
[Language.java]: ["c/d@2.0.0"],
};
for (const language of Object.values(Language)) {
@ -241,32 +230,10 @@ test("status report fields and search path setting", async (t) => {
query: "bar.ql",
},
];
const qlsPackContentCpp = [
{
qlpack: "a/b",
version: "1.0.0",
},
];
const qlsPackContentJava = [
{
qlpack: "c/d",
version: "2.0.0",
},
];
for (const lang of Object.values(Language)) {
t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
const packSuiteName = `${lang}-queries-packs.qls`;
if (lang === Language.cpp) {
t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
} else if (lang === Language.java) {
t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
} else {
t.false(
fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName))
);
}
}
function readContents(name: string) {

50
src/analyze.ts
View file

@ -242,6 +242,7 @@ export async function runQueries(
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
@ -283,12 +284,12 @@ export async function runQueries(
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(
await runQueryGroup(
...(await runQueryPacks(
language,
"packs",
createPackSuiteContents(packsWithVersion),
packsWithVersion,
undefined
)
))
);
ranCustom = true;
}
@ -386,27 +387,38 @@ export async function runQueries(
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
async function runQueryPacks(
language: Language,
type: string,
packs: string[],
searchPath: string | undefined
): Promise<string[]> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Run the queries individually instead of all at once to avoid command
// line length restrictions, particularly on windows.
for (const pack of packs) {
logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
const codeql = await getCodeQL(config.codeQLCmd);
await codeql.databaseRunQueries(
databasePath,
searchPath,
pack,
memoryFlag,
threadsFlag
);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
}
return packs;
}
}
function createQuerySuiteContents(queries: string[]) {
return queries.map((q: string) => `- query: ${q}`).join("\n");
}
function createPackSuiteContents(
packsWithVersion: configUtils.PackWithVersion[]
) {
return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
}
function packWithVersionToQuerySuiteEntry(
pack: configUtils.PackWithVersion
): string {
let text = `- qlpack: ${pack.packName}`;
if (pack.version) {
text += `\n version: ${pack.version}`;
}
return text;
}
export async function runFinalize(
outputDir: string,
threadsFlag: string,

12
src/codeql.ts
View file

@ -9,7 +9,7 @@ import * as semver from "semver";
import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
import * as api from "./api-client";
import { Config, PackWithVersion } from "./config-utils";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { isTracedLanguage, Language } from "./languages";
@ -117,7 +117,7 @@ export interface CodeQL {
/**
* Run 'codeql pack download'.
*/
packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput>;
packDownload(packs: string[]): Promise<PackDownloadOutput>;
/**
* Run 'codeql database cleanup'.
@ -957,13 +957,14 @@ async function getCodeQLForCmd(
* downloaded. The check to determine what the latest version is is done
* each time this package is requested.
*/
async packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput> {
async packDownload(packs: string[]): Promise<PackDownloadOutput> {
const codeqlArgs = [
"pack",
"download",
"--format=json",
"--resolve-query-specs",
...getExtraOptionsFromEnv(["pack", "download"]),
...packs.map(packWithVersionToString),
...packs,
];
const output = await runTool(cmd, codeqlArgs);
@ -1035,9 +1036,6 @@ async function getCodeQLForCmd(
return codeql;
}
function packWithVersionToString(pack: PackWithVersion): string {
return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*/

121
src/config-utils.test.ts
View file

@ -3,7 +3,6 @@ import * as path from "path";
import * as github from "@actions/github";
import test, { ExecutionContext } from "ava";
import { clean } from "semver";
import * as sinon from "sinon";
import * as api from "./api-client";
@ -1132,12 +1131,7 @@ test("Config specifies packages", async (t) => {
getRunnerLogger(true)
);
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "a/b",
version: clean("1.2.3"),
},
],
[Language.javascript]: ["a/b@1.2.3"],
});
});
});
@ -1194,18 +1188,8 @@ test("Config specifies packages for multiple languages", async (t) => {
getRunnerLogger(true)
);
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "a/b",
version: clean("1.2.3"),
},
],
[Language.python]: [
{
packName: "c/d",
version: clean("1.2.3"),
},
],
[Language.javascript]: ["a/b@1.2.3"],
[Language.python]: ["c/d@1.2.3"],
});
t.deepEqual(queries, {
cpp: {
@ -1437,7 +1421,7 @@ const parsePacksMacro = test.macro({
t: ExecutionContext<unknown>,
packsByLanguage: string[] | Record<string, string[]>,
languages: Language[],
expected: Partial<Record<Language, configUtils.PackWithVersion[]>>
expected: Partial<Record<Language, string[]>>
) =>
t.deepEqual(
configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"),
@ -1490,10 +1474,7 @@ const invalidPackNameMacro = test.macro({
test("no packs", parsePacksMacro, {}, [], {});
test("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [Language.cpp], {
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: clean("1.2.3") as string },
],
[Language.cpp]: ["a/b", "c/d@1.2.3"],
});
test(
"two packs with spaces",
@ -1501,10 +1482,7 @@ test(
[" a/b ", " c/d@1.2.3 "],
[Language.cpp],
{
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: clean("1.2.3") as string },
],
[Language.cpp]: ["a/b", "c/d@1.2.3"],
}
);
test(
@ -1515,14 +1493,46 @@ test(
[Language.java]: ["d/e", "f/g@1.2.3"],
},
[Language.cpp, Language.java, Language.csharp],
{
[Language.cpp]: ["a/b", "c/d@1.2.3"],
[Language.java]: ["d/e", "f/g@1.2.3"],
}
);
test(
"packs with other valid names",
parsePacksMacro,
[
// ranges are ok
"c/d@1.0",
"c/d@~1.0.0",
"c/d@~1.0.0:a/b",
"c/d@~1.0.0+abc:a/b",
"c/d@~1.0.0-abc:a/b",
"c/d:a/b",
// whitespace is removed
" c/d @ ~1.0.0 : b.qls ",
// and it is retained within a path
" c/d @ ~1.0.0 : b/a path with/spaces.qls ",
// this is valid. the path is '@'. It will probably fail when passed to the CLI
"c/d@1.2.3:@",
// this is valid, too. It will fail if it doesn't match a path
// (globbing is not done)
"c/d@1.2.3:+*)_(",
],
[Language.cpp],
{
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: clean("1.2.3") as string },
],
[Language.java]: [
{ packName: "d/e", version: undefined },
{ packName: "f/g", version: clean("1.2.3") as string },
"c/d@1.0",
"c/d@~1.0.0",
"c/d@~1.0.0:a/b",
"c/d@~1.0.0+abc:a/b",
"c/d@~1.0.0-abc:a/b",
"c/d:a/b",
"c/d@~1.0.0:b.qls",
"c/d@~1.0.0:b/a path with/spaces.qls",
"c/d@1.2.3:@",
"c/d@1.2.3:+*)_(",
],
}
);
@ -1553,7 +1563,14 @@ test(invalidPackNameMacro, "c"); // all packs require at least a scope and a nam
test(invalidPackNameMacro, "c-/d");
test(invalidPackNameMacro, "-c/d");
test(invalidPackNameMacro, "c/d_d");
test(invalidPackNameMacro, "c/d@x");
test(invalidPackNameMacro, "c/d@@");
test(invalidPackNameMacro, "c/d@1.0.0:");
test(invalidPackNameMacro, "c/d:");
test(invalidPackNameMacro, "c/d:/a");
test(invalidPackNameMacro, "@1.0.0:a");
test(invalidPackNameMacro, "c/d@../a");
test(invalidPackNameMacro, "c/d@b/../a");
test(invalidPackNameMacro, "c/d:z@1");
/**
* Test macro for testing the packs block and the packs input
@ -1598,7 +1615,7 @@ parseInputAndConfigErrorMacro.title = (providedTitle: string) =>
`Parse Packs input and config Error: ${providedTitle}`;
test("input only", parseInputAndConfigMacro, {}, " c/d ", [Language.cpp], {
[Language.cpp]: [{ packName: "c/d", version: undefined }],
[Language.cpp]: ["c/d"],
});
test(
@ -1608,10 +1625,7 @@ test(
"a/b , c/d@1.2.3",
[Language.cpp],
{
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: "1.2.3" },
],
[Language.cpp]: ["a/b", "c/d@1.2.3"],
}
);
@ -1622,10 +1636,7 @@ test(
" + a/b , c/d@1.2.3 ",
[Language.cpp],
{
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: "1.2.3" },
],
[Language.cpp]: ["a/b", "c/d@1.2.3"],
}
);
@ -1636,10 +1647,7 @@ test(
" ",
[Language.cpp],
{
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: undefined },
],
[Language.cpp]: ["a/b", "c/d"],
}
);
@ -1650,10 +1658,7 @@ test(
" e/f, g/h@1.2.3 ",
[Language.cpp],
{
[Language.cpp]: [
{ packName: "e/f", version: undefined },
{ packName: "g/h", version: "1.2.3" },
],
[Language.cpp]: ["e/f", "g/h@1.2.3"],
}
);
@ -1664,12 +1669,7 @@ test(
" +e/f, g/h@1.2.3 ",
[Language.cpp],
{
[Language.cpp]: [
{ packName: "e/f", version: undefined },
{ packName: "g/h", version: "1.2.3" },
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: undefined },
],
[Language.cpp]: ["e/f", "g/h@1.2.3", "a/b", "c/d"],
}
);
@ -1760,10 +1760,7 @@ const mlPoweredQueriesMacro = test.macro({
if (expectedVersionString !== undefined) {
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "codeql/javascript-experimental-atm-queries",
version: expectedVersionString,
},
`codeql/javascript-experimental-atm-queries@${expectedVersionString}`,
],
});
} else {

109
src/config-utils.ts
View file

@ -155,14 +155,7 @@ export interface Config {
injectedMlQueries: boolean;
}
export type Packs = Partial<Record<Language, PackWithVersion[]>>;
export interface PackWithVersion {
/** qualified name of a package reference */
packName: string;
/** version of the package, or undefined, which means latest version */
version?: string;
}
export type Packs = Partial<Record<Language, string[]>>;
/**
* A list of queries from https://github.com/github/codeql that
@ -309,9 +302,7 @@ async function addBuiltinSuiteQueries(
))) &&
languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") &&
!packs.javascript?.some(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
) &&
!packs.javascript?.some(isMlPoweredJsQueriesPack) &&
(await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) &&
(await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES))
) {
@ -327,6 +318,14 @@ async function addBuiltinSuiteQueries(
return injectedMlQueries;
}
function isMlPoweredJsQueriesPack(pack: string) {
return (
pack === ML_POWERED_JS_QUERIES_PACK_NAME ||
pack.startsWith(`${ML_POWERED_JS_QUERIES_PACK_NAME}@`) ||
pack.startsWith(`${ML_POWERED_JS_QUERIES_PACK_NAME}:`)
);
}
/**
* Retrieve the set of queries at localQueryPath and add them to resultMap.
*/
@ -1173,7 +1172,7 @@ export function parsePacksFromConfig(
}
packs[lang] = [];
for (const packStr of packsArr) {
packs[lang].push(toPackWithVersion(packStr, configFile));
packs[lang].push(validatePacksSpecification(packStr, configFile));
}
}
return packs;
@ -1207,35 +1206,89 @@ function parsePacksFromInput(
return {
[languages[0]]: packsInput.split(",").reduce((packs, pack) => {
packs.push(toPackWithVersion(pack, ""));
packs.push(validatePacksSpecification(pack, ""));
return packs;
}, [] as PackWithVersion[]),
}, [] as string[]),
};
}
function toPackWithVersion(packStr, configFile?: string): PackWithVersion {
/**
* Validates that this package specification is syntactically correct.
* It may not point to any real package, but after this function returns
* without throwing, we are guaranteed that the package specification
* is roughly correct.
*
* The CLI itself will do a more thorough validation of the package
* specification.
*
* A package specification looks like this:
*
* `scope/name@version:path`
*
* Version and path are optional.
*
* @param packStr the package specification to verify.
* @param configFile Config file to use for error reporting
*/
export function validatePacksSpecification(
packStr: string,
configFile?: string
): string {
if (typeof packStr !== "string") {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
const nameWithVersion = packStr.trim().split("@");
let version: string | undefined;
if (
nameWithVersion.length > 2 ||
!PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])
) {
packStr = packStr.trim();
const atIndex = packStr.indexOf("@");
const colonIndex = packStr.indexOf(":", atIndex);
const packStart = 0;
const versionStart = atIndex + 1 || undefined;
const pathStart = colonIndex + 1 || undefined;
const packEnd = Math.min(
atIndex > 0 ? atIndex : Infinity,
colonIndex > 0 ? colonIndex : Infinity,
packStr.length
);
const versionEnd = versionStart
? Math.min(colonIndex > 0 ? colonIndex : Infinity, packStr.length)
: undefined;
const pathEnd = pathStart ? packStr.length : undefined;
const packName = packStr.slice(packStart, packEnd).trim();
const version = versionStart
? packStr.slice(versionStart, versionEnd).trim()
: undefined;
const packPath = pathStart
? packStr.slice(pathStart, pathEnd).trim()
: undefined;
if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
throw new Error(getPacksStrInvalid(packStr, configFile));
} else if (nameWithVersion.length === 2) {
version = semver.clean(nameWithVersion[1]) || undefined;
if (!version) {
}
if (version) {
try {
new semver.Range(version);
} catch (e) {
// The range string is invalid. OK to ignore the caught error
throw new Error(getPacksStrInvalid(packStr, configFile));
}
}
return {
packName: nameWithVersion[0].trim(),
version,
};
if (
packPath &&
(path.isAbsolute(packPath) || path.normalize(packPath) !== packPath)
) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
if (!packPath && pathStart) {
// 0 length path
throw new Error(getPacksStrInvalid(packStr, configFile));
}
return (
packName + (version ? `@${version}` : "") + (packPath ? `:${packPath}` : "")
);
}
// exported for testing

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20220421"
"bundleVersion": "codeql-bundle-20220428"
}

31
src/upload-lib.ts
View file

@ -3,6 +3,7 @@ import * as path from "path";
import zlib from "zlib";
import * as core from "@actions/core";
import { OctokitResponse } from "@octokit/types";
import fileUrl from "file-url";
import * as jsonschema from "jsonschema";
import * as semver from "semver";
@ -98,8 +99,7 @@ async function uploadPayload(
logger.info("Uploading results");
// If in test mode we don't want to upload the results
const testMode = process.env["TEST_MODE"] === "true" || false;
if (testMode) {
if (util.isInTestMode()) {
const payloadSaveFile = path.join(
actionsUtil.getTemporaryDirectory(),
"payload.json"
@ -472,8 +472,9 @@ export async function waitForProcessing(
);
break;
}
let response: OctokitResponse<any> | undefined = undefined;
try {
const response = await client.request(
response = await client.request(
"GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id",
{
owner: repositoryNwo.owner,
@ -481,22 +482,24 @@ export async function waitForProcessing(
sarif_id: sarifID,
}
);
const status = response.data.processing_status;
logger.info(`Analysis upload status is ${status}.`);
if (status === "complete") {
break;
} else if (status === "pending") {
logger.debug("Analysis processing is still pending...");
} else if (status === "failed") {
throw new Error(
`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`
);
}
} catch (e) {
logger.warning(
`An error occurred checking the status of the delivery. ${e} It should still be processed in the background, but errors that occur during processing may not be reported.`
);
break;
}
const status = response.data.processing_status;
logger.info(`Analysis upload status is ${status}.`);
if (status === "complete") {
break;
} else if (status === "pending") {
logger.debug("Analysis processing is still pending...");
} else if (status === "failed") {
throw new Error(
`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`
);
}
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
}
logger.endGroup();

7
src/upload-sarif-action.ts
View file

@ -9,6 +9,7 @@ import {
checkActionVersion,
getRequiredEnvParam,
initializeEnvironment,
isInTestMode,
Mode,
} from "./util";
@ -66,7 +67,11 @@ async function run() {
getActionsLogger()
);
core.setOutput("sarif-id", uploadResult.sarifID);
if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
// We don't upload results in test mode, so don't wait for processing
if (isInTestMode()) {
core.debug("In test mode. Waiting for processing is disabled.");
} else if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
await upload_lib.waitForProcessing(
parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
uploadResult.sarifID,

30
src/util.test.ts
View file

@ -8,7 +8,7 @@ import test, { ExecutionContext } from "ava";
import * as sinon from "sinon";
import * as api from "./api-client";
import { Config, PackWithVersion } from "./config-utils";
import { Config } from "./config-utils";
import { getRunnerLogger, Logger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
@ -294,44 +294,32 @@ async function mockStdInForAuthExpectError(
);
}
const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [
const ML_POWERED_JS_STATUS_TESTS: Array<[string[], string]> = [
// If no packs are loaded, status is false.
[[], "false"],
// If another pack is loaded but not the ML-powered query pack, status is false.
[[{ packName: "someOtherPack" }], "false"],
[["someOtherPack"], "false"],
// If the ML-powered query pack is loaded with a specific version, status is that version.
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" }],
"~0.1.0",
],
[[`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
// If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
// status is the version of the ML-powered query pack.
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" },
],
["someOtherPack", `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`],
"~0.1.0",
],
// If the ML-powered query pack is loaded without a version, the status is "latest".
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
[[util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
// If the ML-powered query pack is loaded with two different versions, the status is "other".
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.1`,
`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.2`,
],
"other",
],
// If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
// the status is "latest".
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME },
],
"latest",
],
[["someOtherPack", util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {

29
src/util.ts
View file

@ -11,7 +11,7 @@ import * as api from "./api-client";
import { getApiClient, GitHubApiDetails } from "./api-client";
import * as apiCompatibility from "./api-compatibility.json";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import { Config, PackWithVersion } from "./config-utils";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
@ -663,11 +663,11 @@ export const ML_POWERED_JS_QUERIES_PACK_NAME =
*/
export async function getMlPoweredJsQueriesPack(
codeQL: CodeQL
): Promise<PackWithVersion> {
): Promise<string> {
if (await codeQlVersionAbove(codeQL, "2.8.4")) {
return { packName: ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.2.0" };
return `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.2.0`;
}
return { packName: ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" };
return `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`;
}
/**
@ -692,9 +692,13 @@ export async function getMlPoweredJsQueriesPack(
* explanation as to why this is.
*/
export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
);
const mlPoweredJsQueryPacks = (config.packs.javascript || [])
.map((pack) => pack.split("@"))
.filter(
(packNameVersion) =>
packNameVersion[0] === "codeql/javascript-experimental-atm-queries" &&
packNameVersion.length <= 2
);
switch (mlPoweredJsQueryPacks.length) {
case 1:
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
@ -702,7 +706,7 @@ export function getMlPoweredJsQueriesStatus(config: Config): string {
// with each version of the CodeQL Action. Therefore in practice we should only hit the
// `latest` case here when customers have explicitly added the ML-powered query pack to their
// CodeQL config.
return mlPoweredJsQueryPacks[0].version || "latest";
return mlPoweredJsQueryPacks[0][1] || "latest";
case 0:
return "false";
default:
@ -742,3 +746,12 @@ export async function checkActionVersion(version: string) {
}
}
}
/*
* Returns whether we are in test mode.
*
* In test mode, we don't upload SARIF results or status reports to the GitHub API.
*/
export function isInTestMode(): boolean {
return process.env["TEST_MODE"] === "true" || false;
}

6
tests/multi-language-repo/.github/codeql/codeql-config-packaging.yml vendored
View file

@ -3,8 +3,10 @@ name: Pack testing in the CodeQL Action
disable-default-queries: true
packs:
javascript:
- dsp-testing/codeql-pack1@0.1.0
- dsp-testing/codeql-pack2 # latest
- dsp-testing/codeql-pack1@1.0.0
- dsp-testing/codeql-pack2
- dsp-testing/codeql-pack3:other-query.ql
paths-ignore:
- tests
- lib

3
tests/multi-language-repo/.github/codeql/codeql-config-packaging3.yml vendored
View file

@ -3,7 +3,8 @@ name: Pack testing in the CodeQL Action
disable-default-queries: true
packs:
javascript:
- dsp-testing/codeql-pack2 # latest
- dsp-testing/codeql-pack2
- dsp-testing/codeql-pack3:other-query.ql
paths-ignore:
- tests
- lib