Merge remote-tracking branch 'origin/main' into henrymercer/codeql-cli-2.10.3

This commit is contained in:
Henry Mercer 2022-08-15 14:40:51 +01:00
commit af9d911fb5
63 changed files with 2370 additions and 772 deletions

View file

@ -0,0 +1,60 @@
name: Check Code-Scanning Config
description: |
Checks the code scanning configuration file generated by the
action to ensure it contains the expected contents
inputs:
languages:
required: false
description: The languages field passed to the init action.
packs:
required: false
description: The packs field passed to the init action.
queries:
required: false
description: The queries field passed to the init action.
config-file-test:
required: false
description: |
The location of the config file to use. If empty,
then no config file is used.
expected-config-file-contents:
required: true
description: |
A JSON string containing the exact contents of the config file.
tools:
required: true
description: |
The url of codeql to use.
runs:
using: composite
steps:
- uses: ./../action/init
with:
languages: ${{ inputs.languages }}
config-file: ${{ inputs.config-file-test }}
queries: ${{ inputs.queries }}
packs: ${{ inputs.packs }}
tools: ${{ inputs.tools }}
db-location: ${{ runner.temp }}/codescanning-config-cli-test
- name: Install dependencies
shell: bash
run: npm install --location=global ts-node js-yaml
- name: Check config
working-directory: ${{ github.action_path }}
shell: bash
run: ts-node ./index.ts "${{ runner.temp }}/user-config.yaml" '${{ inputs.expected-config-file-contents }}'
- name: Clean up
shell: bash
if: always()
run: |
rm -rf ${{ runner.temp }}/codescanning-config-cli-test
rm -rf ${{ runner.temp }}/user-config.yaml

View file

@ -0,0 +1,39 @@
import * as core from '@actions/core'
import * as yaml from 'js-yaml'
import * as fs from 'fs'
import * as assert from 'assert'
const actualConfig = loadActualConfig()
const rawExpectedConfig = process.argv[3].trim()
if (!rawExpectedConfig) {
core.info('No expected configuration provided')
} else {
core.startGroup('Expected generated user config')
core.info(yaml.dump(JSON.parse(rawExpectedConfig)))
core.endGroup()
}
const expectedConfig = rawExpectedConfig ? JSON.parse(rawExpectedConfig) : undefined;
assert.deepStrictEqual(
actualConfig,
expectedConfig,
'Expected configuration does not match actual configuration'
);
function loadActualConfig() {
if (!fs.existsSync(process.argv[2])) {
core.info('No configuration file found')
return undefined
} else {
const rawActualConfig = fs.readFileSync(process.argv[2], 'utf8')
core.startGroup('Actual generated user config')
core.info(rawActualConfig)
core.endGroup()
return yaml.load(rawActualConfig)
}
}

View file

@ -49,4 +49,4 @@ runs:
queries-not-run: ${{ inputs.queries-not-run}}
- name: Cleanup after test
shell: bash
run: rm -rf "$RUNNER_TEMP/results" "$RUNNER_TEMP//query-filter-test"
run: rm -rf "$RUNNER_TEMP/results" "$RUNNER_TEMP/query-filter-test"

View file

@ -81,6 +81,15 @@ jobs:
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check sarif
uses: ./../action/.github/check-sarif
if: matrix.os != 'windows-latest' || matrix.version == 'latest' || matrix.version
== 'nightly-latest'
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: js/ml-powered/nosql-injection,js/ml-powered/path-injection,js/ml-powered/sql-injection,js/ml-powered/xss
queries-not-run: foo,bar
- name: Check results
# Running ML-powered queries on Windows requires CodeQL CLI 2.9.0+. We don't run these checks
# against Windows and `cached` while CodeQL CLI 2.9.0 makes its way into `cached` to avoid the

View file

@ -0,0 +1,100 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# pip install ruamel.yaml && python3 sync.py
# to regenerate this file.
name: 'PR Check - Packaging: Config and input passed to the CLI'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- releases/v1
- releases/v2
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
workflow_dispatch: {}
jobs:
packaging-codescanning-config-inputs-js:
strategy:
matrix:
include:
- os: ubuntu-latest
version: latest
- os: macos-latest
version: latest
- os: windows-2019
version: latest
- os: windows-2022
version: latest
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-2019
version: cached
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
- os: windows-2019
version: nightly-latest
- os: windows-2022
version: nightly-latest
name: 'Packaging: Config and input passed to the CLI'
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Prepare test
id: prepare-test
uses: ./.github/prepare-test
with:
version: ${{ matrix.version }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml
packs: +dsp-testing/codeql-pack1@1.0.0
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
shell: bash
run: ./build.sh
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."
exit 1
fi
env:
CODEQL_PASS_CONFIG_TO_CLI: true
INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

View file

@ -72,6 +72,14 @@ jobs:
output: ${{ runner.temp }}/results
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |

View file

@ -71,6 +71,14 @@ jobs:
output: ${{ runner.temp }}/results
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |

View file

@ -72,6 +72,14 @@ jobs:
output: ${{ runner.temp }}/results
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |

View file

@ -63,6 +63,7 @@ jobs:
output: ${{ runner.temp }}/results
env:
TEST_MODE: true
- name: Assert No Results
shell: bash
run: |

View file

@ -0,0 +1,220 @@
# Tests that the generated code scanning config file contains the expected contents
name: Code-Scanning config CLI tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CODEQL_PASS_CONFIG_TO_CLI: true
on:
push:
branches:
- main
- releases/v1
- releases/v2
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
workflow_dispatch: {}
jobs:
code-scanning-config-tests:
continue-on-error: true
strategy:
fail-fast: true
matrix:
include:
- os: ubuntu-latest
version: latest
- os: macos-latest
version: latest
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
# Code-Scanning config not created because environment variable is not set
name: Code Scanning Configuration tests
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Prepare test
id: prepare-test
uses: ./.github/prepare-test
with:
version: ${{ matrix.version }}
- name: Empty file
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: "{}"
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Packs from input
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"packs": ["dsp-testing/codeql-pack1@1.0.0", "dsp-testing/codeql-pack2" ]
}
languages: javascript
packs: dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Packs from input with +
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"packs": ["dsp-testing/codeql-pack1@1.0.0", "dsp-testing/codeql-pack2" ]
}
languages: javascript
packs: + dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Queries from input
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"queries": [{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }]
}
languages: javascript
queries: ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Queries from input with +
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"queries": [{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }]
}
languages: javascript
queries: + ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Queries and packs from input with +
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"queries": [{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }],
"packs": ["dsp-testing/codeql-pack1@1.0.0", "dsp-testing/codeql-pack2" ]
}
languages: javascript
queries: + ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
packs: + dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Queries and packs from config
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"queries": [{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" }],
"packs": {
"javascript": ["dsp-testing/codeql-pack1@1.0.0", "dsp-testing/codeql-pack2" ]
}
}
languages: javascript
config-file-test: .github/codeql/queries-and-packs-config.yml
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Queries and packs from config overriden by input
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"queries": [{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }],
"packs": ["codeql/javascript-queries"]
}
languages: javascript
queries: ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
packs: codeql/javascript-queries
config-file-test: .github/codeql/queries-and-packs-config.yml
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Queries and packs from config merging with input
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"queries": [
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" },
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }
],
"packs": {
"javascript": ["dsp-testing/codeql-pack1@1.0.0", "dsp-testing/codeql-pack2", "codeql/javascript-queries" ]
}
}
languages: javascript
queries: + ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
packs: + codeql/javascript-queries
config-file-test: .github/codeql/queries-and-packs-config.yml
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Multi-language packs from config
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"packs": {
"javascript": ["dsp-testing/codeql-pack1@1.0.0", "dsp-testing/codeql-pack2" ],
"ruby": ["codeql/ruby-queries"]
},
"queries": [
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" }
]
}
languages: javascript,ruby
config-file-test: .github/codeql/multi-language-packs-config.yml
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Other config properties
if: success() || failure()
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: |
{
"name": "Config using all properties",
"packs": ["codeql/javascript-queries" ],
"disable-default-queries": true,
"paths-ignore": ["xxx"],
"paths": ["yyy"]
}
languages: javascript
packs: + codeql/javascript-queries
config-file-test: .github/codeql/other-config-properties.yml
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Config not generated when env var is not set
if: success() || failure()
env:
CODEQL_PASS_CONFIG_TO_CLI: false
uses: ./../action/.github/check-codescanning-config
with:
expected-config-file-contents: ""
languages: javascript
packs: + codeql/javascript-queries
config-file-test: .github/codeql/other-config-properties.yml
tools: ${{ steps.prepare-test.outputs.tools-url }}

View file

@ -44,7 +44,11 @@ const util = __importStar(require("./util"));
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
@ -69,7 +73,11 @@ const util = __importStar(require("./util"));
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
@ -94,7 +102,11 @@ const util = __importStar(require("./util"));
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);

View file

@ -1 +1 @@
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,IAAA,aAAI,EAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,KAAK;YACxB,UAAU,EAAE,EAAE;SACf,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,KAAK;YACxB,UAAU,EAAE,EAAE;SACf,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,EAAE;QACb,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,EAAE;QACT,iBAAiB,EAAE,EAAE;QACrB,OAAO;QACP,SAAS,EAAE,EAAE;QACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;QACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;QACrD,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,KAAK;QAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,KAAK;QACxB,UAAU,EAAE,EAAE;KACf,CAAC;IACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;AACrD,CAAC,CAAC,CAAC"}
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,IAAA,aAAI,EAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,sBAAsB,EAAE;gBACtB,iBAAiB,EAAE,KAAK;gBACxB,kBAAkB,EAAE,KAAK;gBACzB,oBAAoB,EAAE,KAAK;aAC5B;YACD,UAAU,EAAE,EAAE;SACf,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,sBAAsB,EAAE;gBACtB,iBAAiB,EAAE,KAAK;gBACxB,kBAAkB,EAAE,KAAK;gBACzB,oBAAoB,EAAE,KAAK;aAC5B;YACD,UAAU,EAAE,EAAE;SACf,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,EAAE;QACb,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,EAAE;QACT,iBAAiB,EAAE,EAAE;QACrB,OAAO;QACP,SAAS,EAAE,EAAE;QACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;QACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;QACrD,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,KAAK;QAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,sBAAsB,EAAE;YACtB,iBAAiB,EAAE,KAAK;YACxB,kBAAkB,EAAE,KAAK;YACzB,oBAAoB,EAAE,KAAK;SAC5B;QACD,UAAU,EAAE,EAAE;KACf,CAAC;IACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;AACrD,CAAC,CAAC,CAAC"}

121
lib/analyze.js generated
View file

@ -126,6 +126,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
// that here.
config.paths, config.pathsIgnore, config.languages, logger);
}
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
for (const language of config.languages) {
const queries = config.queries[language];
const queryFilters = validateQueryFilters(config.originalUserInput["query-filters"]);
@ -136,56 +137,82 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
if (!hasBuiltinQueries && !hasCustomQueries && !hasPackWithCustomQueries) {
throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
}
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
try {
if (hasPackWithCustomQueries) {
logger.info("Performing analysis with custom CodeQL Packs.");
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths = [];
if (queries["builtin"].length > 0) {
if (await util.useCodeScanningConfigInCli(codeql)) {
// If we are using the codescanning config in the CLI,
// much of the work needed to generate the query suites
// is done in the CLI. We just need to make a single
// call to run all the queries for each language and
// another to interpret the results.
logger.startGroup(`Running queries for ${language}`);
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push(await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"], queryFilters), undefined));
await runQueryGroup(language, "all", undefined, undefined);
// TODO should not be using `builtin` here. We should be using `all` instead.
// The status report does not support `all` yet.
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(language, undefined, sarifFile, config.debugMode);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
}
const startTimeCustom = new Date().getTime();
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
querySuitePaths.push(await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries, queryFilters), queries["custom"][i].searchPath));
else {
if (hasPackWithCustomQueries) {
logger.info("Performing analysis with custom CodeQL Packs.");
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"], queryFilters), undefined)));
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries, queryFilters), queries["custom"][i].searchPath)));
ranCustom = true;
}
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters));
ranCustom = true;
}
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(language, querySuitePaths, sarifFile, config.debugMode);
if (!cliCanCountBaseline) {
await injectLinesOfCode(sarifFile, language, locPromise);
}
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters));
ranCustom = true;
}
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(language, querySuitePaths, sarifFile, config.debugMode);
if (!cliCanCountBaseline)
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
if (!cliCanCountBaseline || countLocDebugMode)
if (!cliCanCountBaseline || countLocDebugMode) {
printLinesOfCodeSummary(logger, language, await locPromise);
if (cliCanCountBaseline)
}
if (cliCanCountBaseline) {
logger.info(await runPrintLinesOfCode(language));
}
}
catch (e) {
logger.info(String(e));
@ -199,7 +226,6 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
return statusReport;
async function runInterpretResults(language, queries, sarifFile, enableDebugLogging) {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
return await codeql.databaseInterpretResults(databasePath, queries, sarifFile, addSnippetsFlag, threadsFlag, enableDebugLogging ? "-vv" : "-v", automationDetailsId);
}
async function cliCanCountLoC() {
@ -207,17 +233,19 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
}
async function runPrintLinesOfCode(language) {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
return await codeql.databasePrintBaseline(databasePath);
}
async function runQueryGroup(language, type, querySuiteContents, searchPath) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
const querySuitePath = querySuiteContents
? `${databasePath}-queries-${type}.qls`
: undefined;
if (querySuiteContents && querySuitePath) {
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
}
await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, memoryFlag, threadsFlag);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
@ -232,7 +260,6 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, memoryFlag, threadsFlag);
return querySuitePath;
}

File diff suppressed because one or more lines are too long

12
lib/analyze.test.js generated
View file

@ -118,7 +118,11 @@ const util = __importStar(require("./util"));
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
@ -360,7 +364,11 @@ const stubConfig = {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
for (const options of [

File diff suppressed because one or more lines are too long

89
lib/codeql.js generated
View file

@ -22,12 +22,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_CONFIG_FILES = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const toolcache = __importStar(require("@actions/tool-cache"));
const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
const yaml = __importStar(require("js-yaml"));
const query_string_1 = __importDefault(require("query-string"));
const semver = __importStar(require("semver"));
const uuid_1 = require("uuid");
@ -80,6 +81,7 @@ exports.CODEQL_VERSION_COUNTS_LINES = "2.6.2";
const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
exports.CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
exports.CODEQL_VERSION_CONFIG_FILES = "2.10.1";
/**
* This variable controls using the new style of tracing from the CodeQL
* CLI. In particular, with versions above this we will use both indirect
@ -494,6 +496,10 @@ async function getCodeQLForCmd(cmd, checkVersion) {
}
}
}
const configLocation = await generateCodescanningConfig(codeql, config);
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
}
await runTool(cmd, [
"database",
"init",
@ -655,7 +661,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
codeqlArgs.push(querySuitePath);
if (querySuitePath) {
codeqlArgs.push(querySuitePath);
}
await runTool(cmd, codeqlArgs);
},
async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, verbosityFlag, automationDetailsId) {
@ -682,7 +690,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
codeqlArgs.push("--sarif-category", automationDetailsId);
}
codeqlArgs.push(databasePath);
codeqlArgs.push(...querySuitePaths);
if (querySuitePaths) {
codeqlArgs.push(...querySuitePaths);
}
// capture stdout, which contains analysis summaries
return await runTool(cmd, codeqlArgs);
},
@ -841,4 +851,77 @@ async function runTool(cmd, args = []) {
throw new CommandInvocationError(cmd, args, exitCode, error);
return output;
}
/**
* If appropriate, generates a code scanning configuration that is to be used for a scan.
* If the configuration is not to be generated, returns undefined.
*
* @param codeql The CodeQL object to use.
* @param config The configuration to use.
* @returns the path to the generated user configuration file.
*/
async function generateCodescanningConfig(codeql, config) {
var _a;
if (!(await util.useCodeScanningConfigInCli(codeql))) {
return;
}
const configLocation = path.resolve(config.tempDir, "user-config.yaml");
// make a copy so we can modify it
const augmentedConfig = cloneObject(config.originalUserInput);
// Inject the queries from the input
if (config.augmentationProperties.queriesInput) {
if (config.augmentationProperties.queriesInputCombines) {
augmentedConfig.queries = (augmentedConfig.queries || []).concat(config.augmentationProperties.queriesInput);
}
else {
augmentedConfig.queries = config.augmentationProperties.queriesInput;
}
}
if (((_a = augmentedConfig.queries) === null || _a === void 0 ? void 0 : _a.length) === 0) {
delete augmentedConfig.queries;
}
// Inject the packs from the input
if (config.augmentationProperties.packsInput) {
if (config.augmentationProperties.packsInputCombines) {
// At this point, we already know that this is a single-language analysis
if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs = (augmentedConfig.packs || []).concat(config.augmentationProperties.packsInput);
}
else if (!augmentedConfig.packs) {
augmentedConfig.packs = config.augmentationProperties.packsInput;
}
else {
// At this point, we know there is only one language.
// If there were more than one language, an error would already have been thrown.
const language = Object.keys(augmentedConfig.packs)[0];
augmentedConfig.packs[language] = augmentedConfig.packs[language].concat(config.augmentationProperties.packsInput);
}
}
else {
augmentedConfig.packs = config.augmentationProperties.packsInput;
}
}
if (Array.isArray(augmentedConfig.packs) && !augmentedConfig.packs.length) {
delete augmentedConfig.packs;
}
if (config.augmentationProperties.injectedMlQueries) {
// We need to inject the ML queries into the original user input before
// we pass this on to the CLI, to make sure these get run.
const packString = await util.getMlPoweredJsQueriesPack(codeql);
if (augmentedConfig.packs === undefined)
augmentedConfig.packs = [];
if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs.push(packString);
}
else {
if (!augmentedConfig.packs.javascript)
augmentedConfig.packs["javascript"] = [];
augmentedConfig.packs["javascript"].push(packString);
}
}
fs.writeFileSync(configLocation, yaml.dump(augmentedConfig));
return configLocation;
}
function cloneObject(obj) {
return JSON.parse(JSON.stringify(obj));
}
//# sourceMappingURL=codeql.js.map

File diff suppressed because one or more lines are too long

299
lib/codeql.test.js generated
View file

@ -23,10 +23,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.stubToolRunnerConstructor = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const toolcache = __importStar(require("@actions/tool-cache"));
const ava_1 = __importDefault(require("ava"));
const del_1 = __importDefault(require("del"));
const yaml = __importStar(require("js-yaml"));
const nock_1 = __importDefault(require("nock"));
const sinon = __importStar(require("sinon"));
const codeql = __importStar(require("./codeql"));
@ -48,8 +51,32 @@ const sampleGHAEApiDetails = {
url: "https://example.githubenterprise.com",
apiURL: undefined,
};
let stubConfig;
ava_1.default.beforeEach(() => {
(0, util_1.initializeEnvironment)(util_1.Mode.actions, "1.2.3");
stubConfig = {
languages: [languages_1.Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
},
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
});
(0, ava_1.default)("download codeql bundle cache", async (t) => {
await util.withTmpDir(async (tmpDir) => {
@ -238,25 +265,6 @@ ava_1.default.beforeEach(() => {
await codeqlObject.databaseInterpretResults("", [], "", "", "", "-v", "");
t.true(runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"), "--sarif-add-query-help should be present, but it is absent");
});
const stubConfig = {
languages: [languages_1.Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
},
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
(0, ava_1.default)("databaseInitCluster() Lua feature flag enabled, but old CLI", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
@ -287,6 +295,259 @@ const stubConfig = {
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, undefined, (0, feature_flags_1.createFeatureFlags)([]));
t.true(runnerConstructorStub.firstCall.args[1].includes("--no-internal-use-lua-tracing"), "--no-internal-use-lua-tracing should be present, but it is absent");
});
(0, ava_1.default)("databaseInitCluster() without injected codescanning config", async (t) => {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.8.1");
const thisStubConfig = {
...stubConfig,
tempDir,
augmentationProperties: {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
},
};
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, undefined, (0, feature_flags_1.createFeatureFlags)([]));
const args = runnerConstructorStub.firstCall.args[1];
// should NOT have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should have injected a codescanning config");
});
});
// Test macro for ensuring different variants of injected augmented configurations
const injectedConfigMacro = ava_1.default.macro({
exec: async (t, augmentationProperties, configOverride, expectedConfig) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "true";
try {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
const thisStubConfig = {
...stubConfig,
...configOverride,
tempDir,
augmentationProperties,
};
await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, undefined, (0, feature_flags_1.createFeatureFlags)([]));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.truthy(configArg, "Should have injected a codescanning config");
const configFile = configArg.split("=")[1];
const augmentedConfig = yaml.load(fs.readFileSync(configFile, "utf8"));
t.deepEqual(augmentedConfig, expectedConfig);
await (0, del_1.default)(configFile, { force: true });
});
}
finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
},
title: (providedTitle = "") => `databaseInitCluster() injected config: ${providedTitle}`,
});
(0, ava_1.default)("basic", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
}, {}, {});
(0, ava_1.default)("injected ML queries", injectedConfigMacro, {
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
}, {}, {
packs: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
});
(0, ava_1.default)("injected ML queries with existing packs", injectedConfigMacro, {
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
}, {
originalUserInput: {
packs: { javascript: ["codeql/something-else"] },
},
}, {
packs: {
javascript: [
"codeql/something-else",
"codeql/javascript-experimental-atm-queries@~0.3.0",
],
},
});
(0, ava_1.default)("injected ML queries with existing packs of different language", injectedConfigMacro, {
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
}, {
originalUserInput: {
packs: { cpp: ["codeql/something-else"] },
},
}, {
packs: {
cpp: ["codeql/something-else"],
javascript: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
},
});
(0, ava_1.default)("injected packs from input", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
packsInput: ["xxx", "yyy"],
}, {}, {
packs: ["xxx", "yyy"],
});
(0, ava_1.default)("injected packs from input with existing packs combines", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: true,
packsInput: ["xxx", "yyy"],
}, {
originalUserInput: {
packs: {
cpp: ["codeql/something-else"],
},
},
}, {
packs: {
cpp: ["codeql/something-else", "xxx", "yyy"],
},
});
(0, ava_1.default)("injected packs from input with existing packs overrides", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
packsInput: ["xxx", "yyy"],
}, {
originalUserInput: {
packs: {
cpp: ["codeql/something-else"],
},
},
}, {
packs: ["xxx", "yyy"],
});
(0, ava_1.default)("injected packs from input with existing packs overrides and ML model inject", injectedConfigMacro, {
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
packsInput: ["xxx", "yyy"],
}, {
originalUserInput: {
packs: {
cpp: ["codeql/something-else"],
},
},
}, {
packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.3.0"],
});
// similar, but with queries
(0, ava_1.default)("injected queries from input", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
}, {}, {
queries: [
{
uses: "xxx",
},
{
uses: "yyy",
},
],
});
(0, ava_1.default)("injected queries from input overrides", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
}, {
originalUserInput: {
queries: [{ uses: "zzz" }],
},
}, {
queries: [
{
uses: "xxx",
},
{
uses: "yyy",
},
],
});
(0, ava_1.default)("injected queries from input combines", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: true,
packsInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
}, {
originalUserInput: {
queries: [{ uses: "zzz" }],
},
}, {
queries: [
{
uses: "zzz",
},
{
uses: "xxx",
},
{
uses: "yyy",
},
],
});
(0, ava_1.default)("injected queries from input combines 2", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: true,
packsInputCombines: true,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
}, {}, {
queries: [
{
uses: "xxx",
},
{
uses: "yyy",
},
],
});
(0, ava_1.default)("injected queries and packs, but empty", injectedConfigMacro, {
injectedMlQueries: false,
queriesInputCombines: true,
packsInputCombines: true,
queriesInput: [],
packsInput: [],
}, {
originalUserInput: {
packs: [],
queries: [],
},
}, {});
(0, ava_1.default)("does not use injected confg", async (t) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
await codeqlObject.databaseInitCluster(stubConfig, "", undefined, undefined, (0, feature_flags_1.createFeatureFlags)([]));
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
t.falsy(configArg, "Should NOT have injected a codescanning config");
}
finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
});
function stubToolRunnerConstructor() {
const runnerObjectStub = sinon.createStubInstance(toolrunner.ToolRunner);
runnerObjectStub.exec.resolves(0);

File diff suppressed because one or more lines are too long

131
lib/config-utils.js generated
View file

@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const yaml = __importStar(require("js-yaml"));
@ -39,6 +39,17 @@ const QUERIES_USES_PROPERTY = "uses";
const PATHS_IGNORE_PROPERTY = "paths-ignore";
const PATHS_PROPERTY = "paths";
const PACKS_PROPERTY = "packs";
/**
* The default, empty augmentation properties. This is most useeful
* for tests.
*/
exports.defaultAugmentationProperties = {
queriesInputCombines: false,
packsInputCombines: false,
injectedMlQueries: false,
packsInput: undefined,
queriesInput: undefined,
};
/**
* A list of queries from https://github.com/github/codeql that
* we don't want to run. Disabling them here is a quicker alternative to
@ -461,8 +472,7 @@ function shouldAddConfigFileQueries(queriesInput) {
/**
* Get the default config for when the user has not supplied one.
*/
async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
var _a;
async function getDefaultConfig(languagesInput, rawQueriesInput, rawPacksInput, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
const languages = await getLanguages(codeQL, languagesInput, repository, apiDetails, logger);
const queries = {};
for (const language of languages) {
@ -472,10 +482,15 @@ async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLoca
};
}
await addDefaultQueries(codeQL, languages, queries);
const packs = (_a = parsePacksFromInput(packsInput, languages)) !== null && _a !== void 0 ? _a : {};
let injectedMlQueries = false;
if (queriesInput) {
injectedMlQueries = await addQueriesAndPacksFromWorkflow(codeQL, queriesInput, languages, queries, packs, tempDir, workspacePath, apiDetails, featureFlags, logger);
const augmentationProperties = calculateAugmentation(rawPacksInput, rawQueriesInput, languages);
const packs = augmentationProperties.packsInput
? {
[languages[0]]: augmentationProperties.packsInput,
}
: {};
if (rawQueriesInput) {
augmentationProperties.injectedMlQueries =
await addQueriesAndPacksFromWorkflow(codeQL, rawQueriesInput, languages, queries, packs, tempDir, workspacePath, apiDetails, featureFlags, logger);
}
return {
languages,
@ -491,7 +506,7 @@ async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLoca
debugMode,
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
augmentationProperties,
trapCaches: trapCachingEnabled
? await (0, trap_caching_1.downloadTrapCaches)(codeQL, languages, logger)
: {},
@ -501,7 +516,7 @@ exports.getDefaultConfig = getDefaultConfig;
/**
* Load the config from the given file.
*/
async function loadConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
async function loadConfig(languagesInput, rawQueriesInput, rawPacksInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
var _a;
let parsedYAML;
if (isLocal(configFile)) {
@ -542,16 +557,17 @@ async function loadConfig(languagesInput, queriesInput, packsInput, configFile,
if (!disableDefaultQueries) {
await addDefaultQueries(codeQL, languages, queries);
}
const packs = parsePacks((_a = parsedYAML[PACKS_PROPERTY]) !== null && _a !== void 0 ? _a : {}, packsInput, languages, configFile, logger);
const augmentationProperties = calculateAugmentation(rawPacksInput, rawQueriesInput, languages);
const packs = parsePacks((_a = parsedYAML[PACKS_PROPERTY]) !== null && _a !== void 0 ? _a : {}, rawPacksInput, augmentationProperties.packsInputCombines, languages, configFile, logger);
// If queries were provided using `with` in the action configuration,
// they should take precedence over the queries in the config file
// unless they're prefixed with "+", in which case they supplement those
// in the config file.
let injectedMlQueries = false;
if (queriesInput) {
injectedMlQueries = await addQueriesAndPacksFromWorkflow(codeQL, queriesInput, languages, queries, packs, tempDir, workspacePath, apiDetails, featureFlags, logger);
if (rawQueriesInput) {
augmentationProperties.injectedMlQueries =
await addQueriesAndPacksFromWorkflow(codeQL, rawQueriesInput, languages, queries, packs, tempDir, workspacePath, apiDetails, featureFlags, logger);
}
if (shouldAddConfigFileQueries(queriesInput) &&
if (shouldAddConfigFileQueries(rawQueriesInput) &&
QUERIES_PROPERTY in parsedYAML) {
const queriesArr = parsedYAML[QUERIES_PROPERTY];
if (!Array.isArray(queriesArr)) {
@ -601,12 +617,57 @@ async function loadConfig(languagesInput, queriesInput, packsInput, configFile,
debugMode,
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
augmentationProperties,
trapCaches: trapCachingEnabled
? await (0, trap_caching_1.downloadTrapCaches)(codeQL, languages, logger)
: {},
};
}
/**
* Calculates how the codeql config file needs to be augmented before passing
* it to the CLI. The reason this is necessary is the codeql-action can be called
* with extra inputs from the workflow. These inputs are not part of the config
* and the CLI does not know about these inputs so we need to inject them into
* the config file sent to the CLI.
*
* @param rawPacksInput The packs input from the action configuration.
* @param rawQueriesInput The queries input from the action configuration.
* @param languages The languages that the config file is for. If the packs input
* is non-empty, then there must be exactly one language. Otherwise, an
* error is thrown.
*
* @returns The properties that need to be augmented in the config file.
*
* @throws An error if the packs input is non-empty and the languages input does
* not have exactly one language.
*/
// exported for testing.
function calculateAugmentation(rawPacksInput, rawQueriesInput, languages) {
const packsInputCombines = shouldCombine(rawPacksInput);
const packsInput = parsePacksFromInput(rawPacksInput, languages, packsInputCombines);
const queriesInputCombines = shouldCombine(rawQueriesInput);
const queriesInput = parseQueriesFromInput(rawQueriesInput, queriesInputCombines);
return {
injectedMlQueries: false,
packsInputCombines,
packsInput: packsInput === null || packsInput === void 0 ? void 0 : packsInput[languages[0]],
queriesInput,
queriesInputCombines,
};
}
exports.calculateAugmentation = calculateAugmentation;
function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) {
if (!rawQueriesInput) {
return undefined;
}
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput === null || rawQueriesInput === void 0 ? void 0 : rawQueriesInput.trim();
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(getConfigFilePropertyError(undefined, "queries", "A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."));
}
return trimmedInput.split(",").map((query) => ({ uses: query.trim() }));
}
/**
* Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
* and `-` allowed as long as not the first or last char.
@ -653,8 +714,8 @@ function parsePacksFromConfig(packsByLanguage, languages, configFile, logger) {
return packs;
}
exports.parsePacksFromConfig = parsePacksFromConfig;
function parsePacksFromInput(packsInput, languages) {
if (!(packsInput === null || packsInput === void 0 ? void 0 : packsInput.trim())) {
function parsePacksFromInput(rawPacksInput, languages, packsInputCombines) {
if (!(rawPacksInput === null || rawPacksInput === void 0 ? void 0 : rawPacksInput.trim())) {
return undefined;
}
if (languages.length > 1) {
@ -663,16 +724,16 @@ function parsePacksFromInput(packsInput, languages) {
else if (languages.length === 0) {
throw new Error("No languages specified. Cannot process the packs input.");
}
packsInput = packsInput.trim();
if (packsInput.startsWith("+")) {
packsInput = packsInput.substring(1).trim();
if (!packsInput) {
throw new Error("A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs.");
rawPacksInput = rawPacksInput.trim();
if (packsInputCombines) {
rawPacksInput = rawPacksInput.trim().substring(1).trim();
if (!rawPacksInput) {
throw new Error(getConfigFilePropertyError(undefined, "packs", "A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."));
}
}
return {
[languages[0]]: packsInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack));
[languages[0]]: rawPacksInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack, ""));
return packs;
}, []),
};
@ -759,20 +820,32 @@ function validatePackSpecification(pack, configFile) {
}
exports.validatePackSpecification = validatePackSpecification;
// exported for testing
function parsePacks(rawPacksFromConfig, rawPacksInput, languages, configFile, logger) {
const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
function parsePacks(rawPacksFromConfig, rawPacksFromInput, packsInputCombines, languages, configFile, logger) {
const packsFomConfig = parsePacksFromConfig(rawPacksFromConfig, languages, configFile, logger);
const packsFromInput = parsePacksFromInput(rawPacksFromInput, languages, packsInputCombines);
if (!packsFromInput) {
return packsFomConfig;
}
if (!shouldCombinePacks(rawPacksInput)) {
if (!packsInputCombines) {
if (!packsFromInput) {
throw new Error(getPacksInvalid(configFile));
}
return packsFromInput;
}
return combinePacks(packsFromInput, packsFomConfig);
}
exports.parsePacks = parsePacks;
function shouldCombinePacks(packsInput) {
return !!(packsInput === null || packsInput === void 0 ? void 0 : packsInput.trim().startsWith("+"));
/**
* The convention in this action is that an input value that is prefixed with a '+' will
* be combined with the corresponding value in the config file.
*
* Without a '+', an input value will override the corresponding value in the config file.
*
* @param inputValue The input value to process.
* @returns true if the input value should replace the corresponding value in the config file, false if it should be appended.
*/
function shouldCombine(inputValue) {
return !!(inputValue === null || inputValue === void 0 ? void 0 : inputValue.trim().startsWith("+"));
}
function combinePacks(packs1, packs2) {
const packs = {};

File diff suppressed because one or more lines are too long

View file

@ -119,7 +119,9 @@ function mockListLanguages(languages) {
const config2 = await configUtils.getConfig(tmpDir, logger);
t.not(config2, undefined);
if (config2 !== undefined) {
t.deepEqual(config1, config2);
// removes properties assigned to undefined.
const expectedConfig = JSON.parse(JSON.stringify(config1));
t.deepEqual(expectedConfig, config2);
}
});
});
@ -220,7 +222,7 @@ function mockListLanguages(languages) {
debugMode: false,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
injectedMlQueries: false,
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
};
const languages = "javascript";
@ -878,7 +880,8 @@ const packSpecPrettyPrintingMacro = ava_1.default.macro({
* Test macro for testing the packs block and the packs input
*/
function parseInputAndConfigMacro(t, packsFromConfig, packsFromInput, languages, expected) {
t.deepEqual(configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b", mockLogger), expected);
t.deepEqual(configUtils.parsePacks(packsFromConfig, packsFromInput, !!(packsFromInput === null || packsFromInput === void 0 ? void 0 : packsFromInput.trim().startsWith("+")), // coerce to boolean
languages, "/a/b", mockLogger), expected);
}
parseInputAndConfigMacro.title = (providedTitle) => `Parse Packs input and config: ${providedTitle}`;
const mockLogger = {
@ -886,9 +889,9 @@ const mockLogger = {
console.log(message);
},
};
function parseInputAndConfigErrorMacro(t, packsFromConfig, packsFromInput, languages, expected) {
function parseInputAndConfigErrorMacro(t, packsFromConfig, packsFromInput, languages, packsFromInputOverride, expected) {
t.throws(() => {
configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b", mockLogger);
configUtils.parsePacks(packsFromConfig, packsFromInput, packsFromInputOverride, languages, "/a/b", mockLogger);
}, {
message: expected,
});
@ -912,10 +915,10 @@ parseInputAndConfigErrorMacro.title = (providedTitle) => `Parse Packs input and
(0, ava_1.default)("input and config", parseInputAndConfigMacro, ["a/b", "c/d"], " +e/f, g/h@1.2.3 ", [languages_1.Language.cpp], {
[languages_1.Language.cpp]: ["e/f", "g/h@1.2.3", "a/b", "c/d"],
});
(0, ava_1.default)("input with no language", parseInputAndConfigErrorMacro, {}, "c/d", [], /No languages specified/);
(0, ava_1.default)("input with two languages", parseInputAndConfigErrorMacro, {}, "c/d", [languages_1.Language.cpp, languages_1.Language.csharp], /multi-language analysis/);
(0, ava_1.default)("input with + only", parseInputAndConfigErrorMacro, {}, " + ", [languages_1.Language.cpp], /remove the '\+'/);
(0, ava_1.default)("input with invalid pack name", parseInputAndConfigErrorMacro, {}, " xxx", [languages_1.Language.cpp], /"xxx" is not a valid pack/);
(0, ava_1.default)("input with no language", parseInputAndConfigErrorMacro, {}, "c/d", [], false, /No languages specified/);
(0, ava_1.default)("input with two languages", parseInputAndConfigErrorMacro, {}, "c/d", [languages_1.Language.cpp, languages_1.Language.csharp], false, /multi-language analysis/);
(0, ava_1.default)("input with + only", parseInputAndConfigErrorMacro, {}, " + ", [languages_1.Language.cpp], true, /remove the '\+'/);
(0, ava_1.default)("input with invalid pack name", parseInputAndConfigErrorMacro, {}, " xxx", [languages_1.Language.cpp], false, /"xxx" is not a valid pack/);
const mlPoweredQueriesMacro = ava_1.default.macro({
exec: async (t, codeQLVersion, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, expectedVersionString) => {
return await util.withTmpDir(async (tmpDir) => {
@ -982,4 +985,57 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.9.3+.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.9.3", true, undefined, "security-and-quality", "~0.3.0");
const calculateAugmentationMacro = ava_1.default.macro({
exec: async (t, _title, rawPacksInput, rawQueriesInput, languages, expectedAugmentationProperties) => {
const actualAugmentationProperties = configUtils.calculateAugmentation(rawPacksInput, rawQueriesInput, languages);
t.deepEqual(actualAugmentationProperties, expectedAugmentationProperties);
},
title: (_, title) => `Calculate Augmentation: ${title}`,
});
(0, ava_1.default)(calculateAugmentationMacro, "All empty", undefined, undefined, [languages_1.Language.javascript], {
queriesInputCombines: false,
queriesInput: undefined,
packsInputCombines: false,
packsInput: undefined,
injectedMlQueries: false,
});
(0, ava_1.default)(calculateAugmentationMacro, "With queries", undefined, " a, b , c, d", [languages_1.Language.javascript], {
queriesInputCombines: false,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
packsInputCombines: false,
packsInput: undefined,
injectedMlQueries: false,
});
(0, ava_1.default)(calculateAugmentationMacro, "With queries combining", undefined, " + a, b , c, d ", [languages_1.Language.javascript], {
queriesInputCombines: true,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
packsInputCombines: false,
packsInput: undefined,
injectedMlQueries: false,
});
(0, ava_1.default)(calculateAugmentationMacro, "With packs", " codeql/a , codeql/b , codeql/c , codeql/d ", undefined, [languages_1.Language.javascript], {
queriesInputCombines: false,
queriesInput: undefined,
packsInputCombines: false,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
injectedMlQueries: false,
});
(0, ava_1.default)(calculateAugmentationMacro, "With packs combining", " + codeql/a, codeql/b, codeql/c, codeql/d", undefined, [languages_1.Language.javascript], {
queriesInputCombines: false,
queriesInput: undefined,
packsInputCombines: true,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
injectedMlQueries: false,
});
const calculateAugmentationErrorMacro = ava_1.default.macro({
exec: async (t, _title, rawPacksInput, rawQueriesInput, languages, expectedError) => {
t.throws(() => configUtils.calculateAugmentation(rawPacksInput, rawQueriesInput, languages), { message: expectedError });
},
title: (_, title) => `Calculate Augmentation Error: ${title}`,
});
(0, ava_1.default)(calculateAugmentationErrorMacro, "Plus (+) with nothing else (queries)", undefined, " + ", [languages_1.Language.javascript], /The workflow property "queries" is invalid/);
(0, ava_1.default)(calculateAugmentationErrorMacro, "Plus (+) with nothing else (packs)", " + ", undefined, [languages_1.Language.javascript], /The workflow property "packs" is invalid/);
(0, ava_1.default)(calculateAugmentationErrorMacro, "Packs input with multiple languages", " + a/b, c/d ", undefined, [languages_1.Language.javascript, languages_1.Language.java], /Cannot specify a 'packs' input in a multi-language analysis/);
(0, ava_1.default)(calculateAugmentationErrorMacro, "Packs input with no languages", " + a/b, c/d ", undefined, [], /No languages specified/);
(0, ava_1.default)(calculateAugmentationErrorMacro, "Invalid packs", " a-pack-without-a-scope ", undefined, [languages_1.Language.javascript], /"a-pack-without-a-scope" is not a valid pack/);
//# sourceMappingURL=config-utils.test.js.map

File diff suppressed because one or more lines are too long

View file

@ -29,6 +29,7 @@ const sinon = __importStar(require("sinon"));
const actionsUtil = __importStar(require("./actions-util"));
const apiClient = __importStar(require("./api-client"));
const codeql_1 = require("./codeql");
const config_utils_1 = require("./config-utils");
const database_upload_1 = require("./database-upload");
const languages_1 = require("./languages");
const testing_utils_1 = require("./testing-utils");
@ -58,7 +59,7 @@ function getTestConfig(tmpDir) {
debugMode: false,
debugArtifactName: util_1.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util_1.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: config_utils_1.defaultAugmentationProperties,
trapCaches: {},
};
}

File diff suppressed because one or more lines are too long

View file

@ -26,6 +26,7 @@ const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const ava_1 = __importDefault(require("ava"));
const codeql_1 = require("./codeql");
const configUtils = __importStar(require("./config-utils"));
const languages_1 = require("./languages");
const testing_utils_1 = require("./testing-utils");
const tracer_config_1 = require("./tracer-config");
@ -46,7 +47,7 @@ function getTestConfig(tmpDir) {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
};
}

File diff suppressed because one or more lines are too long

View file

@ -90,7 +90,11 @@ const testConfigWithoutTmpDir = {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {
javascript: "/some/cache/dir",
},
@ -110,7 +114,11 @@ function getTestConfigWithTempDir(tmpDir) {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {
javascript: path.resolve(tmpDir, "jsCache"),
ruby: path.resolve(tmpDir, "rubyCache"),

File diff suppressed because one or more lines are too long

17
lib/util.js generated
View file

@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.listFolder = exports.doesDirectoryExist = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@ -416,6 +416,12 @@ var EnvVar;
* own sandwiched workflow mechanism
*/
EnvVar["FEATURE_SANDWICH"] = "CODEQL_ACTION_FEATURE_SANDWICH";
/**
* If set to the "true" string and the codeql CLI version is greater than
* `CODEQL_VERSION_CONFIG_FILES`, then the codeql-action will pass the
* the codeql-config file to the codeql CLI to be processed there.
*/
EnvVar["CODEQL_PASS_CONFIG_TO_CLI"] = "CODEQL_PASS_CONFIG_TO_CLI";
})(EnvVar || (EnvVar = {}));
const exportVar = (mode, name, value) => {
if (mode === Mode.actions) {
@ -641,6 +647,15 @@ function isInTestMode() {
return process.env["TEST_MODE"] === "true" || false;
}
exports.isInTestMode = isInTestMode;
/**
* @returns true if the action should generate a conde-scanning config file
* that gets passed to the CLI.
*/
async function useCodeScanningConfigInCli(codeql) {
return (process.env[EnvVar.CODEQL_PASS_CONFIG_TO_CLI] === "true" &&
(await codeQlVersionAbove(codeql, codeql_1.CODEQL_VERSION_CONFIG_FILES)));
}
exports.useCodeScanningConfigInCli = useCodeScanningConfigInCli;
/*
* Returns whether the path in the argument represents an existing directory.
*/

File diff suppressed because one or more lines are too long

6
lib/util.test.js generated
View file

@ -261,7 +261,11 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);

File diff suppressed because one or more lines are too long

12
node_modules/.package-lock.json generated vendored
View file

@ -744,10 +744,14 @@
"dev": true
},
"node_modules/@types/long": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz",
"integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==",
"dev": true
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/@types/long/-/long-5.0.0.tgz",
"integrity": "sha512-eQs9RsucA/LNjnMoJvWG/nXa7Pot/RbBzilF/QRIU/xRl+0ApxrSUFsV5lmf01SvSlqMzJ7Zwxe440wmz2SJGA==",
"deprecated": "This is a stub types definition. long provides its own type definitions, so you do not need this installed.",
"dev": true,
"dependencies": {
"long": "*"
}
},
"node_modules/@types/node": {
"version": "16.11.22",

42
node_modules/@types/long/LICENSE generated vendored Normal file → Executable file
View file

@ -1,21 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE
MIT License
Copyright (c) Microsoft Corporation.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

19
node_modules/@types/long/README.md generated vendored Normal file → Executable file
View file

@ -1,16 +1,3 @@
# Installation
> `npm install --save @types/long`
# Summary
This package contains type definitions for long.js (https://github.com/dcodeIO/long.js).
# Details
Files were exported from https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/long.
### Additional Details
* Last updated: Wed, 22 Jan 2020 19:19:46 GMT
* Dependencies: none
* Global values: `Long`
# Credits
These definitions were written by Peter Kooijmans (https://github.com/peterkooijmans).
This is a stub types definition for @types/long (https://github.com/dcodeIO/long.js#readme).
long provides its own type definitions, so you don't need @types/long installed!

389
node_modules/@types/long/index.d.ts generated vendored
View file

@ -1,389 +0,0 @@
// Type definitions for long.js 4.0.0
// Project: https://github.com/dcodeIO/long.js
// Definitions by: Peter Kooijmans <https://github.com/peterkooijmans>
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
// Definitions by: Denis Cappellin <https://github.com/cappellin>
export = Long;
export as namespace Long;
declare const Long: Long.LongConstructor;
type Long = Long.Long;
declare namespace Long {
interface LongConstructor {
/**
* Constructs a 64 bit two's-complement integer, given its low and high 32 bit values as signed integers. See the from* functions below for more convenient ways of constructing Longs.
*/
new( low: number, high?: number, unsigned?: boolean ): Long;
prototype: Long;
/**
* Maximum unsigned value.
*/
MAX_UNSIGNED_VALUE: Long;
/**
* Maximum signed value.
*/
MAX_VALUE: Long;
/**
* Minimum signed value.
*/
MIN_VALUE: Long;
/**
* Signed negative one.
*/
NEG_ONE: Long;
/**
* Signed one.
*/
ONE: Long;
/**
* Unsigned one.
*/
UONE: Long;
/**
* Unsigned zero.
*/
UZERO: Long;
/**
* Signed zero
*/
ZERO: Long;
/**
* Returns a Long representing the 64 bit integer that comes by concatenating the given low and high bits. Each is assumed to use 32 bits.
*/
fromBits( lowBits:number, highBits:number, unsigned?:boolean ): Long;
/**
* Returns a Long representing the given 32 bit integer value.
*/
fromInt( value: number, unsigned?: boolean ): Long;
/**
* Returns a Long representing the given value, provided that it is a finite number. Otherwise, zero is returned.
*/
fromNumber( value: number, unsigned?: boolean ): Long;
/**
* Returns a Long representation of the given string, written using the specified radix.
*/
fromString( str: string, unsigned?: boolean | number, radix?: number ): Long;
/**
* Creates a Long from its byte representation.
*/
fromBytes( bytes: number[], unsigned?: boolean, le?: boolean ): Long;
/**
* Creates a Long from its little endian byte representation.
*/
fromBytesLE( bytes: number[], unsigned?: boolean ): Long;
/**
* Creates a Long from its little endian byte representation.
*/
fromBytesBE( bytes: number[], unsigned?: boolean ): Long;
/**
* Tests if the specified object is a Long.
*/
isLong( obj: any ): obj is Long;
/**
* Converts the specified value to a Long.
*/
fromValue( val: Long | number | string | {low: number, high: number, unsigned: boolean} ): Long;
}
interface Long
{
/**
* The high 32 bits as a signed value.
*/
high: number;
/**
* The low 32 bits as a signed value.
*/
low: number;
/**
* Whether unsigned or not.
*/
unsigned: boolean;
/**
* Returns the sum of this and the specified Long.
*/
add( addend: number | Long | string ): Long;
/**
* Returns the bitwise AND of this Long and the specified.
*/
and( other: Long | number | string ): Long;
/**
* Compares this Long's value with the specified's.
*/
compare( other: Long | number | string ): number;
/**
* Compares this Long's value with the specified's.
*/
comp( other: Long | number | string ): number;
/**
* Returns this Long divided by the specified.
*/
divide( divisor: Long | number | string ): Long;
/**
* Returns this Long divided by the specified.
*/
div( divisor: Long | number | string ): Long;
/**
* Tests if this Long's value equals the specified's.
*/
equals( other: Long | number | string ): boolean;
/**
* Tests if this Long's value equals the specified's.
*/
eq( other: Long | number | string ): boolean;
/**
* Gets the high 32 bits as a signed integer.
*/
getHighBits(): number;
/**
* Gets the high 32 bits as an unsigned integer.
*/
getHighBitsUnsigned(): number;
/**
* Gets the low 32 bits as a signed integer.
*/
getLowBits(): number;
/**
* Gets the low 32 bits as an unsigned integer.
*/
getLowBitsUnsigned(): number;
/**
* Gets the number of bits needed to represent the absolute value of this Long.
*/
getNumBitsAbs(): number;
/**
* Tests if this Long's value is greater than the specified's.
*/
greaterThan( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is greater than the specified's.
*/
gt( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is greater than or equal the specified's.
*/
greaterThanOrEqual( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is greater than or equal the specified's.
*/
gte( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is even.
*/
isEven(): boolean;
/**
* Tests if this Long's value is negative.
*/
isNegative(): boolean;
/**
* Tests if this Long's value is odd.
*/
isOdd(): boolean;
/**
* Tests if this Long's value is positive.
*/
isPositive(): boolean;
/**
* Tests if this Long's value equals zero.
*/
isZero(): boolean;
/**
* Tests if this Long's value is less than the specified's.
*/
lessThan( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is less than the specified's.
*/
lt( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is less than or equal the specified's.
*/
lessThanOrEqual( other: Long | number | string ): boolean;
/**
* Tests if this Long's value is less than or equal the specified's.
*/
lte( other: Long | number | string ): boolean;
/**
* Returns this Long modulo the specified.
*/
modulo( other: Long | number | string ): Long;
/**
* Returns this Long modulo the specified.
*/
mod( other: Long | number | string ): Long;
/**
* Returns the product of this and the specified Long.
*/
multiply( multiplier: Long | number | string ): Long;
/**
* Returns the product of this and the specified Long.
*/
mul( multiplier: Long | number | string ): Long;
/**
* Negates this Long's value.
*/
negate(): Long;
/**
* Negates this Long's value.
*/
neg(): Long;
/**
* Returns the bitwise NOT of this Long.
*/
not(): Long;
/**
* Tests if this Long's value differs from the specified's.
*/
notEquals( other: Long | number | string ): boolean;
/**
* Tests if this Long's value differs from the specified's.
*/
neq( other: Long | number | string ): boolean;
/**
* Returns the bitwise OR of this Long and the specified.
*/
or( other: Long | number | string ): Long;
/**
* Returns this Long with bits shifted to the left by the given amount.
*/
shiftLeft( numBits: number | Long ): Long;
/**
* Returns this Long with bits shifted to the left by the given amount.
*/
shl( numBits: number | Long ): Long;
/**
* Returns this Long with bits arithmetically shifted to the right by the given amount.
*/
shiftRight( numBits: number | Long ): Long;
/**
* Returns this Long with bits arithmetically shifted to the right by the given amount.
*/
shr( numBits: number | Long ): Long;
/**
* Returns this Long with bits logically shifted to the right by the given amount.
*/
shiftRightUnsigned( numBits: number | Long ): Long;
/**
* Returns this Long with bits logically shifted to the right by the given amount.
*/
shru( numBits: number | Long ): Long;
/**
* Returns the difference of this and the specified Long.
*/
subtract( subtrahend: number | Long | string ): Long;
/**
* Returns the difference of this and the specified Long.
*/
sub( subtrahend: number | Long |string ): Long;
/**
* Converts the Long to a 32 bit integer, assuming it is a 32 bit integer.
*/
toInt(): number;
/**
* Converts the Long to a the nearest floating-point representation of this value (double, 53 bit mantissa).
*/
toNumber(): number;
/**
* Converts this Long to its byte representation.
*/
toBytes( le?: boolean ): number[];
/**
* Converts this Long to its little endian byte representation.
*/
toBytesLE(): number[];
/**
* Converts this Long to its big endian byte representation.
*/
toBytesBE(): number[];
/**
* Converts this Long to signed.
*/
toSigned(): Long;
/**
* Converts the Long to a string written in the specified radix.
*/
toString( radix?: number ): string;
/**
* Converts this Long to unsigned.
*/
toUnsigned(): Long;
/**
* Returns the bitwise XOR of this Long and the given one.
*/
xor( other: Long | number | string ): Long;
}
}

26
node_modules/@types/long/package.json generated vendored Normal file → Executable file
View file

@ -1,24 +1,12 @@
{
"name": "@types/long",
"version": "4.0.1",
"description": "TypeScript definitions for long.js",
"license": "MIT",
"contributors": [
{
"name": "Peter Kooijmans",
"url": "https://github.com/peterkooijmans",
"githubUsername": "peterkooijmans"
}
],
"version": "5.0.0",
"description": "Stub TypeScript definitions entry for long, which provides its own types definitions",
"main": "",
"types": "index.d.ts",
"repository": {
"type": "git",
"url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git",
"directory": "types/long"
},
"scripts": {},
"dependencies": {},
"typesPublisherContentHash": "5a2ae1424989c49d7303e1f5cc510288bfab1e71e0e2143cdcb9d24ff1c3dc8e",
"typeScriptVersion": "2.8"
"license": "MIT",
"dependencies": {
"long": "*"
},
"deprecated": "This is a stub types definition. long provides its own type definitions, so you do not need this installed."
}

25
package-lock.json generated
View file

@ -42,7 +42,7 @@
"@ava/typescript": "3.0.1",
"@types/adm-zip": "^0.5.0",
"@types/js-yaml": "^4.0.5",
"@types/long": "4.0.1",
"@types/long": "5.0.0",
"@types/node": "16.11.22",
"@types/semver": "^7.3.8",
"@types/sinon": "^10.0.2",
@ -800,10 +800,14 @@
"dev": true
},
"node_modules/@types/long": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz",
"integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==",
"dev": true
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/@types/long/-/long-5.0.0.tgz",
"integrity": "sha512-eQs9RsucA/LNjnMoJvWG/nXa7Pot/RbBzilF/QRIU/xRl+0ApxrSUFsV5lmf01SvSlqMzJ7Zwxe440wmz2SJGA==",
"deprecated": "This is a stub types definition. long provides its own type definitions, so you do not need this installed.",
"dev": true,
"dependencies": {
"long": "*"
}
},
"node_modules/@types/node": {
"version": "16.11.22",
@ -6575,10 +6579,13 @@
"dev": true
},
"@types/long": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz",
"integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==",
"dev": true
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/@types/long/-/long-5.0.0.tgz",
"integrity": "sha512-eQs9RsucA/LNjnMoJvWG/nXa7Pot/RbBzilF/QRIU/xRl+0ApxrSUFsV5lmf01SvSlqMzJ7Zwxe440wmz2SJGA==",
"dev": true,
"requires": {
"long": "*"
}
},
"@types/node": {
"version": "16.11.22",

View file

@ -57,7 +57,7 @@
"@ava/typescript": "3.0.1",
"@types/adm-zip": "^0.5.0",
"@types/js-yaml": "^4.0.5",
"@types/long": "4.0.1",
"@types/long": "5.0.0",
"@types/node": "16.11.22",
"@types/semver": "^7.3.8",
"@types/sinon": "^10.0.2",

View file

@ -31,6 +31,14 @@ steps:
path: "${{ runner.temp }}/results/javascript.sarif"
retention-days: 7
- name: Check sarif
uses: ./../action/.github/check-sarif
if: matrix.os != 'windows-latest' || matrix.version == 'latest' || matrix.version == 'nightly-latest'
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: js/ml-powered/nosql-injection,js/ml-powered/path-injection,js/ml-powered/sql-injection,js/ml-powered/xss
queries-not-run: foo,bar
- name: Check results
# Running ML-powered queries on Windows requires CodeQL CLI 2.9.0+. We don't run these checks
# against Windows and `cached` while CodeQL CLI 2.9.0 makes its way into `cached` to avoid the

View file

@ -0,0 +1,44 @@
name: "Packaging: Config and input passed to the CLI"
description: "Checks that specifying packages using a combination of a config file and input to the Action works"
versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
env:
CODEQL_PASS_CONFIG_TO_CLI: true
steps:
- uses: ./../action/init
with:
config-file: ".github/codeql/codeql-config-packaging3.yml"
packs: +dsp-testing/codeql-pack1@1.0.0
languages: javascript
tools: ${{ steps.prepare-test.outputs.tools-url }}
- name: Build code
shell: bash
run: ./build.sh
- uses: ./../action/analyze
with:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should have 4 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."
exit 1
fi

View file

@ -16,6 +16,14 @@ steps:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |

View file

@ -15,6 +15,14 @@ steps:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |

View file

@ -16,6 +16,14 @@ steps:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Check results
uses: ./../action/.github/check-sarif
with:
sarif-file: ${{ runner.temp }}/results/javascript.sarif
queries-run: javascript/example/empty-or-one-block,javascript/example/empty-or-one-block,javascript/example/other-query-block,javascript/example/two-block
queries-not-run: foo,bar
- name: Assert Results
shell: bash
run: |

View file

@ -18,6 +18,7 @@ steps:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Assert No Results
shell: bash
run: |

View file

@ -24,7 +24,11 @@ test("emptyPaths", async (t) => {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
@ -50,7 +54,11 @@ test("nonEmptyPaths", async (t) => {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
@ -79,7 +87,11 @@ test("exclude temp dir", async (t) => {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);

View file

@ -118,7 +118,11 @@ test("status report fields and search path setting", async (t) => {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
@ -432,7 +436,11 @@ const stubConfig: Config = {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};

View file

@ -221,6 +221,7 @@ export async function runQueries(
);
}
const codeql = await getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
const queries = config.queries[language];
const queryFilters = validateQueryFilters(
@ -238,85 +239,122 @@ export async function runQueries(
);
}
const codeql = await getCodeQL(config.codeQLCmd);
try {
if (hasPackWithCustomQueries) {
logger.info("Performing analysis with custom CodeQL Packs.");
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {
if (await util.useCodeScanningConfigInCli(codeql)) {
// If we are using the codescanning config in the CLI,
// much of the work needed to generate the query suites
// is done in the CLI. We just need to make a single
// call to run all the queries for each language and
// another to interpret the results.
logger.startGroup(`Running queries for ${language}`);
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push(
await runQueryGroup(
language,
"builtin",
createQuerySuiteContents(queries["builtin"], queryFilters),
undefined
)
);
await runQueryGroup(language, "all", undefined, undefined);
// TODO should not be using `builtin` here. We should be using `all` instead.
// The status report does not support `all` yet.
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(
language,
undefined,
sarifFile,
config.debugMode
);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
} else {
if (hasPackWithCustomQueries) {
logger.info("Performing analysis with custom CodeQL Packs.");
logger.startGroup(`Downloading custom packs for ${language}`);
const results = await codeql.packDownload(packsWithVersion);
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
querySuitePaths.push(
await runQueryGroup(
(await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(
queries["custom"][i].queries,
queryFilters
),
queries["custom"][i].searchPath
"builtin",
createQuerySuiteContents(queries["builtin"], queryFilters),
undefined
)) as string
);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
querySuitePaths.push(
(await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(
queries["custom"][i].queries,
queryFilters
),
queries["custom"][i].searchPath
)) as string
);
ranCustom = true;
}
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(
await runQueryPacks(
language,
"packs",
packsWithVersion,
queryFilters
)
);
ranCustom = true;
}
}
if (packsWithVersion.length > 0) {
querySuitePaths.push(
await runQueryPacks(language, "packs", packsWithVersion, queryFilters)
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(
language,
querySuitePaths,
sarifFile,
config.debugMode
);
ranCustom = true;
if (!cliCanCountBaseline) {
await injectLinesOfCode(sarifFile, language, locPromise);
}
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
}
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(
language,
querySuitePaths,
sarifFile,
config.debugMode
);
if (!cliCanCountBaseline)
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
if (!cliCanCountBaseline || countLocDebugMode)
if (!cliCanCountBaseline || countLocDebugMode) {
printLinesOfCodeSummary(logger, language, await locPromise);
if (cliCanCountBaseline) logger.info(await runPrintLinesOfCode(language));
}
if (cliCanCountBaseline) {
logger.info(await runPrintLinesOfCode(language));
}
} catch (e) {
logger.info(String(e));
if (e instanceof Error) {
@ -334,12 +372,11 @@ export async function runQueries(
async function runInterpretResults(
language: Language,
queries: string[],
queries: string[] | undefined,
sarifFile: string,
enableDebugLogging: boolean
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = await getCodeQL(config.codeQLCmd);
return await codeql.databaseInterpretResults(
databasePath,
queries,
@ -360,26 +397,27 @@ export async function runQueries(
async function runPrintLinesOfCode(language: Language): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = await getCodeQL(config.codeQLCmd);
return await codeql.databasePrintBaseline(databasePath);
}
async function runQueryGroup(
language: Language,
type: string,
querySuiteContents: string,
querySuiteContents: string | undefined,
searchPath: string | undefined
): Promise<string> {
): Promise<string | undefined> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(
`Query suite file for ${language}-${type}...\n${querySuiteContents}`
);
const codeql = await getCodeQL(config.codeQLCmd);
const querySuitePath = querySuiteContents
? `${databasePath}-queries-${type}.qls`
: undefined;
if (querySuiteContents && querySuitePath) {
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(
`Query suite file for ${language}-${type}...\n${querySuiteContents}`
);
}
await codeql.databaseRunQueries(
databasePath,
searchPath,
@ -413,7 +451,6 @@ export async function runQueries(
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
const codeql = await getCodeQL(config.codeQLCmd);
await codeql.databaseRunQueries(
databasePath,
undefined,

View file

@ -1,13 +1,16 @@
import * as fs from "fs";
import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
import test from "ava";
import test, { ExecutionContext } from "ava";
import del from "del";
import * as yaml from "js-yaml";
import nock from "nock";
import * as sinon from "sinon";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
import { createFeatureFlags, FeatureFlag } from "./feature-flags";
import { Language } from "./languages";
@ -30,8 +33,34 @@ const sampleGHAEApiDetails = {
apiURL: undefined,
};
let stubConfig: Config;
test.beforeEach(() => {
initializeEnvironment(Mode.actions, "1.2.3");
stubConfig = {
languages: [Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};
});
test("download codeql bundle cache", async (t) => {
@ -420,26 +449,6 @@ test("databaseInterpretResults() sets --sarif-add-query-help for 2.7.1", async (
);
});
const stubConfig: Config = {
languages: [Language.cpp],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
trapCaches: {},
};
test("databaseInitCluster() Lua feature flag enabled, but old CLI", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
@ -532,6 +541,392 @@ test("databaseInitCluster() Lua feature flag disabled, compatible CLI", async (t
);
});
test("databaseInitCluster() without injected codescanning config", async (t) => {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves("2.8.1");
const thisStubConfig: Config = {
...stubConfig,
tempDir,
augmentationProperties: {
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
},
};
await codeqlObject.databaseInitCluster(
thisStubConfig,
"",
undefined,
undefined,
createFeatureFlags([])
);
const args = runnerConstructorStub.firstCall.args[1];
// should NOT have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should have injected a codescanning config");
});
});
// Test macro for ensuring different variants of injected augmented configurations
const injectedConfigMacro = test.macro({
exec: async (
t: ExecutionContext<unknown>,
augmentationProperties: AugmentationProperties,
configOverride: Partial<Config>,
expectedConfig: any
) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "true";
try {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
const thisStubConfig: Config = {
...stubConfig,
...configOverride,
tempDir,
augmentationProperties,
};
await codeqlObject.databaseInitCluster(
thisStubConfig,
"",
undefined,
undefined,
createFeatureFlags([])
);
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.truthy(configArg, "Should have injected a codescanning config");
const configFile = configArg.split("=")[1];
const augmentedConfig = yaml.load(fs.readFileSync(configFile, "utf8"));
t.deepEqual(augmentedConfig, expectedConfig);
await del(configFile, { force: true });
});
} finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
},
title: (providedTitle = "") =>
`databaseInitCluster() injected config: ${providedTitle}`,
});
test(
"basic",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
},
{},
{}
);
test(
"injected ML queries",
injectedConfigMacro,
{
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
},
{},
{
packs: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
}
);
test(
"injected ML queries with existing packs",
injectedConfigMacro,
{
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
},
{
originalUserInput: {
packs: { javascript: ["codeql/something-else"] },
},
},
{
packs: {
javascript: [
"codeql/something-else",
"codeql/javascript-experimental-atm-queries@~0.3.0",
],
},
}
);
test(
"injected ML queries with existing packs of different language",
injectedConfigMacro,
{
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
},
{
originalUserInput: {
packs: { cpp: ["codeql/something-else"] },
},
},
{
packs: {
cpp: ["codeql/something-else"],
javascript: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
},
}
);
test(
"injected packs from input",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
packsInput: ["xxx", "yyy"],
},
{},
{
packs: ["xxx", "yyy"],
}
);
test(
"injected packs from input with existing packs combines",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: true,
packsInput: ["xxx", "yyy"],
},
{
originalUserInput: {
packs: {
cpp: ["codeql/something-else"],
},
},
},
{
packs: {
cpp: ["codeql/something-else", "xxx", "yyy"],
},
}
);
test(
"injected packs from input with existing packs overrides",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
packsInput: ["xxx", "yyy"],
},
{
originalUserInput: {
packs: {
cpp: ["codeql/something-else"],
},
},
},
{
packs: ["xxx", "yyy"],
}
);
test(
"injected packs from input with existing packs overrides and ML model inject",
injectedConfigMacro,
{
injectedMlQueries: true,
queriesInputCombines: false,
packsInputCombines: false,
packsInput: ["xxx", "yyy"],
},
{
originalUserInput: {
packs: {
cpp: ["codeql/something-else"],
},
},
},
{
packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.3.0"],
}
);
// similar, but with queries
test(
"injected queries from input",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
},
{},
{
queries: [
{
uses: "xxx",
},
{
uses: "yyy",
},
],
}
);
test(
"injected queries from input overrides",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: false,
packsInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
},
{
originalUserInput: {
queries: [{ uses: "zzz" }],
},
},
{
queries: [
{
uses: "xxx",
},
{
uses: "yyy",
},
],
}
);
test(
"injected queries from input combines",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: true,
packsInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
},
{
originalUserInput: {
queries: [{ uses: "zzz" }],
},
},
{
queries: [
{
uses: "zzz",
},
{
uses: "xxx",
},
{
uses: "yyy",
},
],
}
);
test(
"injected queries from input combines 2",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: true,
packsInputCombines: true,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
},
{},
{
queries: [
{
uses: "xxx",
},
{
uses: "yyy",
},
],
}
);
test(
"injected queries and packs, but empty",
injectedConfigMacro,
{
injectedMlQueries: false,
queriesInputCombines: true,
packsInputCombines: true,
queriesInput: [],
packsInput: [],
},
{
originalUserInput: {
packs: [],
queries: [],
},
},
{}
);
test("does not use injected confg", async (t: ExecutionContext<unknown>) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
try {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
await codeqlObject.databaseInitCluster(
stubConfig,
"",
undefined,
undefined,
createFeatureFlags([])
);
const args = runnerConstructorStub.firstCall.args[1];
// should have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.falsy(configArg, "Should NOT have injected a codescanning config");
} finally {
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
}
});
export function stubToolRunnerConstructor(): sinon.SinonStub<
any[],
toolrunner.ToolRunner

View file

@ -5,6 +5,7 @@ import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as yaml from "js-yaml";
import { default as queryString } from "query-string";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
@ -152,7 +153,7 @@ export interface CodeQL {
databaseRunQueries(
databasePath: string,
extraSearchPath: string | undefined,
querySuitePath: string,
querySuitePath: string | undefined,
memoryFlag: string,
threadsFlag: string
): Promise<void>;
@ -161,7 +162,7 @@ export interface CodeQL {
*/
databaseInterpretResults(
databasePath: string,
querySuitePaths: string[],
querySuitePaths: string[] | undefined,
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
@ -248,6 +249,7 @@ export const CODEQL_VERSION_COUNTS_LINES = "2.6.2";
const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
export const CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
export const CODEQL_VERSION_CONFIG_FILES = "2.10.1";
/**
* This variable controls using the new style of tracing from the CodeQL
@ -778,6 +780,12 @@ async function getCodeQLForCmd(
}
}
}
const configLocation = await generateCodescanningConfig(codeql, config);
if (configLocation) {
extraArgs.push(`--codescanning-config=${configLocation}`);
}
await runTool(cmd, [
"database",
"init",
@ -967,7 +975,7 @@ async function getCodeQLForCmd(
async databaseRunQueries(
databasePath: string,
extraSearchPath: string | undefined,
querySuitePath: string,
querySuitePath: string | undefined,
memoryFlag: string,
threadsFlag: string
): Promise<void> {
@ -984,12 +992,14 @@ async function getCodeQLForCmd(
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
codeqlArgs.push(querySuitePath);
if (querySuitePath) {
codeqlArgs.push(querySuitePath);
}
await runTool(cmd, codeqlArgs);
},
async databaseInterpretResults(
databasePath: string,
querySuitePaths: string[],
querySuitePaths: string[] | undefined,
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
@ -1021,7 +1031,9 @@ async function getCodeQLForCmd(
codeqlArgs.push("--sarif-category", automationDetailsId);
}
codeqlArgs.push(databasePath);
codeqlArgs.push(...querySuitePaths);
if (querySuitePaths) {
codeqlArgs.push(...querySuitePaths);
}
// capture stdout, which contains analysis summaries
return await runTool(cmd, codeqlArgs);
},
@ -1213,3 +1225,84 @@ async function runTool(cmd: string, args: string[] = []) {
throw new CommandInvocationError(cmd, args, exitCode, error);
return output;
}
/**
* If appropriate, generates a code scanning configuration that is to be used for a scan.
* If the configuration is not to be generated, returns undefined.
*
* @param codeql The CodeQL object to use.
* @param config The configuration to use.
* @returns the path to the generated user configuration file.
*/
async function generateCodescanningConfig(
codeql: CodeQL,
config: Config
): Promise<string | undefined> {
if (!(await util.useCodeScanningConfigInCli(codeql))) {
return;
}
const configLocation = path.resolve(config.tempDir, "user-config.yaml");
// make a copy so we can modify it
const augmentedConfig = cloneObject(config.originalUserInput);
// Inject the queries from the input
if (config.augmentationProperties.queriesInput) {
if (config.augmentationProperties.queriesInputCombines) {
augmentedConfig.queries = (augmentedConfig.queries || []).concat(
config.augmentationProperties.queriesInput
);
} else {
augmentedConfig.queries = config.augmentationProperties.queriesInput;
}
}
if (augmentedConfig.queries?.length === 0) {
delete augmentedConfig.queries;
}
// Inject the packs from the input
if (config.augmentationProperties.packsInput) {
if (config.augmentationProperties.packsInputCombines) {
// At this point, we already know that this is a single-language analysis
if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs = (augmentedConfig.packs || []).concat(
config.augmentationProperties.packsInput
);
} else if (!augmentedConfig.packs) {
augmentedConfig.packs = config.augmentationProperties.packsInput;
} else {
// At this point, we know there is only one language.
// If there were more than one language, an error would already have been thrown.
const language = Object.keys(augmentedConfig.packs)[0];
augmentedConfig.packs[language] = augmentedConfig.packs[
language
].concat(config.augmentationProperties.packsInput);
}
} else {
augmentedConfig.packs = config.augmentationProperties.packsInput;
}
}
if (Array.isArray(augmentedConfig.packs) && !augmentedConfig.packs.length) {
delete augmentedConfig.packs;
}
if (config.augmentationProperties.injectedMlQueries) {
// We need to inject the ML queries into the original user input before
// we pass this on to the CLI, to make sure these get run.
const packString = await util.getMlPoweredJsQueriesPack(codeql);
if (augmentedConfig.packs === undefined) augmentedConfig.packs = [];
if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs.push(packString);
} else {
if (!augmentedConfig.packs.javascript)
augmentedConfig.packs["javascript"] = [];
augmentedConfig.packs["javascript"].push(packString);
}
}
fs.writeFileSync(configLocation, yaml.dump(augmentedConfig));
return configLocation;
}
function cloneObject(obj: any) {
return JSON.parse(JSON.stringify(obj));
}

View file

@ -174,7 +174,9 @@ test("loading config saves config", async (t) => {
const config2 = await configUtils.getConfig(tmpDir, logger);
t.not(config2, undefined);
if (config2 !== undefined) {
t.deepEqual(config1, config2);
// removes properties assigned to undefined.
const expectedConfig = JSON.parse(JSON.stringify(config1));
t.deepEqual(expectedConfig, config2);
}
});
});
@ -356,7 +358,7 @@ test("load non-empty input", async (t) => {
debugMode: false,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
injectedMlQueries: false,
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
};
@ -1676,6 +1678,7 @@ function parseInputAndConfigMacro(
configUtils.parsePacks(
packsFromConfig,
packsFromInput,
!!packsFromInput?.trim().startsWith("+"), // coerce to boolean
languages,
"/a/b",
mockLogger
@ -1697,6 +1700,7 @@ function parseInputAndConfigErrorMacro(
packsFromConfig: string[] | Record<string, string[]>,
packsFromInput: string | undefined,
languages: Language[],
packsFromInputOverride: boolean,
expected: RegExp
) {
t.throws(
@ -1704,6 +1708,7 @@ function parseInputAndConfigErrorMacro(
configUtils.parsePacks(
packsFromConfig,
packsFromInput,
packsFromInputOverride,
languages,
"/a/b",
mockLogger
@ -1782,6 +1787,7 @@ test(
{},
"c/d",
[],
false,
/No languages specified/
);
@ -1791,6 +1797,7 @@ test(
{},
"c/d",
[Language.cpp, Language.csharp],
false,
/multi-language analysis/
);
@ -1800,6 +1807,7 @@ test(
{},
" + ",
[Language.cpp],
true,
/remove the '\+'/
);
@ -1809,6 +1817,7 @@ test(
{},
" xxx",
[Language.cpp],
false,
/"xxx" is not a valid pack/
);
@ -1988,3 +1997,164 @@ test(
"security-and-quality",
"~0.3.0"
);
const calculateAugmentationMacro = test.macro({
exec: async (
t: ExecutionContext,
_title: string,
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
expectedAugmentationProperties: configUtils.AugmentationProperties
) => {
const actualAugmentationProperties = configUtils.calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages
);
t.deepEqual(actualAugmentationProperties, expectedAugmentationProperties);
},
title: (_, title) => `Calculate Augmentation: ${title}`,
});
test(
calculateAugmentationMacro,
"All empty",
undefined,
undefined,
[Language.javascript],
{
queriesInputCombines: false,
queriesInput: undefined,
packsInputCombines: false,
packsInput: undefined,
injectedMlQueries: false,
} as configUtils.AugmentationProperties
);
test(
calculateAugmentationMacro,
"With queries",
undefined,
" a, b , c, d",
[Language.javascript],
{
queriesInputCombines: false,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
packsInputCombines: false,
packsInput: undefined,
injectedMlQueries: false,
} as configUtils.AugmentationProperties
);
test(
calculateAugmentationMacro,
"With queries combining",
undefined,
" + a, b , c, d ",
[Language.javascript],
{
queriesInputCombines: true,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
packsInputCombines: false,
packsInput: undefined,
injectedMlQueries: false,
} as configUtils.AugmentationProperties
);
test(
calculateAugmentationMacro,
"With packs",
" codeql/a , codeql/b , codeql/c , codeql/d ",
undefined,
[Language.javascript],
{
queriesInputCombines: false,
queriesInput: undefined,
packsInputCombines: false,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
injectedMlQueries: false,
} as configUtils.AugmentationProperties
);
test(
calculateAugmentationMacro,
"With packs combining",
" + codeql/a, codeql/b, codeql/c, codeql/d",
undefined,
[Language.javascript],
{
queriesInputCombines: false,
queriesInput: undefined,
packsInputCombines: true,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
injectedMlQueries: false,
} as configUtils.AugmentationProperties
);
const calculateAugmentationErrorMacro = test.macro({
exec: async (
t: ExecutionContext,
_title: string,
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
expectedError: RegExp | string
) => {
t.throws(
() =>
configUtils.calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages
),
{ message: expectedError }
);
},
title: (_, title) => `Calculate Augmentation Error: ${title}`,
});
test(
calculateAugmentationErrorMacro,
"Plus (+) with nothing else (queries)",
undefined,
" + ",
[Language.javascript],
/The workflow property "queries" is invalid/
);
test(
calculateAugmentationErrorMacro,
"Plus (+) with nothing else (packs)",
" + ",
undefined,
[Language.javascript],
/The workflow property "packs" is invalid/
);
test(
calculateAugmentationErrorMacro,
"Packs input with multiple languages",
" + a/b, c/d ",
undefined,
[Language.javascript, Language.java],
/Cannot specify a 'packs' input in a multi-language analysis/
);
test(
calculateAugmentationErrorMacro,
"Packs input with no languages",
" + a/b, c/d ",
undefined,
[],
/No languages specified/
);
test(
calculateAugmentationErrorMacro,
"Invalid packs",
" a-pack-without-a-scope ",
undefined,
[Language.javascript],
/"a-pack-without-a-scope" is not a valid pack/
);

View file

@ -175,10 +175,9 @@ export interface Config {
* Specifies the name of the database in the debugging artifact.
*/
debugDatabaseName: string;
/**
* Whether we injected ML queries into this configuration.
*/
injectedMlQueries: boolean;
augmentationProperties: AugmentationProperties;
/**
* Partial map from languages to locations of TRAP caches for that language.
* If a key is omitted, then TRAP caching should not be used for that language.
@ -186,6 +185,52 @@ export interface Config {
trapCaches: Partial<Record<Language, string>>;
}
/**
* Describes how to augment the user config with inputs from the action.
*
* When running a CodeQL analysis, the user can supply a config file. When
* running a CodeQL analysis from a GitHub action, the user can supply a
* config file _and_ a set of inputs.
*
* The inputs from the action are used to augment the user config before
* passing the user config to the CodeQL CLI invocation.
*/
export interface AugmentationProperties {
/**
* Whether or not the queries input combines with the queries in the config.
*/
queriesInputCombines: boolean;
/**
* The queries input from the `with` block of the action declaration
*/
queriesInput?: Array<{ uses: string }>;
/**
* Whether or not the packs input combines with the packs in the config.
*/
packsInputCombines: boolean;
/**
* The packs input from the `with` block of the action declaration
*/
packsInput?: string[];
/**
* Whether we injected ML queries into this configuration.
*/
injectedMlQueries: boolean;
}
/**
* The default, empty augmentation properties. This is most useeful
* for tests.
*/
export const defaultAugmentationProperties: AugmentationProperties = {
queriesInputCombines: false,
packsInputCombines: false,
injectedMlQueries: false,
packsInput: undefined,
queriesInput: undefined,
};
export type Packs = Partial<Record<Language, string[]>>;
export interface Pack {
@ -913,8 +958,8 @@ function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
*/
export async function getDefaultConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
packsInput: string | undefined,
rawQueriesInput: string | undefined,
rawPacksInput: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
@ -944,21 +989,30 @@ export async function getDefaultConfig(
};
}
await addDefaultQueries(codeQL, languages, queries);
const packs = parsePacksFromInput(packsInput, languages) ?? {};
let injectedMlQueries = false;
if (queriesInput) {
injectedMlQueries = await addQueriesAndPacksFromWorkflow(
codeQL,
queriesInput,
languages,
queries,
packs,
tempDir,
workspacePath,
apiDetails,
featureFlags,
logger
);
const augmentationProperties = calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages
);
const packs = augmentationProperties.packsInput
? {
[languages[0]]: augmentationProperties.packsInput,
}
: {};
if (rawQueriesInput) {
augmentationProperties.injectedMlQueries =
await addQueriesAndPacksFromWorkflow(
codeQL,
rawQueriesInput,
languages,
queries,
packs,
tempDir,
workspacePath,
apiDetails,
featureFlags,
logger
);
}
return {
@ -975,7 +1029,7 @@ export async function getDefaultConfig(
debugMode,
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
augmentationProperties,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
@ -987,8 +1041,8 @@ export async function getDefaultConfig(
*/
async function loadConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
packsInput: string | undefined,
rawQueriesInput: string | undefined,
rawPacksInput: string | undefined,
configFile: string,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
@ -1053,10 +1107,15 @@ async function loadConfig(
if (!disableDefaultQueries) {
await addDefaultQueries(codeQL, languages, queries);
}
const augmentationProperties = calculateAugmentation(
rawPacksInput,
rawQueriesInput,
languages
);
const packs = parsePacks(
parsedYAML[PACKS_PROPERTY] ?? {},
packsInput,
rawPacksInput,
augmentationProperties.packsInputCombines,
languages,
configFile,
logger
@ -1066,23 +1125,23 @@ async function loadConfig(
// they should take precedence over the queries in the config file
// unless they're prefixed with "+", in which case they supplement those
// in the config file.
let injectedMlQueries = false;
if (queriesInput) {
injectedMlQueries = await addQueriesAndPacksFromWorkflow(
codeQL,
queriesInput,
languages,
queries,
packs,
tempDir,
workspacePath,
apiDetails,
featureFlags,
logger
);
if (rawQueriesInput) {
augmentationProperties.injectedMlQueries =
await addQueriesAndPacksFromWorkflow(
codeQL,
rawQueriesInput,
languages,
queries,
packs,
tempDir,
workspacePath,
apiDetails,
featureFlags,
logger
);
}
if (
shouldAddConfigFileQueries(queriesInput) &&
shouldAddConfigFileQueries(rawQueriesInput) &&
QUERIES_PROPERTY in parsedYAML
) {
const queriesArr = parsedYAML[QUERIES_PROPERTY];
@ -1159,13 +1218,81 @@ async function loadConfig(
debugMode,
debugArtifactName,
debugDatabaseName,
injectedMlQueries,
augmentationProperties,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
};
}
/**
* Calculates how the codeql config file needs to be augmented before passing
* it to the CLI. The reason this is necessary is the codeql-action can be called
* with extra inputs from the workflow. These inputs are not part of the config
* and the CLI does not know about these inputs so we need to inject them into
* the config file sent to the CLI.
*
* @param rawPacksInput The packs input from the action configuration.
* @param rawQueriesInput The queries input from the action configuration.
* @param languages The languages that the config file is for. If the packs input
* is non-empty, then there must be exactly one language. Otherwise, an
* error is thrown.
*
* @returns The properties that need to be augmented in the config file.
*
* @throws An error if the packs input is non-empty and the languages input does
* not have exactly one language.
*/
// exported for testing.
export function calculateAugmentation(
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[]
): AugmentationProperties {
const packsInputCombines = shouldCombine(rawPacksInput);
const packsInput = parsePacksFromInput(
rawPacksInput,
languages,
packsInputCombines
);
const queriesInputCombines = shouldCombine(rawQueriesInput);
const queriesInput = parseQueriesFromInput(
rawQueriesInput,
queriesInputCombines
);
return {
injectedMlQueries: false, // filled in later
packsInputCombines,
packsInput: packsInput?.[languages[0]],
queriesInput,
queriesInputCombines,
};
}
function parseQueriesFromInput(
rawQueriesInput: string | undefined,
queriesInputCombines: boolean
) {
if (!rawQueriesInput) {
return undefined;
}
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: rawQueriesInput?.trim();
if (queriesInputCombines && trimmedInput.length === 0) {
throw new Error(
getConfigFilePropertyError(
undefined,
"queries",
"A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."
)
);
}
return trimmedInput.split(",").map((query) => ({ uses: query.trim() }));
}
/**
* Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
* and `-` allowed as long as not the first or last char.
@ -1224,10 +1351,11 @@ export function parsePacksFromConfig(
}
function parsePacksFromInput(
packsInput: string | undefined,
languages: Language[]
rawPacksInput: string | undefined,
languages: Language[],
packsInputCombines: boolean
): Packs | undefined {
if (!packsInput?.trim()) {
if (!rawPacksInput?.trim()) {
return undefined;
}
@ -1239,19 +1367,23 @@ function parsePacksFromInput(
throw new Error("No languages specified. Cannot process the packs input.");
}
packsInput = packsInput.trim();
if (packsInput.startsWith("+")) {
packsInput = packsInput.substring(1).trim();
if (!packsInput) {
rawPacksInput = rawPacksInput.trim();
if (packsInputCombines) {
rawPacksInput = rawPacksInput.trim().substring(1).trim();
if (!rawPacksInput) {
throw new Error(
"A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."
getConfigFilePropertyError(
undefined,
"packs",
"A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."
)
);
}
}
return {
[languages[0]]: packsInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack));
[languages[0]]: rawPacksInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack, ""));
return packs;
}, [] as string[]),
};
@ -1357,12 +1489,12 @@ export function validatePackSpecification(pack: string, configFile?: string) {
// exported for testing
export function parsePacks(
rawPacksFromConfig: string[] | Record<string, string[]>,
rawPacksInput: string | undefined,
rawPacksFromInput: string | undefined,
packsInputCombines: boolean,
languages: Language[],
configFile: string,
logger: Logger
) {
const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
): Packs {
const packsFomConfig = parsePacksFromConfig(
rawPacksFromConfig,
languages,
@ -1370,18 +1502,35 @@ export function parsePacks(
logger
);
const packsFromInput = parsePacksFromInput(
rawPacksFromInput,
languages,
packsInputCombines
);
if (!packsFromInput) {
return packsFomConfig;
}
if (!shouldCombinePacks(rawPacksInput)) {
if (!packsInputCombines) {
if (!packsFromInput) {
throw new Error(getPacksInvalid(configFile));
}
return packsFromInput;
}
return combinePacks(packsFromInput, packsFomConfig);
}
function shouldCombinePacks(packsInput?: string): boolean {
return !!packsInput?.trim().startsWith("+");
/**
* The convention in this action is that an input value that is prefixed with a '+' will
* be combined with the corresponding value in the config file.
*
* Without a '+', an input value will override the corresponding value in the config file.
*
* @param inputValue The input value to process.
* @returns true if the input value should replace the corresponding value in the config file, false if it should be appended.
*/
function shouldCombine(inputValue?: string): boolean {
return !!inputValue?.trim().startsWith("+");
}
function combinePacks(packs1: Packs, packs2: Packs): Packs {

View file

@ -8,7 +8,7 @@ import * as actionsUtil from "./actions-util";
import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Config, defaultAugmentationProperties } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { Language } from "./languages";
import { RepositoryNwo } from "./repository";
@ -56,7 +56,7 @@ function getTestConfig(tmpDir: string): Config {
debugMode: false,
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: defaultAugmentationProperties,
trapCaches: {},
};
}

View file

@ -31,7 +31,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
};
}

View file

@ -79,7 +79,11 @@ const testConfigWithoutTmpDir: Config = {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {
javascript: "/some/cache/dir",
},
@ -100,7 +104,11 @@ function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {
javascript: path.resolve(tmpDir, "jsCache"),
ruby: path.resolve(tmpDir, "rubyCache"),

View file

@ -351,7 +351,11 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
injectedMlQueries: false,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
};

View file

@ -10,7 +10,11 @@ import * as semver from "semver";
import * as api from "./api-client";
import { getApiClient, GitHubApiDetails } from "./api-client";
import * as apiCompatibility from "./api-compatibility.json";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import {
CodeQL,
CODEQL_VERSION_CONFIG_FILES,
CODEQL_VERSION_NEW_TRACING,
} from "./codeql";
import {
Config,
parsePacksSpecification,
@ -510,6 +514,13 @@ enum EnvVar {
* own sandwiched workflow mechanism
*/
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
/**
* If set to the "true" string and the codeql CLI version is greater than
* `CODEQL_VERSION_CONFIG_FILES`, then the codeql-action will pass the
* the codeql-config file to the codeql CLI to be processed there.
*/
CODEQL_PASS_CONFIG_TO_CLI = "CODEQL_PASS_CONFIG_TO_CLI",
}
const exportVar = (mode: Mode, name: string, value: string) => {
@ -763,6 +774,19 @@ export function isInTestMode(): boolean {
return process.env["TEST_MODE"] === "true" || false;
}
/**
* @returns true if the action should generate a conde-scanning config file
* that gets passed to the CLI.
*/
export async function useCodeScanningConfigInCli(
codeql: CodeQL
): Promise<boolean> {
return (
process.env[EnvVar.CODEQL_PASS_CONFIG_TO_CLI] === "true" &&
(await codeQlVersionAbove(codeql, CODEQL_VERSION_CONFIG_FILES))
);
}
/*
* Returns whether the path in the argument represents an existing directory.
*/

View file

@ -6,9 +6,6 @@ packs:
- dsp-testing/codeql-pack1@1.0.0
- dsp-testing/codeql-pack2
- dsp-testing/codeql-pack3:other-query.ql
ruby:
- dsp-testing/hucairz
- dsp-testing/i-dont-exist@1.0.0
paths-ignore:
- tests

View file

@ -0,0 +1,9 @@
packs:
javascript:
- dsp-testing/codeql-pack1@1.0.0
- dsp-testing/codeql-pack2
ruby:
- codeql/ruby-queries
queries:
- uses: ./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql

View file

@ -0,0 +1,9 @@
name: Config using all properties
disable-default-queries: true
paths-ignore:
- xxx
paths:
- yyy

View file

@ -0,0 +1,7 @@
packs:
javascript:
- dsp-testing/codeql-pack1@1.0.0
- dsp-testing/codeql-pack2
queries:
- uses: ./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql