robojerk/codeql-action

Merge remote-tracking branch 'upstream/main' into aeisenberg/pack-run

Browse source
This commit is contained in:
Andrew Eisenberg 2021-06-09 12:43:17 -07:00
commit 82388fd94a
36 changed files with 490 additions and 289 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/post-release-mergeback.yml vendored
View file

@ -124,4 +124,4 @@ jobs:
--base "$BASE_BRANCH" \
--title "$PR_TITLE" \
--body "$PR_BODY" \
"$DRAFT"
${DRAFT:+"$DRAFT"} # no quotes around $DRAFT. gh will error out if there is an empty ""

40
.github/workflows/pr-checks.yml vendored
View file

@ -65,20 +65,38 @@ jobs:
shell: bash
run: ./build.sh
- uses: ./../action/analyze
id: analysis
env:
TEST_MODE: true
- run: |
cd "$RUNNER_TEMP/customDbLocation"
# List all directories as there will be precisely one directory per database
# but there may be other files in this directory such as query suites.
if [ "$(ls -d */ | wc -l)" != 6 ] || \
[[ ! -d cpp ]] || \
[[ ! -d csharp ]] || \
[[ ! -d go ]] || \
[[ ! -d java ]] || \
[[ ! -d javascript ]] || \
[[ ! -d python ]]; then
echo "Did not find expected number of databases. Database dir contains: $(ls)"
CPP_DB=${{ fromJson(steps.analysis.outputs.db-locations).cpp }}
if [[ ! -d $CPP_DB ]] || [[ ! $CPP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for CPP, or created it in the wrong location."
exit 1
fi
CSHARP_DB=${{ fromJson(steps.analysis.outputs.db-locations).csharp }}
if [[ ! -d $CSHARP_DB ]] || [[ ! $CSHARP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for C Sharp, or created it in the wrong location."
exit 1
fi
GO_DB=${{ fromJson(steps.analysis.outputs.db-locations).go }}
if [[ ! -d $GO_DB ]] || [[ ! $GO_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Go, or created it in the wrong location."
exit 1
fi
JAVA_DB=${{ fromJson(steps.analysis.outputs.db-locations).java }}
if [[ ! -d $JAVA_DB ]] || [[ ! $JAVA_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Java, or created it in the wrong location."
exit 1
fi
JAVASCRIPT_DB=${{ fromJson(steps.analysis.outputs.db-locations).javascript }}
if [[ ! -d $JAVASCRIPT_DB ]] || [[ ! $JAVASCRIPT_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Javascript, or created it in the wrong location."
exit 1
fi
PYTHON_DB=${{ fromJson(steps.analysis.outputs.db-locations).python }}
if [[ ! -d $PYTHON_DB ]] || [[ ! $PYTHON_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Python, or created it in the wrong location."
exit 1
fi

5
CHANGELOG.md
View file

@ -2,6 +2,11 @@
## [UNRELEASED]
## 1.0.1 - 07 Jun 2021
- Pass the `--sarif-group-rules-by-pack` argument to CodeQL CLI invocations that generate SARIF. This means the SARIF rule object for each query will now be found underneath its corresponding query pack in `runs[].tool.extensions`. [#546](https://github.com/github/codeql-action/pull/546)
- Output the location of CodeQL databases created in the analyze step. [#543](https://github.com/github/codeql-action/pull/543)
## 1.0.0 - 31 May 2021
- Add this changelog file. [#507](https://github.com/github/codeql-action/pull/507)

7
analyze/action.yml
View file

@ -13,6 +13,10 @@ inputs:
description: Upload the SARIF file
required: false
default: "true"
cleanup-level:
description: "Level of cleanup to perform on CodeQL databases at the end of the analyze step. This should either be 'none' to skip cleanup, or be a valid argument for the --mode flag of the CodeQL CLI command 'codeql database cleanup' as documented at https://codeql.github.com/docs/codeql-cli/manual/database-cleanup"
required: false
default: "brutal"
ram:
description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
required: false
@ -34,6 +38,9 @@ inputs:
default: ${{ github.token }}
matrix:
default: ${{ toJson(matrix) }}
outputs:
db-locations:
description: A map from language to absolute path for each database created by CodeQL.
runs:
using: 'node12'
main: '../lib/analyze-action.js'

8
lib/analyze-action.js generated
View file

@ -50,6 +50,14 @@ async function run() {
};
const outputDir = actionsUtil.getRequiredInput("output");
const queriesStats = await analyze_1.runAnalyze(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger), actionsUtil.getOptionalInput("category"), config, logger);
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await analyze_1.runCleanup(config, actionsUtil.getOptionalInput("cleanup-level") || "brutal", logger);
}
const dbLocations = {};
for (const language of config.languages) {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
}
core.setOutput("db-locations", dbLocations);
if (actionsUtil.getRequiredInput("upload") === "true") {
const uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
stats = { ...queriesStats, ...uploadStats };

2
lib/analyze-action.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAImB;AACnB,iDAAmD;AACnD,uCAA6C;AAC7C,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAKmB;AACnB,iDAAmD;AACnD,uCAA6C;AAC7C,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

78
lib/analyze.js generated
View file

@ -15,7 +15,6 @@ const codeql_1 = require("./codeql");
const count_loc_1 = require("./count-loc");
const languages_1 = require("./languages");
const sharedEnv = __importStar(require("./shared-environment"));
const upload_lib_1 = require("./upload-lib");
const util = __importStar(require("./util"));
class CodeQLAnalysisError extends Error {
constructor(queriesStatusReport, message) {
@ -86,7 +85,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
// that here.
config.paths, config.pathsIgnore, config.languages, logger);
for (const language of config.languages) {
logger.startGroup(`Analyzing ${language}`);
logger.startGroup(`Running queries for ${language}`);
const queries = config.queries[language];
const packsWithVersion = config.packs[language] || [];
const hasBuiltinQueries = ((_a = queries) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
@ -103,57 +102,40 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`);
}
let analysisSummaryBuiltIn = "";
const customAnalysisSummaries = [];
const querySuitePaths = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
const { sarifFile, stdout } = await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"]), sarifFolder, undefined);
analysisSummaryBuiltIn = stdout;
await injectLinesOfCode(sarifFile, language, locPromise);
querySuitePaths.push(await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"]), undefined));
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = config.tempDir;
const temporarySarifFiles = [];
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
const { sarifFile, stdout } = await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries), temporarySarifDir, queries["custom"][i].searchPath);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
querySuitePaths.push(await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries), queries["custom"][i].searchPath));
ranCustom = true;
}
}
if (packsWithVersion.length > 0) {
const { sarifFile, stdout } = await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), temporarySarifDir, undefined);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
querySuitePaths.push(await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), undefined));
ranCustom = true;
}
if (temporarySarifFiles.length > 0) {
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles));
await injectLinesOfCode(sarifFile, language, locPromise);
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
// Print the LoC baseline and the summary results from database analyze for the standard
// query suite and (if appropriate) each custom query suite.
logger.startGroup(`Analysis summary for ${language}`);
printLinesOfCodeSummary(logger, language, await locPromise);
logger.info(analysisSummaryBuiltIn);
for (const [i, customSummary] of customAnalysisSummaries.entries()) {
if (customSummary.trim() === "") {
continue;
}
const description = customAnalysisSummaries.length === 1
? "custom queries"
: `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
logger.info(`Analysis summary for ${description}:`);
logger.info("");
logger.info(customSummary);
logger.info("");
}
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(language, querySuitePaths, sarifFile);
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
printLinesOfCodeSummary(logger, language, await locPromise);
}
catch (e) {
logger.info(e);
@ -163,18 +145,22 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
}
}
return statusReport;
async function runQueryGroup(language, type, querySuiteContents, destinationFolder, searchPath) {
async function runInterpretResults(language, queries, sarifFile) {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
return await codeql.databaseInterpretResults(databasePath, queries, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId);
}
async function runQueryGroup(language, type, querySuiteContents, searchPath) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
const databaseAnalyzeStdout = await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
return { sarifFile, stdout: databaseAnalyzeStdout };
await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, memoryFlag, threadsFlag);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
}
exports.runQueries = runQueries;
@ -202,6 +188,16 @@ async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, a
return { ...queriesStats };
}
exports.runAnalyze = runAnalyze;
async function runCleanup(config, cleanupLevel, logger) {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
logger.endGroup();
}
exports.runCleanup = runCleanup;
async function injectLinesOfCode(sarifFile, language, locPromise) {
const lineCounts = await locPromise;
const idPrefix = count_loc_1.getIdPrefix(language);

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

13
lib/analyze.test.js generated
View file

@ -58,7 +58,10 @@ ava_1.default("status report fields and search path setting", async (t) => {
for (const language of Object.values(languages_1.Language)) {
codeql_1.setCodeQL({
packDownload: async () => ({ packs: [] }),
databaseAnalyze: async (_, sarifFile, searchPath) => {
databaseRunQueries: async (_db, searchPath) => {
searchPathsUsed.push(searchPath);
},
databaseInterpretResults: async (_db, _queriesRun, sarifFile) => {
fs.writeFileSync(sarifFile, JSON.stringify({
runs: [
// variant 1 uses ruleId
@ -88,7 +91,6 @@ ava_1.default("status report fields and search path setting", async (t) => {
{},
],
}));
searchPathsUsed.push(searchPath);
return "";
},
});
@ -119,6 +121,7 @@ ava_1.default("status report fields and search path setting", async (t) => {
const hasPacks = language in packs;
t.deepEqual(Object.keys(builtinStatusReport).length, hasPacks ? 2 : 1);
t.true(`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport);
t.true(`interpret_results_${language}_duration_ms` in builtinStatusReport);
config.queries[language] = {
builtin: [],
custom: [
@ -133,12 +136,13 @@ ava_1.default("status report fields and search path setting", async (t) => {
],
};
const customStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
t.deepEqual(Object.keys(customStatusReport).length, 1);
t.deepEqual(Object.keys(customStatusReport).length, 2);
t.true(`analyze_custom_queries_${language}_duration_ms` in customStatusReport);
const expectedSearchPathsUsed = hasPacks
? [undefined, undefined, "/1", "/2", undefined]
: [undefined, "/1", "/2"];
t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
t.true(`interpret_results_${language}_duration_ms` in customStatusReport);
}
verifyLineCounts(tmpDir);
verifyQuerySuites(tmpDir);
@ -146,8 +150,7 @@ ava_1.default("status report fields and search path setting", async (t) => {
function verifyLineCounts(tmpDir) {
// eslint-disable-next-line github/array-foreach
Object.keys(languages_1.Language).forEach((lang, i) => {
verifyLineCountForFile(lang, path.join(tmpDir, `${lang}-builtin.sarif`), i + 1);
verifyLineCountForFile(lang, path.join(tmpDir, `${lang}-custom.sarif`), i + 1);
verifyLineCountForFile(lang, path.join(tmpDir, `${lang}.sarif`), i + 1);
});
}
function verifyLineCountForFile(lang, filePath, lineCount) {

2
lib/analyze.test.js.map
View file

File diff suppressed because one or more lines are too long

4
lib/api-client.js generated
View file

@ -12,6 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
Object.defineProperty(exports, "__esModule", { value: true });
const path = __importStar(require("path"));
const githubUtils = __importStar(require("@actions/github/lib/utils"));
const retry = __importStar(require("@octokit/plugin-retry"));
const console_log_level_1 = __importDefault(require("console-log-level"));
const actions_util_1 = require("./actions-util");
const util_1 = require("./util");
@ -24,7 +25,8 @@ var DisallowedAPIVersionReason;
})(DisallowedAPIVersionReason = exports.DisallowedAPIVersionReason || (exports.DisallowedAPIVersionReason = {}));
exports.getApiClient = function (apiDetails, { allowExternal = false } = {}) {
const auth = (allowExternal && apiDetails.externalRepoAuth) || apiDetails.auth;
return new githubUtils.GitHub(githubUtils.getOctokitOptions(auth, {
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
return new retryingOctokit(githubUtils.getOctokitOptions(auth, {
baseUrl: getApiUrl(apiDetails.url),
userAgent: `CodeQL-${util_1.getMode()}/${pkg.version}`,
log: console_log_level_1.default({ level: "debug" }),

2
lib/api-client.js.map
View file

@ -1 +1 @@
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,uEAAyD;AACzD,0EAAgD;AAEhD,iDAAkD;AAClD,iCAAsD;AAEtD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,GAA1B,kCAA0B,KAA1B,kCAA0B,QAGrC;AAeY,QAAA,YAAY,GAAG,UAC1B,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,OAAO,IAAI,WAAW,CAAC,MAAM,CAC3B,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,UAAU,cAAO,EAAE,IAAI,GAAG,CAAC,OAAO,EAAE;QAC/C,GAAG,EAAE,2BAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,SAAS,CAAC,SAAiB;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAE/B,uDAAuD;IACvD,0CAA0C;IAC1C,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,IAAI,GAAG,CAAC,QAAQ,KAAK,gBAAgB,EAAE;QACtE,OAAO,wBAAwB,CAAC;KACjC;IAED,6BAA6B;IAC7B,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,uFAAuF;AACvF,qFAAqF;AACrF,+CAA+C;AAC/C,SAAgB,mBAAmB;IACjC,MAAM,UAAU,GAAG;QACjB,IAAI,EAAE,+BAAgB,CAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,0BAAmB,CAAC,mBAAmB,CAAC;KAC9C,CAAC;IAEF,OAAO,oBAAY,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAPD,kDAOC"}
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,uEAAyD;AACzD,6DAA+C;AAC/C,0EAAgD;AAEhD,iDAAkD;AAClD,iCAAsD;AAEtD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,GAA1B,kCAA0B,KAA1B,kCAA0B,QAGrC;AAeY,QAAA,YAAY,GAAG,UAC1B,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,IAAI,eAAe,CACxB,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,UAAU,cAAO,EAAE,IAAI,GAAG,CAAC,OAAO,EAAE;QAC/C,GAAG,EAAE,2BAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,SAAS,CAAC,SAAiB;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAE/B,uDAAuD;IACvD,0CAA0C;IAC1C,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,IAAI,GAAG,CAAC,QAAQ,KAAK,gBAAgB,EAAE;QACtE,OAAO,wBAAwB,CAAC;KACjC;IAED,6BAA6B;IAC7B,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,uFAAuF;AACvF,qFAAqF;AACrF,+CAA+C;AAC/C,SAAgB,mBAAmB;IACjC,MAAM,UAAU,GAAG;QACjB,IAAI,EAAE,+BAAgB,CAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,0BAAmB,CAAC,mBAAmB,CAAC;KAC9C,CAAC;IAEF,OAAO,oBAAY,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAPD,kDAOC"}

5
lib/api-client.test.js generated
View file

@ -19,9 +19,12 @@ const util_1 = require("./util");
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
testing_utils_1.setupTests(ava_1.default);
let pluginStub;
let githubStub;
ava_1.default.beforeEach(() => {
githubStub = sinon_1.default.stub(githubUtils, "GitHub");
pluginStub = sinon_1.default.stub(githubUtils.GitHub, "plugin");
githubStub = sinon_1.default.stub();
pluginStub.returns(githubStub);
util_1.initializeEnvironment(util_1.Mode.actions, pkg.version);
});
ava_1.default("Get the client API", async (t) => {

2
lib/api-client.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"api-client.test.js","sourceRoot":"","sources":["../src/api-client.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uEAAyD;AACzD,8CAA6C;AAC7C,kDAA0B;AAE1B,6CAA4C;AAC5C,mDAA6C;AAC7C,iCAAqD;AAErD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,IAAI,UAA2B,CAAC;AAEhC,aAAI,CAAC,UAAU,CAAC,GAAG,EAAE;IACnB,UAAU,GAAG,eAAK,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,4BAAqB,CAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oBAAoB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrC,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,6BAA6B,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC9C,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,yCAAyC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC1D,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oCAAoC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrD,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,qCAAqC;KAC3C,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,wBAAwB;QACjC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,SAAS,MAAM,CACb,CAA4B,EAC5B,UAAe,EACf,aAAkB,EAClB,QAAa;IAEb,yBAAY,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAExC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,iEAAiE;IACjE,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5B,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzC,CAAC"}
{"version":3,"file":"api-client.test.js","sourceRoot":"","sources":["../src/api-client.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uEAAyD;AACzD,8CAA6C;AAC7C,kDAA0B;AAE1B,6CAA4C;AAC5C,mDAA6C;AAC7C,iCAAqD;AAErD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,IAAI,UAA2B,CAAC;AAChC,IAAI,UAA2B,CAAC;AAEhC,aAAI,CAAC,UAAU,CAAC,GAAG,EAAE;IACnB,UAAU,GAAG,eAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACtD,UAAU,GAAG,eAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC/B,4BAAqB,CAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oBAAoB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrC,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,6BAA6B,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC9C,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,yCAAyC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC1D,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oCAAoC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrD,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,qCAAqC;KAC3C,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,wBAAwB;QACjC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,SAAS,MAAM,CACb,CAA4B,EAC5B,UAAe,EACf,aAAkB,EAClB,QAAa;IAEb,yBAAY,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAExC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,iEAAiE;IACjE,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5B,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzC,CAAC"}

44
lib/codeql.js generated
View file

@ -285,8 +285,10 @@ function setCodeQL(partialCodeql) {
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
databaseInterpretResults: resolveFunction(partialCodeql, "databaseInterpretResults"),
};
return cachedCodeQL;
}
@ -416,6 +418,7 @@ function getCodeQLForCmd(cmd) {
await toolrunner_error_catcher_1.toolrunnerErrorCatcher(cmd, [
"database",
"finalize",
"--finalize-dataset",
threadsFlag,
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
@ -464,30 +467,40 @@ function getCodeQLForCmd(cmd) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseAnalyze(databasePath, sarifFile, extraSearchPath, querySuite, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId) {
async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, memoryFlag, threadsFlag) {
const args = [
"database",
"analyze",
"run-queries",
memoryFlag,
threadsFlag,
databasePath,
"--min-disk-free=1024",
"--format=sarif-latest",
"--sarif-multicause-markdown",
`--output=${sarifFile}`,
addSnippetsFlag,
// Enable progress verbosity so we log each query as it's interpreted. This aids debugging
// when interpretation takes a while for one of the queries being analyzed.
"-v",
...getExtraOptionsFromEnv(["database", "analyze"]),
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (extraSearchPath !== undefined) {
args.push("--additional-packs", extraSearchPath);
}
args.push(querySuitePath);
await new toolrunner.ToolRunner(cmd, args).exec();
},
async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId) {
const args = [
"database",
"interpret-results",
threadsFlag,
"--format=sarif-latest",
"--print-metrics-summary",
"--sarif-group-rules-by-pack",
"-v",
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "interpret-results"]),
];
if (automationDetailsId !== undefined) {
args.push("--sarif-category", automationDetailsId);
}
args.push(querySuite);
args.push(databasePath, ...querySuitePaths);
// capture stdout, which contains analysis summaries
let output = "";
await new toolrunner.ToolRunner(cmd, args, {
@ -542,6 +555,15 @@ function getCodeQLForCmd(cmd) {
throw new Error(`Attempted to download specified packs but got an error:\n${output}\n${e}`);
}
},
async databaseCleanup(databasePath, cleanupLevel) {
const args = [
"database",
"cleanup",
databasePath,
`--mode=${cleanupLevel}`,
];
await new toolrunner.ToolRunner(cmd, args).exec();
},
};
}
function packWithVersionToString(pack) {

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

28
lib/fingerprints.js generated
View file

@ -16,6 +16,7 @@ const tab = "\t".charCodeAt(0);
const space = " ".charCodeAt(0);
const lf = "\n".charCodeAt(0);
const cr = "\r".charCodeAt(0);
const EOF = 65535;
const BLOCK_SIZE = 100;
const MOD = long_1.default.fromInt(37); // L
// Compute the starting point for the hash mod
@ -36,9 +37,9 @@ function computeFirstMod() {
* the hashes of the lines near the end of the file.
*
* @param callback function that is called with the line number (1-based) and hash for every line
* @param input The file's contents
* @param filepath The path to the file to hash
*/
function hash(callback, input) {
async function hash(callback, filepath) {
// A rolling view in to the input
const window = Array(BLOCK_SIZE).fill(0);
// If the character in the window is the start of a new line
@ -82,12 +83,11 @@ function hash(callback, input) {
// as we go. Once we reach a point in the window again then we've processed
// BLOCK_SIZE characters and if the last character at this point in the window
// was the start of a line then we should output the hash for that line.
for (let i = 0, len = input.length; i <= len; i++) {
let current = i === len ? 65535 : input.charCodeAt(i);
const processCharacter = function (current) {
// skip tabs, spaces, and line feeds that come directly after a carriage return
if (current === space || current === tab || (prevCR && current === lf)) {
prevCR = false;
continue;
return;
}
// replace CR with LF
if (current === cr) {
@ -109,7 +109,14 @@ function hash(callback, input) {
lineStart = true;
}
updateHash(current);
};
const readStream = fs.createReadStream(filepath, "utf8");
for await (const data of readStream) {
for (let i = 0; i < data.length; ++i) {
processCharacter(data.charCodeAt(i));
}
}
processCharacter(EOF);
// Flush the remaining lines
for (let i = 0; i < BLOCK_SIZE; i++) {
if (lineNumbers[index] !== -1) {
@ -206,8 +213,8 @@ function resolveUriToFile(location, artifacts, checkoutPath, logger) {
exports.resolveUriToFile = resolveUriToFile;
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
function addFingerprints(sarifContents, checkoutPath, logger) {
var _a, _b;
async function addFingerprints(sarifContents, checkoutPath, logger) {
var _a, _b, _c, _d, _e;
const sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
// callbacks to accept hashes for that file and update the location
@ -222,6 +229,10 @@ function addFingerprints(sarifContents, checkoutPath, logger) {
logger.debug(`Unable to compute fingerprint for invalid location: ${JSON.stringify(primaryLocation)}`);
continue;
}
if (((_e = (_d = (_c = primaryLocation) === null || _c === void 0 ? void 0 : _c.physicalLocation) === null || _d === void 0 ? void 0 : _d.region) === null || _e === void 0 ? void 0 : _e.startLine) === undefined) {
// Locations without a line number are unlikely to be source files
continue;
}
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, checkoutPath, logger);
if (!filepath) {
continue;
@ -240,8 +251,7 @@ function addFingerprints(sarifContents, checkoutPath, logger) {
c(lineNumber, hashValue);
}
};
const fileContents = fs.readFileSync(filepath).toString();
hash(teeCallback, fileContents);
await hash(teeCallback, filepath);
}
return JSON.stringify(sarif);
}

2
lib/fingerprints.js.map
View file

File diff suppressed because one or more lines are too long

51
lib/fingerprints.test.js generated
View file

@ -16,28 +16,33 @@ const ava_1 = __importDefault(require("ava"));
const fingerprints = __importStar(require("./fingerprints"));
const logging_1 = require("./logging");
const testing_utils_1 = require("./testing-utils");
const util = __importStar(require("./util"));
testing_utils_1.setupTests(ava_1.default);
function testHash(t, input, expectedHashes) {
let index = 0;
const callback = function (lineNumber, hash) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
fingerprints.hash(callback, input);
t.is(index, input.split(/\r\n|\r|\n/).length);
async function testHash(t, input, expectedHashes) {
await util.withTmpDir(async (tmpDir) => {
const tmpFile = path.resolve(tmpDir, "testfile");
fs.writeFileSync(tmpFile, input);
let index = 0;
const callback = function (lineNumber, hash) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
await fingerprints.hash(callback, tmpFile);
t.is(index, input.split(/\r\n|\r|\n/).length);
});
}
ava_1.default("hash", (t) => {
ava_1.default("hash", async (t) => {
// Try empty file
testHash(t, "", ["c129715d7a2bc9a3:1"]);
await testHash(t, "", ["c129715d7a2bc9a3:1"]);
// Try various combinations of newline characters
testHash(t, " a\nb\n \t\tc\n d", [
await testHash(t, " a\nb\n \t\tc\n d", [
"271789c17abda88f:1",
"54703d4cd895b18:1",
"180aee12dab6264:1",
"a23a3dc5e078b07b:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
"8b7cf3e952e7aeb2:1",
"b1ae1287ec4718d9:1",
"bff680108adb0fcc:1",
@ -45,7 +50,7 @@ ava_1.default("hash", (t) => {
"b86d3392aea1be30:1",
"e6ceba753e1a442:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -54,7 +59,7 @@ ava_1.default("hash", (t) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
await testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -63,7 +68,7 @@ ava_1.default("hash", (t) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
await testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -72,7 +77,7 @@ ava_1.default("hash", (t) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
await testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -82,7 +87,7 @@ ava_1.default("hash", (t) => {
"c129715d7a2bc9a3:1",
]);
// Try repeating line that will generate identical hashes
testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
await testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
"a7f2ff13bc495cf2:1",
"a7f2ff13bc495cf2:2",
"a7f2ff13bc495cf2:3",
@ -95,7 +100,7 @@ ava_1.default("hash", (t) => {
"cc97dc7b1d7d8f7b:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
await testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
@ -150,7 +155,7 @@ ava_1.default("resolveUriToFile", (t) => {
t.is(testResolveUriToFile(dirpath, undefined, []), undefined);
t.is(testResolveUriToFile(`file://${dirpath}`, undefined, []), undefined);
});
ava_1.default("addFingerprints", (t) => {
ava_1.default("addFingerprints", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
@ -163,9 +168,9 @@ ava_1.default("addFingerprints", (t) => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
t.deepEqual(await fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
});
ava_1.default("missingRegions", (t) => {
ava_1.default("missingRegions", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
@ -178,6 +183,6 @@ ava_1.default("missingRegions", (t) => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
t.deepEqual(await fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
});
//# sourceMappingURL=fingerprints.test.js.map

2
lib/fingerprints.test.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/upload-lib.js generated
View file

@ -243,7 +243,7 @@ async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKe
validateSarifFileSchema(file, logger);
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
sarifPayload = await fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
sarifPayload = populateRunAutomationDetails(sarifPayload, category, analysisKey, environment);
const zippedSarif = zlib_1.default.gzipSync(sarifPayload).toString("base64");
const checkoutURI = file_url_1.default(checkoutPath);

2
lib/upload-lib.js.map
View file

File diff suppressed because one or more lines are too long

2
package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "1.0.1",
"version": "1.0.2",
"lockfileVersion": 2,
"requires": true,
"packages": {

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "1.0.1",
"version": "1.0.2",
"private": true,
"description": "CodeQL action",
"scripts": {

2
runner/package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "1.0.1",
"version": "1.0.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
runner/package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "1.0.1",
"version": "1.0.2",
"private": true,
"description": "CodeQL runner",
"scripts": {

15
src/analyze-action.ts
View file

@ -8,6 +8,7 @@ import {
runAnalyze,
CodeQLAnalysisError,
QueriesStatusReport,
runCleanup,
} from "./analyze";
import { Config, getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
@ -89,6 +90,20 @@ async function run() {
logger
);
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await runCleanup(
config,
actionsUtil.getOptionalInput("cleanup-level") || "brutal",
logger
);
}
const dbLocations: { [lang: string]: string } = {};
for (const language of config.languages) {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
}
core.setOutput("db-locations", dbLocations);
if (actionsUtil.getRequiredInput("upload") === "true") {
const uploadStats = await upload_lib.uploadFromActions(
outputDir,

26
src/analyze.test.ts
View file

@ -53,10 +53,16 @@ test("status report fields and search path setting", async (t) => {
for (const language of Object.values(Language)) {
setCodeQL({
packDownload: async () => ({ packs: [] }),
databaseAnalyze: async (
_,
sarifFile: string,
databaseRunQueries: async (
_db: string,
searchPath: string | undefined
) => {
searchPathsUsed.push(searchPath);
},
databaseInterpretResults: async (
_db: string,
_queriesRun: string[],
sarifFile: string
) => {
fs.writeFileSync(
sarifFile,
@ -92,7 +98,6 @@ test("status report fields and search path setting", async (t) => {
],
})
);
searchPathsUsed.push(searchPath);
return "";
},
});
@ -135,6 +140,9 @@ test("status report fields and search path setting", async (t) => {
t.true(
`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport
);
t.true(
`interpret_results_${language}_duration_ms` in builtinStatusReport
);
config.queries[language] = {
builtin: [],
@ -158,7 +166,7 @@ test("status report fields and search path setting", async (t) => {
config,
getRunnerLogger(true)
);
t.deepEqual(Object.keys(customStatusReport).length, 1);
t.deepEqual(Object.keys(customStatusReport).length, 2);
t.true(
`analyze_custom_queries_${language}_duration_ms` in customStatusReport
);
@ -166,6 +174,7 @@ test("status report fields and search path setting", async (t) => {
? [undefined, undefined, "/1", "/2", undefined]
: [undefined, "/1", "/2"];
t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
t.true(`interpret_results_${language}_duration_ms` in customStatusReport);
}
verifyLineCounts(tmpDir);
@ -177,12 +186,7 @@ test("status report fields and search path setting", async (t) => {
Object.keys(Language).forEach((lang, i) => {
verifyLineCountForFile(
lang as Language,
path.join(tmpDir, `${lang}-builtin.sarif`),
i + 1
);
verifyLineCountForFile(
lang as Language,
path.join(tmpDir, `${lang}-custom.sarif`),
path.join(tmpDir, `${lang}.sarif`),
i + 1
);
});

189
src/analyze.ts
View file

@ -10,7 +10,6 @@ import { countLoc, getIdPrefix } from "./count-loc";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
import { combineSarifFiles } from "./upload-lib";
import * as util from "./util";
export class CodeQLAnalysisError extends Error {
@ -25,34 +24,48 @@ export class CodeQLAnalysisError extends Error {
}
export interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze builtin queries for ruby (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for ruby (or undefined if this language was not analyzed)
analyze_builtin_queries_ruby_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for ruby (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for ruby (or undefined if this language was not analyzed)
analyze_custom_queries_ruby_duration_ms?: number;
// Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed)
interpret_results_cpp_duration_ms?: number;
// Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed)
interpret_results_csharp_duration_ms?: number;
// Time taken in ms to interpret results for go (or undefined if this language was not analyzed)
interpret_results_go_duration_ms?: number;
// Time taken in ms to interpret results for java (or undefined if this language was not analyzed)
interpret_results_java_duration_ms?: number;
// Time taken in ms to interpret results for javascript (or undefined if this language was not analyzed)
interpret_results_javascript_duration_ms?: number;
// Time taken in ms to interpret results for python (or undefined if this language was not analyzed)
interpret_results_python_duration_ms?: number;
// Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed)
interpret_results_ruby_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no language failed)
analyze_failure_language?: string;
}
@ -163,7 +176,7 @@ export async function runQueries(
);
for (const language of config.languages) {
logger.startGroup(`Analyzing ${language}`);
logger.startGroup(`Running queries for ${language}`);
const queries = config.queries[language];
const packsWithVersion = config.packs[language] || [];
@ -188,82 +201,65 @@ export async function runQueries(
);
}
let analysisSummaryBuiltIn = "";
const customAnalysisSummaries: string[] = [];
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
const { sarifFile, stdout } = await runQueryGroup(
language,
"builtin",
createQuerySuiteContents(queries["builtin"]),
sarifFolder,
undefined
querySuitePaths.push(
await runQueryGroup(
language,
"builtin",
createQuerySuiteContents(queries["builtin"]),
undefined
)
);
analysisSummaryBuiltIn = stdout;
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = config.tempDir;
const temporarySarifFiles: string[] = [];
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
const { sarifFile, stdout } = await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(queries["custom"][i].queries),
temporarySarifDir,
queries["custom"][i].searchPath
querySuitePaths.push(
await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(queries["custom"][i].queries),
queries["custom"][i].searchPath
)
);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
ranCustom = true;
}
}
if (packsWithVersion.length > 0) {
const { sarifFile, stdout } = await runQueryGroup(
language,
"packs",
createPackSuiteContents(packsWithVersion),
temporarySarifDir,
undefined
querySuitePaths.push(
await runQueryGroup(
language,
"packs",
createPackSuiteContents(packsWithVersion),
undefined
)
);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
ranCustom = true;
}
if (temporarySarifFiles.length > 0) {
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
fs.writeFileSync(sarifFile, combineSarifFiles(temporarySarifFiles));
await injectLinesOfCode(sarifFile, language, locPromise);
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
// Print the LoC baseline and the summary results from database analyze for the standard
// query suite and (if appropriate) each custom query suite.
logger.startGroup(`Analysis summary for ${language}`);
printLinesOfCodeSummary(logger, language, await locPromise);
logger.info(analysisSummaryBuiltIn);
for (const [i, customSummary] of customAnalysisSummaries.entries()) {
if (customSummary.trim() === "") {
continue;
}
const description =
customAnalysisSummaries.length === 1
? "custom queries"
: `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
logger.info(`Analysis summary for ${description}:`);
logger.info("");
logger.info(customSummary);
logger.info("");
}
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(
language,
querySuitePaths,
sarifFile
);
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
printLinesOfCodeSummary(logger, language, await locPromise);
} catch (e) {
logger.info(e);
logger.info(e.stack);
@ -277,13 +273,29 @@ export async function runQueries(
return statusReport;
async function runInterpretResults(
language: Language,
queries: string[],
sarifFile: string
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = getCodeQL(config.codeQLCmd);
return await codeql.databaseInterpretResults(
databasePath,
queries,
sarifFile,
addSnippetsFlag,
threadsFlag,
automationDetailsId
);
}
async function runQueryGroup(
language: Language,
type: string,
querySuiteContents: string,
destinationFolder: string,
searchPath: string | undefined
): Promise<{ sarifFile: string; stdout: string }> {
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
@ -293,24 +305,17 @@ export async function runQueries(
`Query suite file for ${language}-${type}...\n${querySuiteContents}`
);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
const codeql = getCodeQL(config.codeQLCmd);
const databaseAnalyzeStdout = await codeql.databaseAnalyze(
await codeql.databaseRunQueries(
databasePath,
sarifFile,
searchPath,
querySuitePath,
memoryFlag,
addSnippetsFlag,
threadsFlag,
automationDetailsId
threadsFlag
);
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
return { sarifFile, stdout: databaseAnalyzeStdout };
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
}
@ -365,6 +370,20 @@ export async function runAnalyze(
return { ...queriesStats };
}
export async function runCleanup(
config: configUtils.Config,
cleanupLevel: string,
logger: Logger
): Promise<void> {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {
const codeql = getCodeQL(config.codeQLCmd);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
logger.endGroup();
}
async function injectLinesOfCode(
sarifFile: string,
language: Language,

5
src/api-client.test.ts
View file

@ -11,10 +11,13 @@ const pkg = require("../package.json");
setupTests(test);
let pluginStub: sinon.SinonStub;
let githubStub: sinon.SinonStub;
test.beforeEach(() => {
githubStub = sinon.stub(githubUtils, "GitHub");
pluginStub = sinon.stub(githubUtils.GitHub, "plugin");
githubStub = sinon.stub();
pluginStub.returns(githubStub);
initializeEnvironment(Mode.actions, pkg.version);
});

4
src/api-client.ts
View file

@ -1,6 +1,7 @@
import * as path from "path";
import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getRequiredInput } from "./actions-util";
@ -33,7 +34,8 @@ export const getApiClient = function (
) {
const auth =
(allowExternal && apiDetails.externalRepoAuth) || apiDetails.auth;
return new githubUtils.GitHub(
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
return new retryingOctokit(
githubUtils.getOctokitOptions(auth, {
baseUrl: getApiUrl(apiDetails.url),
userAgent: `CodeQL-${getMode()}/${pkg.version}`,

86
src/codeql.ts
View file

@ -96,14 +96,26 @@ export interface CodeQL {
packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput>;
/**
* Run 'codeql database analyze'.
* Run 'codeql database cleanup'.
*/
databaseAnalyze(
databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
/**
* Run 'codeql database run-queries'.
*/
databaseRunQueries(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
querySuitePath: string,
memoryFlag: string,
threadsFlag: string
): Promise<void>;
/**
* Run 'codeql database interpret-results'.
*/
databaseInterpretResults(
databasePath: string,
querySuitePaths: string[],
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
automationDetailsId: string | undefined
@ -498,8 +510,13 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
databaseInterpretResults: resolveFunction(
partialCodeql,
"databaseInterpretResults"
),
};
return cachedCodeQL;
}
@ -667,6 +684,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
[
"database",
"finalize",
"--finalize-dataset",
threadsFlag,
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
@ -722,39 +740,53 @@ function getCodeQLForCmd(cmd: string): CodeQL {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseAnalyze(
async databaseRunQueries(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
querySuitePath: string,
memoryFlag: string,
threadsFlag: string
): Promise<void> {
const args = [
"database",
"run-queries",
memoryFlag,
threadsFlag,
databasePath,
"--min-disk-free=1024", // Try to leave at least 1GB free
"-v",
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (extraSearchPath !== undefined) {
args.push("--additional-packs", extraSearchPath);
}
args.push(querySuitePath);
await new toolrunner.ToolRunner(cmd, args).exec();
},
async databaseInterpretResults(
databasePath: string,
querySuitePaths: string[],
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
automationDetailsId: string | undefined
): Promise<string> {
const args = [
"database",
"analyze",
memoryFlag,
"interpret-results",
threadsFlag,
databasePath,
"--min-disk-free=1024", // Try to leave at least 1GB free
"--format=sarif-latest",
"--sarif-multicause-markdown",
"--print-metrics-summary",
"--sarif-group-rules-by-pack",
"-v",
`--output=${sarifFile}`,
addSnippetsFlag,
// Enable progress verbosity so we log each query as it's interpreted. This aids debugging
// when interpretation takes a while for one of the queries being analyzed.
"-v",
...getExtraOptionsFromEnv(["database", "analyze"]),
...getExtraOptionsFromEnv(["database", "interpret-results"]),
];
if (extraSearchPath !== undefined) {
args.push("--additional-packs", extraSearchPath);
}
if (automationDetailsId !== undefined) {
args.push("--sarif-category", automationDetailsId);
}
args.push(querySuite);
args.push(databasePath, ...querySuitePaths);
// capture stdout, which contains analysis summaries
let output = "";
await new toolrunner.ToolRunner(cmd, args, {
@ -814,6 +846,18 @@ function getCodeQLForCmd(cmd: string): CodeQL {
);
}
},
async databaseCleanup(
databasePath: string,
cleanupLevel: string
): Promise<void> {
const args = [
"database",
"cleanup",
databasePath,
`--mode=${cleanupLevel}`,
];
await new toolrunner.ToolRunner(cmd, args).exec();
},
};
}

105
src/fingerprints.test.ts
View file

@ -7,32 +7,41 @@ import test from "ava";
import * as fingerprints from "./fingerprints";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
function testHash(t: ava.Assertions, input: string, expectedHashes: string[]) {
let index = 0;
const callback = function (lineNumber: number, hash: string) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
fingerprints.hash(callback, input);
t.is(index, input.split(/\r\n|\r|\n/).length);
async function testHash(
t: ava.Assertions,
input: string,
expectedHashes: string[]
) {
await util.withTmpDir(async (tmpDir) => {
const tmpFile = path.resolve(tmpDir, "testfile");
fs.writeFileSync(tmpFile, input);
let index = 0;
const callback = function (lineNumber: number, hash: string) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
await fingerprints.hash(callback, tmpFile);
t.is(index, input.split(/\r\n|\r|\n/).length);
});
}
test("hash", (t: ava.Assertions) => {
test("hash", async (t: ava.Assertions) => {
// Try empty file
testHash(t, "", ["c129715d7a2bc9a3:1"]);
await testHash(t, "", ["c129715d7a2bc9a3:1"]);
// Try various combinations of newline characters
testHash(t, " a\nb\n \t\tc\n d", [
await testHash(t, " a\nb\n \t\tc\n d", [
"271789c17abda88f:1",
"54703d4cd895b18:1",
"180aee12dab6264:1",
"a23a3dc5e078b07b:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
"8b7cf3e952e7aeb2:1",
"b1ae1287ec4718d9:1",
"bff680108adb0fcc:1",
@ -40,7 +49,7 @@ test("hash", (t: ava.Assertions) => {
"b86d3392aea1be30:1",
"e6ceba753e1a442:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -49,7 +58,7 @@ test("hash", (t: ava.Assertions) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
await testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -58,16 +67,20 @@ test("hash", (t: ava.Assertions) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
await testHash(
t,
" hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n",
[
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]
);
await testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@ -78,7 +91,7 @@ test("hash", (t: ava.Assertions) => {
]);
// Try repeating line that will generate identical hashes
testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
await testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
"a7f2ff13bc495cf2:1",
"a7f2ff13bc495cf2:2",
"a7f2ff13bc495cf2:3",
@ -92,16 +105,20 @@ test("hash", (t: ava.Assertions) => {
"c129715d7a2bc9a3:1",
]);
testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
"5c482a0f8b35ea28:1",
"54517377da7028d2:1",
"2c644846cb18d53e:1",
"f1b89f20de0d133:1",
"c129715d7a2bc9a3:1",
]);
await testHash(
t,
"x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n",
[
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
"5c482a0f8b35ea28:1",
"54517377da7028d2:1",
"2c644846cb18d53e:1",
"f1b89f20de0d133:1",
"c129715d7a2bc9a3:1",
]
);
});
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
@ -170,7 +187,7 @@ test("resolveUriToFile", (t) => {
t.is(testResolveUriToFile(`file://${dirpath}`, undefined, []), undefined);
});
test("addFingerprints", (t) => {
test("addFingerprints", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
@ -187,12 +204,16 @@ test("addFingerprints", (t) => {
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(
fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
await fingerprints.addFingerprints(
input,
checkoutPath,
getRunnerLogger(true)
),
expected
);
});
test("missingRegions", (t) => {
test("missingRegions", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
@ -209,7 +230,11 @@ test("missingRegions", (t) => {
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(
fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
await fingerprints.addFingerprints(
input,
checkoutPath,
getRunnerLogger(true)
),
expected
);
});

30
src/fingerprints.ts
View file

@ -8,6 +8,7 @@ const tab = "\t".charCodeAt(0);
const space = " ".charCodeAt(0);
const lf = "\n".charCodeAt(0);
const cr = "\r".charCodeAt(0);
const EOF = 65535;
const BLOCK_SIZE = 100;
const MOD = Long.fromInt(37); // L
@ -34,9 +35,9 @@ type hashCallback = (lineNumber: number, hash: string) => void;
* the hashes of the lines near the end of the file.
*
* @param callback function that is called with the line number (1-based) and hash for every line
* @param input The file's contents
* @param filepath The path to the file to hash
*/
export function hash(callback: hashCallback, input: string) {
export async function hash(callback: hashCallback, filepath: string) {
// A rolling view in to the input
const window = Array(BLOCK_SIZE).fill(0);
@ -87,12 +88,11 @@ export function hash(callback: hashCallback, input: string) {
// as we go. Once we reach a point in the window again then we've processed
// BLOCK_SIZE characters and if the last character at this point in the window
// was the start of a line then we should output the hash for that line.
for (let i = 0, len = input.length; i <= len; i++) {
let current = i === len ? 65535 : input.charCodeAt(i);
const processCharacter = function (current: number) {
// skip tabs, spaces, and line feeds that come directly after a carriage return
if (current === space || current === tab || (prevCR && current === lf)) {
prevCR = false;
continue;
return;
}
// replace CR with LF
if (current === cr) {
@ -113,7 +113,15 @@ export function hash(callback: hashCallback, input: string) {
lineStart = true;
}
updateHash(current);
};
const readStream = fs.createReadStream(filepath, "utf8");
for await (const data of readStream) {
for (let i = 0; i < data.length; ++i) {
processCharacter(data.charCodeAt(i));
}
}
processCharacter(EOF);
// Flush the remaining lines
for (let i = 0; i < BLOCK_SIZE; i++) {
@ -237,11 +245,11 @@ export function resolveUriToFile(
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(
export async function addFingerprints(
sarifContents: string,
checkoutPath: string,
logger: Logger
): string {
): Promise<string> {
const sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
@ -263,6 +271,11 @@ export function addFingerprints(
continue;
}
if (primaryLocation?.physicalLocation?.region?.startLine === undefined) {
// Locations without a line number are unlikely to be source files
continue;
}
const filepath = resolveUriToFile(
primaryLocation.physicalLocation.artifactLocation,
artifacts,
@ -289,8 +302,7 @@ export function addFingerprints(
c(lineNumber, hashValue);
}
};
const fileContents = fs.readFileSync(filepath).toString();
hash(teeCallback, fileContents);
await hash(teeCallback, filepath);
}
return JSON.stringify(sarif);

4
src/testdata/fingerprinting2.expected.sarif vendored
View file

@ -30,9 +30,7 @@
"message": {
"text": "This header file should contain a header guard to prevent multiple inclusion."
},
"partialFingerprints": {
"primaryLocationLineHash": "599c824c91d0f75e:1"
},
"partialFingerprints": {},
"ruleId": "cpp/missing-header-guard",
"ruleIndex": 0
}

2
src/upload-lib.ts
View file

@ -356,7 +356,7 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(
sarifPayload = await fingerprints.addFingerprints(
sarifPayload,
checkoutPath,
logger