robojerk/codeql-action

Merge pull request #533 from github/update-v1.0.0-ff75ec7f

Browse source 
Merge main into v1
This commit is contained in:
Andrew Eisenberg 2021-05-31 08:13:07 -07:00 • committed by GitHub
commit 4917d3c7e8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 350 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

78
.github/update-release-branch.py vendored
View file

@ -4,6 +4,15 @@ import random
import requests
import subprocess
import sys
import json
import datetime
import os
EMPTY_CHANGELOG = """# CodeQL Action and CodeQL Runner Changelog
## [UNRELEASED]
"""
# The branch being merged from.
# This is the one that contains day-to-day development work.
@ -49,32 +58,40 @@ def open_pr(repo, all_commits, short_main_sha, branch_name):
commits_without_pull_requests = sorted(commits_without_pull_requests, key=lambda c: c.commit.author.date)
# Start constructing the body text
body = 'Merging ' + short_main_sha + ' into ' + LATEST_RELEASE_BRANCH
body = []
body.append('Merging ' + short_main_sha + ' into ' + LATEST_RELEASE_BRANCH)
conductor = get_conductor(repo, pull_requests, commits_without_pull_requests)
body += '\n\nConductor for this PR is @' + conductor
body.append('')
body.append('Conductor for this PR is @' + conductor)
# List all PRs merged
if len(pull_requests) > 0:
body += '\n\nContains the following pull requests:'
body.append('')
body.append('Contains the following pull requests:')
for pr in pull_requests:
merger = get_merger_of_pr(repo, pr)
body += '\n- #' + str(pr.number)
body += ' - ' + pr.title
body += ' (@' + merger + ')'
body.append('- #' + str(pr.number) + ' - ' + pr.title +' (@' + merger + ')')
# List all commits not part of a PR
if len(commits_without_pull_requests) > 0:
body += '\n\nContains the following commits not from a pull request:'
body.append('')
body.append('Contains the following commits not from a pull request:')
for commit in commits_without_pull_requests:
body += '\n- ' + commit.sha
body += ' - ' + get_truncated_commit_message(commit)
body += ' (@' + commit.author.login + ')'
body.append('- ' + commit.sha + ' - ' + get_truncated_commit_message(commit) + ' (@' + commit.author.login + ')')
body.append('')
body.append('Please review the following:')
body.append(' - [ ] The CHANGELOG displays the correct version and date.')
body.append(' - [ ] The CHANGELOG includes all relevant, user-facing changes since the last release.')
body.append(' - [ ] There are no unexpected commits being merged into the ' + LATEST_RELEASE_BRANCH + ' branch.')
body.append(' - [ ] The docs team is aware of any documentation changes that need to be released.')
body.append(' - [ ] The mergeback PR is merged back into ' + MAIN_BRANCH + ' after this PR is merged.')
title = 'Merge ' + MAIN_BRANCH + ' into ' + LATEST_RELEASE_BRANCH
# Create the pull request
pr = repo.create_pull(title=title, body=body, head=branch_name, base=LATEST_RELEASE_BRANCH)
pr = repo.create_pull(title=title, body='\n'.join(body), head=branch_name, base=LATEST_RELEASE_BRANCH)
print('Created PR #' + str(pr.number))
# Assign the conductor
@ -95,7 +112,7 @@ def get_conductor(repo, pull_requests, other_commits):
# This will not include any commits that exist on the release branch
# that aren't on main.
def get_commit_difference(repo):
commits = run_git('log', '--pretty=format:%H', ORIGIN + '/' + LATEST_RELEASE_BRANCH + '..' + MAIN_BRANCH).strip().split('\n')
commits = run_git('log', '--pretty=format:%H', ORIGIN + '/' + LATEST_RELEASE_BRANCH + '..' + ORIGIN + '/' + MAIN_BRANCH).strip().split('\n')
# Convert to full-fledged commit objects
commits = [repo.get_commit(c) for c in commits]
@ -135,6 +152,28 @@ def get_pr_for_commit(repo, commit):
def get_merger_of_pr(repo, pr):
return repo.get_commit(pr.merge_commit_sha).author.login
def get_current_version():
with open('package.json', 'r') as f:
return json.load(f)['version']
def get_today_string():
today = datetime.datetime.today()
return '{:%d %b %Y}'.format(today)
def update_changelog(version):
if (os.path.exists('CHANGELOG.md')):
content = ''
with open('CHANGELOG.md', 'r') as f:
content = f.read()
else:
content = EMPTY_CHANGELOG
newContent = content.replace('[UNRELEASED]', version + ' - ' + get_today_string(), 1)
with open('CHANGELOG.md', 'w') as f:
f.write(newContent)
def main():
if len(sys.argv) != 3:
raise Exception('Usage: update-release.branch.py <github token> <repository nwo>')
@ -142,10 +181,11 @@ def main():
repository_nwo = sys.argv[2]
repo = Github(github_token).get_repo(repository_nwo)
version = get_current_version()
# Print what we intend to go
print('Considering difference between ' + MAIN_BRANCH + ' and ' + LATEST_RELEASE_BRANCH)
short_main_sha = run_git('rev-parse', '--short', MAIN_BRANCH).strip()
short_main_sha = run_git('rev-parse', '--short', ORIGIN + '/' + MAIN_BRANCH).strip()
print('Current head of ' + MAIN_BRANCH + ' is ' + short_main_sha)
# See if there are any commits to merge in
@ -157,7 +197,7 @@ def main():
# The branch name is based off of the name of branch being merged into
# and the SHA of the branch being merged from. Thus if the branch already
# exists we can assume we don't need to recreate it.
new_branch_name = 'update-' + LATEST_RELEASE_BRANCH + '-' + short_main_sha
new_branch_name = 'update-v' + version + '-' + short_main_sha
print('Branch name is ' + new_branch_name)
# Check if the branch already exists. If so we can abort as this script
@ -168,7 +208,15 @@ def main():
# Create the new branch and push it to the remote
print('Creating branch ' + new_branch_name)
run_git('checkout', '-b', new_branch_name, MAIN_BRANCH)
run_git('checkout', '-b', new_branch_name, ORIGIN + '/' + MAIN_BRANCH)
print('Updating changelog')
update_changelog(version)
# Create a commit that updates the CHANGELOG
run_git('add', 'CHANGELOG.md')
run_git('commit', '-m', version)
run_git('push', ORIGIN, new_branch_name)
# Open a PR to update the branch

124
.github/workflows/post-release-mergeback.yml vendored Normal file
View file

@ -0,0 +1,124 @@
# This workflow runs after a release of the action.
# It merges any changes from the release back into the
# main branch. Typically, this is just a single commit
# that updates the changelog.
name: Tag release and merge back
on:
workflow_dispatch:
inputs:
baseBranch:
description: 'The base branch to merge into'
default: main
required: false
push:
branches:
- v1
pull_request:
paths:
- .github/workflows/post-release-mergeback.yml
jobs:
merge-back:
runs-on: ubuntu-latest
if: github.repository == 'github/codeql-action'
env:
BASE_BRANCH: "${{ github.event.inputs.baseBranch || 'main' }}"
HEAD_BRANCH: "${{ github.head_ref || github.ref }}"
steps:
- name: Dump GitHub Event context
env:
GITHUB_EVENT_CONTEXT: "${{ toJson(github.event) }}"
run: echo "$GITHUB_EVENT_CONTEXT"
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
- name: Update git config
run: |
git config --global user.email "github-actions@github.com"
git config --global user.name "github-actions[bot]"
- name: Get version and new branch
id: getVersion
run: |
VERSION="v$(jq '.version' -r 'package.json')"
SHORT_SHA="${GITHUB_SHA:0:8}"
echo "::set-output name=version::$VERSION"
NEW_BRANCH="mergeback/${VERSION}-to-${BASE_BRANCH}-${SHORT_SHA}"
echo "::set-output name=newBranch::$NEW_BRANCH"
- name: Dump branches
env:
NEW_BRANCH: "${{ steps.getVersion.outputs.newBranch }}"
run: |
echo "BASE_BRANCH $BASE_BRANCH"
echo "HEAD_BRANCH $HEAD_BRANCH"
echo "NEW_BRANCH $NEW_BRANCH"
- name: Create mergeback branch
env:
NEW_BRANCH: "${{ steps.getVersion.outputs.newBranch }}"
run: |
git checkout -b "$NEW_BRANCH"
- name: Check for tag
id: check
env:
VERSION: "${{ steps.getVersion.outputs.version }}"
run: |
set +e # don't fail on an errored command
git ls-remote --tags origin | grep "$VERSION"
EXISTS="$?"
if [ "$EXISTS" -ne 0 ]; then
echo "::set-output name=exists::true"
echo "Tag $TAG exists. Not going to re-release."
fi
# we didn't tag the release during the update-release-branch workflow because the
# commit that actually makes it to the release branch is a merge commit,
# and not yet known during the first workflow. We tag now because we know the correct commit.
- name: Tag release
if: steps.check.outputs.exists == 'true'
env:
VERSION: ${{ steps.getVersion.outputs.version }}
run: |
git tag -a "$VERSION" -m "$VERSION"
git push origin --follow-tags "$VERSION"
- name: Create mergeback branch
if: steps.check.outputs.exists == 'true'
env:
VERSION: "${{ steps.getVersion.outputs.version }}"
NEW_BRANCH: "${{ steps.getVersion.outputs.newBranch }}"
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
run: |
set -exu
PR_TITLE="Mergeback $VERSION $HEAD_BRANCH into $BASE_BRANCH"
PR_BODY="Updates version and changelog."
# Update the changelog
perl -i -pe 's/^/## \[UNRELEASED\]\n\n/ if($.==3)' CHANGELOG.md
git add .
git commit -m "Update changelog and version after $VERSION"
npm version patch
# when running this workflow on a PR, this is just a test.
# so put into draft mode.
if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
DRAFT="--draft"
else
DRAFT=""
fi
git push origin "$NEW_BRANCH"
gh pr create \
--head "$NEW_BRANCH" \
--base "$BASE_BRANCH" \
--title "$PR_TITLE" \
--body "$PR_BODY" \
"$DRAFT"

7
.github/workflows/update-release-branch.yml vendored
View file

@ -22,12 +22,17 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.5
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install PyGithub==1.51 requests
- name: Update git config
run: |
git config --global user.email "github-actions@github.com"
git config --global user.name "github-actions[bot]"
- name: Update release branch
run: python .github/update-release-branch.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }}

1
.github/workflows/update-supported-enterprise-server-versions.yml vendored
View file

@ -7,6 +7,7 @@ on:
jobs:
update-supported-enterprise-server-versions:
runs-on: ubuntu-latest
if: ${{ github.repository == 'github/codeql-action' }}
steps:
- name: Setup Python

5
CHANGELOG.md
View file

@ -1,6 +1,7 @@
# CodeQL Action and CodeQL Runner Changelog
## [UNRELEASED]
## 1.0.0 - 31 May 2021
- Add this changelog file. [#507](https://github.com/github/codeql-action/pull/507)
- Improve grouping of analysis logs. Add a new log group containing a summary of metrics and diagnostics, if they were produced by CodeQL builtin queries. [#515](https://github.com/github/codeql-action/pull/515)
- Improve grouping of analysis logs. Add a new log group containing a summary of metrics and diagnostics, if they were produced by CodeQL builtin queries. [#515](https://github.com/github/codeql-action/pull/515)
- Add metrics and diagnostics summaries from custom query suites to the analysis summary log group. [#532](https://github.com/github/codeql-action/pull/532)

29
lib/analyze.js generated
View file

@ -92,21 +92,23 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
}
try {
let analysisSummary = "";
let analysisSummaryBuiltIn = "";
const customAnalysisSummaries = [];
if (queries["builtin"].length > 0) {
const startTimeBuliltIn = new Date().getTime();
const startTimeBuiltIn = new Date().getTime();
const { sarifFile, stdout } = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
analysisSummary = stdout;
analysisSummaryBuiltIn = stdout;
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuliltIn;
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = config.tempDir;
const temporarySarifFiles = [];
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
const { sarifFile } = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
const { sarifFile, stdout } = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
}
}
@ -118,10 +120,23 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
// Print the LoC baseline and the summary results from database analyze.
// Print the LoC baseline and the summary results from database analyze for the standard
// query suite and (if appropriate) each custom query suite.
logger.startGroup(`Analysis summary for ${language}`);
printLinesOfCodeSummary(logger, language, await locPromise);
logger.info(analysisSummary);
logger.info(analysisSummaryBuiltIn);
for (const [i, customSummary] of customAnalysisSummaries.entries()) {
if (customSummary.trim() === "") {
continue;
}
const description = customAnalysisSummaries.length === 1
? "custom queries"
: `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
logger.info(`Analysis summary for ${description}:`);
logger.info("");
logger.info(customSummary);
logger.info("");
}
logger.endGroup();
}
catch (e) {

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

25
lib/codeql.js generated
View file

@ -282,6 +282,7 @@ function setCodeQL(partialCodeql) {
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(partialCodeql, "extractScannedLanguage"),
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
};
@ -418,6 +419,23 @@ function getCodeQLForCmd(cmd) {
databasePath,
], error_matcher_1.errorMatchers);
},
async resolveLanguages() {
const codeqlArgs = ["resolve", "languages", "--format=json"];
let output = "";
await new toolrunner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data) => {
output += data.toString();
},
},
}).exec();
try {
return JSON.parse(output);
}
catch (e) {
throw new Error(`Unexpected output from codeql resolve languages: ${e}`);
}
},
async resolveQueries(queries, extraSearchPath) {
const codeqlArgs = [
"resolve",
@ -437,7 +455,12 @@ function getCodeQLForCmd(cmd) {
},
},
}).exec();
return JSON.parse(output);
try {
return JSON.parse(output);
}
catch (e) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseAnalyze(databasePath, sarifFile, extraSearchPath, querySuite, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId) {
const args = [

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

8
lib/config-utils.js generated
View file

@ -336,7 +336,7 @@ async function getLanguagesInRepo(repository, apiDetails, logger) {
* If no languages could be detected from either the workflow or the repository
* then throw an error.
*/
async function getLanguages(languagesInput, repository, apiDetails, logger) {
async function getLanguages(codeQL, languagesInput, repository, apiDetails, logger) {
// Obtain from action input 'languages' if set
let languages = (languagesInput || "")
.split(",")
@ -346,6 +346,8 @@ async function getLanguages(languagesInput, repository, apiDetails, logger) {
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
languages = await getLanguagesInRepo(repository, apiDetails, logger);
const availableLanguages = await codeQL.resolveLanguages();
languages = languages.filter((value) => value in availableLanguages);
logger.info(`Automatically detected languages: ${JSON.stringify(languages)}`);
}
// If the languages parameter was not given and no languages were
@ -392,7 +394,7 @@ function shouldAddConfigFileQueries(queriesInput) {
* Get the default config for when the user has not supplied one.
*/
async function getDefaultConfig(languagesInput, queriesInput, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
const languages = await getLanguages(languagesInput, repository, apiDetails, logger);
const languages = await getLanguages(codeQL, languagesInput, repository, apiDetails, logger);
const queries = {};
for (const language of languages) {
queries[language] = {
@ -441,7 +443,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
throw new Error(getNameInvalid(configFile));
}
}
const languages = await getLanguages(languagesInput, repository, apiDetails, logger);
const languages = await getLanguages(codeQL, languagesInput, repository, apiDetails, logger);
const queries = {};
for (const language of languages) {
queries[language] = {

2
lib/config-utils.js.map
View file

File diff suppressed because one or more lines are too long

11
lib/config-utils.test.js generated
View file

@ -531,8 +531,13 @@ ava_1.default("Invalid format of remote config handled correctly", async (t) =>
ava_1.default("No detected languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
mockListLanguages([]);
const codeQL = codeql_1.setCodeQL({
async resolveLanguages() {
return {};
},
});
try {
await configUtils.initConfig(undefined, undefined, undefined, undefined, { owner: "github", repo: "example " }, tmpDir, tmpDir, codeql_1.getCachedCodeQL(), tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
await configUtils.initConfig(undefined, undefined, undefined, undefined, { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
throw new Error("initConfig did not throw error");
}
catch (err) {
@ -542,13 +547,13 @@ ava_1.default("No detected languages", async (t) => {
});
ava_1.default("Unknown languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const languages = "ruby,english";
const languages = "rubbish,english";
try {
await configUtils.initConfig(languages, undefined, undefined, undefined, { owner: "github", repo: "example " }, tmpDir, tmpDir, codeql_1.getCachedCodeQL(), tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
throw new Error("initConfig did not throw error");
}
catch (err) {
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(["ruby", "english"])));
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(["rubbish", "english"])));
}
});
});

2
lib/config-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

3
lib/count-loc.js generated
View file

@ -12,6 +12,7 @@ const linguistToMetrics = {
java: languages_1.Language.java,
javascript: languages_1.Language.javascript,
python: languages_1.Language.python,
ruby: languages_1.Language.ruby,
typescript: languages_1.Language.javascript,
};
const nameToLinguist = Object.entries(linguistToMetrics).reduce((obj, [key, name]) => {
@ -35,6 +36,8 @@ function getIdPrefix(language) {
return "js";
case languages_1.Language.python:
return "py";
case languages_1.Language.ruby:
return "rb";
default:
util_1.assertNever(language);
}

2
lib/count-loc.js.map
View file

@ -1 +1 @@
{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AAlBD,kCAkBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,4EAA4E;YAC1E,0EAA0E;YAC1E,sEAAsE;YACtE,4EAA4E;YAC5E,iEAAiE;YACjE,wEAAwE;YACxE,gFAAgF;YAChF,yBAAyB,CAC5B,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA/CD,4BA+CC"}
{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,4EAA4E;YAC1E,0EAA0E;YAC1E,sEAAsE;YACtE,4EAA4E;YAC5E,iEAAiE;YACjE,wEAAwE;YACxE,gFAAgF;YAChF,yBAAyB,CAC5B,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA/CD,4BA+CC"}

1
lib/languages.js generated
View file

@ -9,6 +9,7 @@ var Language;
Language["java"] = "java";
Language["javascript"] = "javascript";
Language["python"] = "python";
Language["ruby"] = "ruby";
})(Language = exports.Language || (exports.Language = {}));
// Additional names for languages
const LANGUAGE_ALIASES = {

2
lib/languages.js.map
View file

@ -1 +1 @@
{"version":3,"file":"languages.js","sourceRoot":"","sources":["../src/languages.ts"],"names":[],"mappings":";;AAAA,wCAAwC;AACxC,IAAY,QAOX;AAPD,WAAY,QAAQ;IAClB,6BAAiB,CAAA;IACjB,uBAAW,CAAA;IACX,qBAAS,CAAA;IACT,yBAAa,CAAA;IACb,qCAAyB,CAAA;IACzB,6BAAiB,CAAA;AACnB,CAAC,EAPW,QAAQ,GAAR,gBAAQ,KAAR,gBAAQ,QAOnB;AAED,iCAAiC;AACjC,MAAM,gBAAgB,GAAiC;IACrD,CAAC,EAAE,QAAQ,CAAC,GAAG;IACf,KAAK,EAAE,QAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,QAAQ,CAAC,MAAM;IACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,gGAAgG;AAChG,SAAgB,aAAa,CAAC,QAAgB;IAC5C,0BAA0B;IAC1B,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAElC,6BAA6B;IAC7B,IAAI,QAAQ,IAAI,QAAQ,EAAE;QACxB,OAAO,QAAoB,CAAC;KAC7B;IAED,yBAAyB;IACzB,IAAI,QAAQ,IAAI,gBAAgB,EAAE;QAChC,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;KACnC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAfD,sCAeC;AAED,SAAgB,gBAAgB,CAAC,QAAkB;IACjD,OAAO,CACL,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC5C,CAAC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,KAAK,IAAI;YACxD,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC,CAC5B,CAAC;AACJ,CAAC;AAND,4CAMC;AAED,SAAgB,iBAAiB,CAAC,QAAkB;IAClD,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAFD,8CAEC"}
{"version":3,"file":"languages.js","sourceRoot":"","sources":["../src/languages.ts"],"names":[],"mappings":";;AAAA,wCAAwC;AACxC,IAAY,QAQX;AARD,WAAY,QAAQ;IAClB,6BAAiB,CAAA;IACjB,uBAAW,CAAA;IACX,qBAAS,CAAA;IACT,yBAAa,CAAA;IACb,qCAAyB,CAAA;IACzB,6BAAiB,CAAA;IACjB,yBAAa,CAAA;AACf,CAAC,EARW,QAAQ,GAAR,gBAAQ,KAAR,gBAAQ,QAQnB;AAED,iCAAiC;AACjC,MAAM,gBAAgB,GAAiC;IACrD,CAAC,EAAE,QAAQ,CAAC,GAAG;IACf,KAAK,EAAE,QAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,QAAQ,CAAC,MAAM;IACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,gGAAgG;AAChG,SAAgB,aAAa,CAAC,QAAgB;IAC5C,0BAA0B;IAC1B,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAElC,6BAA6B;IAC7B,IAAI,QAAQ,IAAI,QAAQ,EAAE;QACxB,OAAO,QAAoB,CAAC;KAC7B;IAED,yBAAyB;IACzB,IAAI,QAAQ,IAAI,gBAAgB,EAAE;QAChC,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;KACnC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAfD,sCAeC;AAED,SAAgB,gBAAgB,CAAC,QAAkB;IACjD,OAAO,CACL,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC5C,CAAC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,KAAK,IAAI;YACxD,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC,CAC5B,CAAC;AACJ,CAAC;AAND,4CAMC;AAED,SAAgB,iBAAiB,CAAC,QAAkB;IAClD,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAFD,8CAEC"}

3
package.json
View file

@ -9,7 +9,8 @@
"test-debug": "ava src/** --serial --verbose --timeout=20m",
"lint": "eslint --report-unused-disable-directives --max-warnings=0 . --ext .js,.ts",
"lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --ext .js,.ts --fix",
"removeNPMAbsolutePaths": "removeNPMAbsolutePaths . --force"
"removeNPMAbsolutePaths": "removeNPMAbsolutePaths . --force",
"version": "cd runner && npm version patch && git add ."
},
"ava": {
"typescript": {

2
runner/package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "0.0.0",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
runner/package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "0.0.0",
"version": "1.0.0",
"private": true,
"description": "CodeQL runner",
"scripts": {

37
src/analyze.ts
View file

@ -37,6 +37,8 @@ export interface QueriesStatusReport {
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze builtin queries for ruby (or undefined if this language was not analyzed)
analyze_builtin_queries_ruby_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
@ -49,6 +51,8 @@ export interface QueriesStatusReport {
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for ruby (or undefined if this language was not analyzed)
analyze_custom_queries_ruby_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no language failed)
analyze_failure_language?: string;
}
@ -172,9 +176,10 @@ export async function runQueries(
}
try {
let analysisSummary = "";
let analysisSummaryBuiltIn = "";
const customAnalysisSummaries: string[] = [];
if (queries["builtin"].length > 0) {
const startTimeBuliltIn = new Date().getTime();
const startTimeBuiltIn = new Date().getTime();
const { sarifFile, stdout } = await runQueryGroup(
language,
"builtin",
@ -182,24 +187,25 @@ export async function runQueries(
sarifFolder,
undefined
);
analysisSummary = stdout;
analysisSummaryBuiltIn = stdout;
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuliltIn;
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = config.tempDir;
const temporarySarifFiles: string[] = [];
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
const { sarifFile } = await runQueryGroup(
const { sarifFile, stdout } = await runQueryGroup(
language,
`custom-${i}`,
queries["custom"][i].queries,
temporarySarifDir,
queries["custom"][i].searchPath
);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
}
}
@ -213,10 +219,27 @@ export async function runQueries(
}
logger.endGroup();
// Print the LoC baseline and the summary results from database analyze.
// Print the LoC baseline and the summary results from database analyze for the standard
// query suite and (if appropriate) each custom query suite.
logger.startGroup(`Analysis summary for ${language}`);
printLinesOfCodeSummary(logger, language, await locPromise);
logger.info(analysisSummary);
logger.info(analysisSummaryBuiltIn);
for (const [i, customSummary] of customAnalysisSummaries.entries()) {
if (customSummary.trim() === "") {
continue;
}
const description =
customAnalysisSummaries.length === 1
? "custom queries"
: `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
logger.info(`Analysis summary for ${description}:`);
logger.info("");
logger.info(customSummary);
logger.info("");
}
logger.endGroup();
} catch (e) {
logger.info(e);

34
src/codeql.ts
View file

@ -77,6 +77,10 @@ export interface CodeQL {
* Finalize a database using 'codeql database finalize'.
*/
finalizeDatabase(databasePath: string, threadsFlag: string): Promise<void>;
/**
* Run 'codeql resolve languages'.
*/
resolveLanguages(): Promise<ResolveLanguagesOutput>;
/**
* Run 'codeql resolve queries'.
*/
@ -99,6 +103,10 @@ export interface CodeQL {
): Promise<string>;
}
export interface ResolveLanguagesOutput {
[language: string]: [string];
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
@ -478,6 +486,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
"extractScannedLanguage"
),
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
};
@ -654,6 +663,25 @@ function getCodeQLForCmd(cmd: string): CodeQL {
errorMatchers
);
},
async resolveLanguages() {
const codeqlArgs = ["resolve", "languages", "--format=json"];
let output = "";
await new toolrunner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
}).exec();
try {
return JSON.parse(output);
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages: ${e}`
);
}
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined
@ -677,7 +705,11 @@ function getCodeQLForCmd(cmd: string): CodeQL {
},
}).exec();
return JSON.parse(output);
try {
return JSON.parse(output);
} catch (e) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseAnalyze(
databasePath: string,

11
src/config-utils.test.ts
View file

@ -926,6 +926,11 @@ test("Invalid format of remote config handled correctly", async (t) => {
test("No detected languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
mockListLanguages([]);
const codeQL = setCodeQL({
async resolveLanguages() {
return {};
},
});
try {
await configUtils.initConfig(
@ -936,7 +941,7 @@ test("No detected languages", async (t) => {
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
getCachedCodeQL(),
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
@ -951,7 +956,7 @@ test("No detected languages", async (t) => {
test("Unknown languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const languages = "ruby,english";
const languages = "rubbish,english";
try {
await configUtils.initConfig(
@ -972,7 +977,7 @@ test("Unknown languages", async (t) => {
} catch (err) {
t.deepEqual(
err,
new Error(configUtils.getUnknownLanguagesError(["ruby", "english"]))
new Error(configUtils.getUnknownLanguagesError(["rubbish", "english"]))
);
}
});

5
src/config-utils.ts
View file

@ -653,6 +653,7 @@ async function getLanguagesInRepo(
* then throw an error.
*/
async function getLanguages(
codeQL: CodeQL,
languagesInput: string | undefined,
repository: RepositoryNwo,
apiDetails: api.GitHubApiDetails,
@ -668,6 +669,8 @@ async function getLanguages(
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
languages = await getLanguagesInRepo(repository, apiDetails, logger);
const availableLanguages = await codeQL.resolveLanguages();
languages = languages.filter((value) => value in availableLanguages);
logger.info(
`Automatically detected languages: ${JSON.stringify(languages)}`
);
@ -754,6 +757,7 @@ export async function getDefaultConfig(
logger: Logger
): Promise<Config> {
const languages = await getLanguages(
codeQL,
languagesInput,
repository,
apiDetails,
@ -833,6 +837,7 @@ async function loadConfig(
}
const languages = await getLanguages(
codeQL,
languagesInput,
repository,
apiDetails,

3
src/count-loc.ts
View file

@ -16,6 +16,7 @@ const linguistToMetrics: Record<string, Language> = {
java: Language.java,
javascript: Language.javascript,
python: Language.python,
ruby: Language.ruby,
typescript: Language.javascript,
};
@ -44,6 +45,8 @@ export function getIdPrefix(language: Language): IdPrefix {
return "js";
case Language.python:
return "py";
case Language.ruby:
return "rb";
default:
assertNever(language);

1
src/languages.ts
View file

@ -6,6 +6,7 @@ export enum Language {
java = "java",
javascript = "javascript",
python = "python",
ruby = "ruby",
}
// Additional names for languages