robojerk/codeql-action

Add a CLI interface to the upload-sarif action

Browse source
This commit is contained in:
Robert Brignull 2020-08-11 12:42:42 +01:00
parent bcf676e52d
commit 6d7a135fea
15 changed files with 355 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
/cli/

7
package.json
View file

@ -5,6 +5,7 @@
"description": "CodeQL action",
"scripts": {
"build": "tsc",
"build-cli": "webpack --mode production",
"test": "ava src/** --serial --verbose",
"lint": "tslint -p . -c tslint.json 'src/**/*.ts'",
"removeNPMAbsolutePaths": "removeNPMAbsolutePaths . --force"
@ -23,6 +24,7 @@
"@actions/github": "^2.2.0",
"@actions/http-client": "^1.0.8",
"@actions/tool-cache": "^1.5.5",
"commander": "6.0.0",
"console-log-level": "^1.4.1",
"file-url": "^3.0.0",
"fs": "0.0.1-security",
@ -52,6 +54,9 @@
"sinon": "^9.0.2",
"tslint": "^6.1.0",
"tslint-eslint-rules": "^5.4.0",
"typescript": "^3.7.5"
"ts-loader": "8.0.2",
"typescript": "^3.7.5",
"webpack": "4.44.1",
"webpack-cli": "3.3.12"
}
}

35
src/api-client.ts
View file

@ -2,16 +2,45 @@ import * as core from "@actions/core";
import * as github from "@actions/github";
import consoleLogLevel from "console-log-level";
import { isLocalRun } from "./util";
import { getRequiredEnvParam, isLocalRun } from "./util";
export const getApiClient = function(allowLocalRun = false) {
export const getApiClient = function(githubAuth: string, githubApiUrl: string, allowLocalRun = false) {
if (isLocalRun() && !allowLocalRun) {
throw new Error('Invalid API call in local run');
}
return new github.GitHub(
core.getInput('token'),
{
auth: parseAuth(githubAuth),
baseUrl: githubApiUrl,
userAgent: "CodeQL Action",
log: consoleLogLevel({ level: "debug" })
});
};
// Parses the user input as either a single token,
// or a username and password / PAT.
function parseAuth(auth: string): string {
// Check if it's a username:password pair
const c = auth.indexOf(':');
if (c !== -1) {
return 'basic ' + Buffer.from(auth).toString('base64');
// return {
// username: auth.substring(0, c),
// password: auth.substring(c + 1),
// on2fa: async () => { throw new Error(''); }
// };
}
// Otherwise use the token as it is
return auth;
}
// Temporary function to aid in the transition to running on and off of github actions.
// Once all code has been coverted this function should be removed or made canonical
// and called only from the action entrypoints.
export function getActionsApiClient(allowLocalRun = false) {
return getApiClient(
core.getInput('token'),
getRequiredEnvParam('GITHUB_API_URL'),
allowLocalRun);
}

82
src/cli.ts Normal file
View file

@ -0,0 +1,82 @@
import { Command } from 'commander';
import * as path from 'path';
import { getCLILogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
const program = new Command();
program.version('0.0.1');
interface UploadArgs {
sarifFile: string;
repository: string;
commit: string;
ref: string;
analysisKey: string;
githubUrl: string;
githubAuth: string;
analysisName: string | undefined;
checkoutPath: string | undefined;
environment: string | undefined;
}
function parseGithubApiUrl(inputUrl: string): string {
try {
const url = new URL(inputUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
}
// Add the API path if it's not already present.
if (url.pathname.indexOf('/api/v3') === -1) {
url.pathname = path.join(url.pathname, 'api', 'v3');
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload')
.requiredOption('--repository <repository>', 'Repository name')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed')
.requiredOption('--analysis-key <key>', 'Identifies the analysis, for use matching up equivalent analyses on different commits')
.requiredOption('--github-url <url>', 'URL of GitHub instance')
.requiredOption('--github-auth <auth>', 'GitHub Apps token, or of the form "username:token" if using a personal access token')
.option('--checkout-path <path>', 'Checkout path (default: current working directory)')
.option('--analysis-name <name>', 'Display name of the analysis (default: same as analysis-key')
.option('--environment <env>', 'Environment (default: empty)')
.action(async (cmd: UploadArgs) => {
const logger = getCLILogger();
try {
await upload_lib.upload(
cmd.sarifFile,
parseRepositoryNwo(cmd.repository),
cmd.commit,
cmd.ref,
cmd.analysisKey,
cmd.analysisName || cmd.analysisKey,
undefined,
cmd.checkoutPath || process.cwd(),
cmd.environment,
cmd.githubAuth,
parseGithubApiUrl(cmd.githubUrl),
'cli',
logger);
} catch (e) {
logger.error("Upload failed");
logger.error(e);
}
});
program.parse(process.argv);

2
src/codeql.ts
View file

@ -121,7 +121,7 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
}
let [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getApiClient().repos.getReleaseByTag({
const release = await api.getActionsApiClient().repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION

4
src/config-utils.ts
View file

@ -441,7 +441,7 @@ async function getLanguagesInRepo(): Promise<Language[]> {
let repo = repo_nwo[1];
core.debug(`GitHub repo ${owner} ${repo}`);
const response = await api.getApiClient(true).repos.listLanguages({
const response = await api.getActionsApiClient(true).repos.listLanguages({
owner,
repo
});
@ -689,7 +689,7 @@ async function getRemoteConfig(configFile: string): Promise<UserConfig> {
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
}
const response = await api.getApiClient(true).repos.getContents({
const response = await api.getActionsApiClient(true).repos.getContents({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,

17
src/finalize-db.ts
View file

@ -4,6 +4,8 @@ import * as path from 'path';
import { getCodeQL, isScannedLanguage } from './codeql';
import * as configUtils from './config-utils';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
@ -144,7 +146,20 @@ async function run() {
queriesStats = await runQueries(databaseFolder, sarifFolder, config);
if ('true' === core.getInput('upload')) {
uploadStats = await upload_lib.upload(sarifFolder);
uploadStats = await upload_lib.upload(
sarifFolder,
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
'actions',
getActionsLogger());
}
} catch (error) {

26
src/logging.ts Normal file
View file

@ -0,0 +1,26 @@
import * as core from '@actions/core';
export interface Logger {
debug: (message: string) => void;
info: (message: string) => void;
warning: (message: string) => void;
error: (message: string) => void;
startGroup: (name: string) => void;
endGroup: () => void;
}
export function getActionsLogger(): Logger {
return core;
}
export function getCLILogger(): Logger {
return {
debug: console.debug,
info: console.info,
warning: console.warn,
error: console.error,
startGroup: () => undefined,
endGroup: () => undefined,
};
}

16
src/repository.ts Normal file
View file

@ -0,0 +1,16 @@
// A repository name with owner, parsed into its two parts
export interface RepositoryNwo {
owner: string;
repo: string;
}
export function parseRepositoryNwo(input: string): RepositoryNwo {
const parts = input.split('/');
if (parts.length !== 2) {
throw new Error(`"${input}" is not a valid repository name`);
}
return {
owner: parts[0],
repo: parts[1],
};
}

4
src/testing-utils.ts
View file

@ -55,6 +55,10 @@ export function setupTests(test: TestInterface<any>) {
// process.env only has strings fields, so a shallow copy is fine.
t.context.env = {};
Object.assign(t.context.env, process.env);
// Any test that runs code that expects to only be run on actions
// will depend on various environment variables.
process.env['GITHUB_API_URL'] = 'https://github.localhost/api/v3';
});
typedTest.afterEach.always(t => {

5
src/upload-lib.test.ts
View file

@ -1,5 +1,6 @@
import test from 'ava';
import { getCLILogger } from './logging';
import {setupTests} from './testing-utils';
import * as uploadLib from './upload-lib';
@ -7,10 +8,10 @@ setupTests(test);
test('validateSarifFileSchema - valid', t => {
const inputFile = __dirname + '/../src/testdata/valid-sarif.sarif';
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile));
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile, getCLILogger()));
});
test('validateSarifFileSchema - invalid', t => {
const inputFile = __dirname + '/../src/testdata/invalid-sarif.sarif';
t.throws(() => uploadLib.validateSarifFileSchema(inputFile));
t.throws(() => uploadLib.validateSarifFileSchema(inputFile, getCLILogger()));
});

182
src/upload-lib.ts
View file

@ -1,4 +1,3 @@
import * as core from '@actions/core';
import fileUrl from 'file-url';
import * as fs from 'fs';
import * as jsonschema from 'jsonschema';
@ -7,9 +6,12 @@ import zlib from 'zlib';
import * as api from './api-client';
import * as fingerprints from './fingerprints';
import * as sharedEnv from './shared-environment';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as util from './util';
type UploadMode = 'actions' | 'cli';
// Takes a list of paths to sarif files and combines them together,
// returning the contents of the combined sarif file.
export function combineSarifFiles(sarifFiles: string[]): string {
@ -35,8 +37,15 @@ export function combineSarifFiles(sarifFiles: string[]): string {
// Upload the given payload.
// If the request fails then this will retry a small number of times.
async function uploadPayload(payload) {
core.info('Uploading results');
async function uploadPayload(
payload: any,
repositoryNwo: RepositoryNwo,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
logger: Logger) {
logger.info('Uploading results');
// If in test mode we don't want to upload the results
const testMode = process.env['TEST_MODE'] === 'true' || false;
@ -44,26 +53,29 @@ async function uploadPayload(payload) {
return;
}
const [owner, repo] = util.getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
// Make up to 4 attempts to upload, and sleep for these
// number of seconds between each attempt.
// We don't want to backoff too much to avoid wasting action
// minutes, but just waiting a little bit could maybe help.
const backoffPeriods = [1, 5, 15];
const client = api.getApiClient(githubAuth, githubApiUrl);
for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
const response = await api.getApiClient().request("PUT /repos/:owner/:repo/code-scanning/analysis", ({
owner: owner,
repo: repo,
const reqURL = mode === 'actions'
? 'PUT /repos/:owner/:repo/code-scanning/analysis'
: 'POST /repos/:owner/:repo/code-scanning/sarifs';
const response = await client.request(reqURL, ({
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
data: payload,
}));
core.debug('response status: ' + response.status);
logger.debug('response status: ' + response.status);
const statusCode = response.status;
if (statusCode === 202) {
core.info("Successfully uploaded results");
logger.info("Successfully uploaded results");
return;
}
@ -77,7 +89,7 @@ async function uploadPayload(payload) {
// On a 5xx status code we may retry the request
if (attempt < backoffPeriods.length) {
// Log the failure as a warning but don't mark the action as failed yet
core.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
logger.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
' seconds: (' + statusCode + ') ' + JSON.stringify(response.data));
// Sleep for the backoff period
@ -109,18 +121,48 @@ export interface UploadStatusReport {
// Uploads a single sarif file or a directory of sarif files
// depending on what the path happens to refer to.
// Returns true iff the upload occurred and succeeded
export async function upload(input: string): Promise<UploadStatusReport> {
if (fs.lstatSync(input).isDirectory()) {
const sarifFiles = fs.readdirSync(input)
export async function upload(
sarifFile: string,
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string,
analysisName: string,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
logger: Logger): Promise<UploadStatusReport> {
const sarifFiles: string[] = [];
if (fs.lstatSync(sarifFile).isDirectory()) {
fs.readdirSync(sarifFile)
.filter(f => f.endsWith(".sarif"))
.map(f => path.resolve(input, f));
.map(f => path.resolve(sarifFile, f))
.forEach(f => sarifFiles.push(f));
if (sarifFiles.length === 0) {
throw new Error("No SARIF files found to upload in \"" + input + "\".");
throw new Error("No SARIF files found to upload in \"" + sarifFile + "\".");
}
return await uploadFiles(sarifFiles);
} else {
return await uploadFiles([input]);
sarifFiles.push(sarifFile);
}
return await uploadFiles(
sarifFiles,
repositoryNwo,
commitOid,
ref,
analysisKey,
analysisName,
workflowRunID,
checkoutPath,
environment,
githubAuth,
githubApiUrl,
mode,
logger);
}
// Counts the number of results in the given SARIF file
@ -134,17 +176,17 @@ export function countResultsInSarif(sarif: string): number {
// Validates that the given file path refers to a valid SARIF file.
// Throws an error if the file is invalid.
export function validateSarifFileSchema(sarifFilePath: string) {
export function validateSarifFileSchema(sarifFilePath: string, logger: Logger) {
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, 'utf8'));
const schema = JSON.parse(fs.readFileSync(__dirname + '/../src/sarif_v2.1.0_schema.json', 'utf8'));
const schema = require('../src/sarif_v2.1.0_schema.json');
const result = new jsonschema.Validator().validate(sarif, schema);
if (!result.valid) {
// Output the more verbose error messages in groups as these may be very large.
for (const error of result.errors) {
core.startGroup("Error details: " + error.stack);
core.info(JSON.stringify(error, null, 2));
core.endGroup();
logger.startGroup("Error details: " + error.stack);
logger.info(JSON.stringify(error, null, 2));
logger.endGroup();
}
// Set the main error message to the stacks of all the errors.
@ -156,75 +198,69 @@ export function validateSarifFileSchema(sarifFilePath: string) {
// Uploads the given set of sarif files.
// Returns true iff the upload occurred and succeeded
async function uploadFiles(sarifFiles: string[]): Promise<UploadStatusReport> {
core.startGroup("Uploading results");
core.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
async function uploadFiles(
sarifFiles: string[],
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string,
analysisName: string,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
logger: Logger): Promise<UploadStatusReport> {
const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
if (process.env[sentinelEnvVar]) {
throw new Error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
logger.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
// Validate that the files we were asked to upload are all valid SARIF files
for (const file of sarifFiles) {
validateSarifFileSchema(file);
validateSarifFileSchema(file, logger);
}
const commitOid = await util.getCommitOid();
const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID');
const ref = util.getRef();
const analysisKey = await util.getAnalysisKey();
const analysisName = util.getRequiredEnvParam('GITHUB_WORKFLOW');
const startedAt = process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT];
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutPath = core.getInput('checkout_path');
let checkoutURI = fileUrl(checkoutPath);
const workflowRunID = parseInt(workflowRunIDStr, 10);
if (Number.isNaN(workflowRunID)) {
throw new Error('GITHUB_RUN_ID must define a non NaN workflow run ID');
}
let matrix: string | undefined = core.getInput('matrix');
if (matrix === "null" || matrix === "") {
matrix = undefined;
}
const toolNames = util.getToolNames(sarifPayload);
const payload = JSON.stringify({
"commit_oid": commitOid,
"ref": ref,
"analysis_key": analysisKey,
"analysis_name": analysisName,
"sarif": zipped_sarif,
"workflow_run_id": workflowRunID,
"checkout_uri": checkoutURI,
"environment": matrix,
"started_at": startedAt,
"tool_names": toolNames,
});
let payload: string;
if (mode === 'actions') {
payload = JSON.stringify({
"commit_oid": commitOid,
"ref": ref,
"analysis_key": analysisKey,
"analysis_name": analysisName,
"sarif": zipped_sarif,
"workflow_run_id": workflowRunID,
"checkout_uri": checkoutURI,
"environment": environment,
"tool_names": toolNames,
});
} else {
payload = JSON.stringify({
"commit_sha": commitOid,
"ref": ref,
"sarif": zipped_sarif,
"checkout_uri": checkoutURI,
"tool_name": toolNames[0],
});
}
// Log some useful debug info about the info
const rawUploadSizeBytes = sarifPayload.length;
core.debug("Raw upload size: " + rawUploadSizeBytes + " bytes");
console.debug("Raw upload size: " + rawUploadSizeBytes + " bytes");
const zippedUploadSizeBytes = zipped_sarif.length;
core.debug("Base64 zipped upload size: " + zippedUploadSizeBytes + " bytes");
console.debug("Base64 zipped upload size: " + zippedUploadSizeBytes + " bytes");
const numResultInSarif = countResultsInSarif(sarifPayload);
core.debug("Number of results in upload: " + numResultInSarif);
console.debug("Number of results in upload: " + numResultInSarif);
if (!util.isLocalRun()) {
// Make the upload
await uploadPayload(payload);
} else {
core.debug("Not uploading because this is a local run.");
}
core.endGroup();
// Make the upload
await uploadPayload(payload, repositoryNwo, githubAuth, githubApiUrl, mode, logger);
return {
raw_upload_size_bytes: rawUploadSizeBytes,

17
src/upload-sarif.ts
View file

@ -1,5 +1,7 @@
import * as core from '@actions/core';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import * as util from './util';
@ -21,7 +23,20 @@ async function run() {
}
try {
const uploadStats = await upload_lib.upload(core.getInput('sarif_file'));
const uploadStats = await upload_lib.upload(
core.getInput('sarif_file'),
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
'actions',
getActionsLogger());
await sendSuccessStatusReport(startedAt, uploadStats);
} catch (error) {

19
src/util.ts
View file

@ -90,15 +90,12 @@ export async function getCommitOid(): Promise<string> {
* Get the path of the currently executing workflow.
*/
async function getWorkflowPath(): Promise<string> {
if (isLocalRun()) {
return 'LOCAL';
}
const repo_nwo = getRequiredEnvParam('GITHUB_REPOSITORY').split("/");
const owner = repo_nwo[0];
const repo = repo_nwo[1];
const run_id = Number(getRequiredEnvParam('GITHUB_RUN_ID'));
const apiClient = api.getApiClient();
const apiClient = api.getActionsApiClient();
const runsResponse = await apiClient.request('GET /repos/:owner/:repo/actions/runs/:run_id', {
owner,
repo,
@ -111,6 +108,17 @@ async function getWorkflowPath(): Promise<string> {
return workflowResponse.data.path;
}
/**
* Get the workflow run ID.
*/
export function getWorkflowRunID(): number {
const workflowRunID = parseInt(getRequiredEnvParam('GITHUB_RUN_ID'), 10);
if (Number.isNaN(workflowRunID)) {
throw new Error('GITHUB_RUN_ID must define a non NaN workflow run ID');
}
return workflowRunID;
}
/**
* Get the analysis key paramter for the current job.
*
@ -283,7 +291,8 @@ export async function sendStatusReport<S extends StatusReportBase>(
const nwo = getRequiredEnvParam("GITHUB_REPOSITORY");
const [owner, repo] = nwo.split("/");
const statusResponse = await api.getApiClient().request('PUT /repos/:owner/:repo/code-scanning/analysis/status', {
const client = api.getActionsApiClient();
const statusResponse = await client.request('PUT /repos/:owner/:repo/code-scanning/analysis/status', {
owner: owner,
repo: repo,
data: statusReportJSON,

26
webpack.config.js Normal file
View file

@ -0,0 +1,26 @@
const path = require('path');
module.exports = {
entry: './src/cli.ts',
module: {
rules: [
{
test: /\.ts$/,
use: 'ts-loader',
exclude: /node_modules/,
},
],
},
target: 'node',
resolve: {
extensions: [ '.ts', '.js' ],
},
output: {
filename: 'cli.js',
path: path.resolve(__dirname, 'cli'),
},
optimization: {
// We no not want to minimize our code.
minimize: false
},
};