robojerk/codeql-action

Add a CLI interface to the upload-sarif action

Browse source
This commit is contained in:
Robert Brignull 2020-08-11 12:42:42 +01:00
parent bcf676e52d
commit 6d7a135fea
15 changed files with 355 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

182
src/upload-lib.ts
View file

@ -1,4 +1,3 @@
import * as core from '@actions/core';
import fileUrl from 'file-url';
import * as fs from 'fs';
import * as jsonschema from 'jsonschema';
@ -7,9 +6,12 @@ import zlib from 'zlib';
import * as api from './api-client';
import * as fingerprints from './fingerprints';
import * as sharedEnv from './shared-environment';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as util from './util';
type UploadMode = 'actions' | 'cli';
// Takes a list of paths to sarif files and combines them together,
// returning the contents of the combined sarif file.
export function combineSarifFiles(sarifFiles: string[]): string {
@ -35,8 +37,15 @@ export function combineSarifFiles(sarifFiles: string[]): string {
// Upload the given payload.
// If the request fails then this will retry a small number of times.
async function uploadPayload(payload) {
core.info('Uploading results');
async function uploadPayload(
payload: any,
repositoryNwo: RepositoryNwo,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
logger: Logger) {
logger.info('Uploading results');
// If in test mode we don't want to upload the results
const testMode = process.env['TEST_MODE'] === 'true' || false;
@ -44,26 +53,29 @@ async function uploadPayload(payload) {
return;
}
const [owner, repo] = util.getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
// Make up to 4 attempts to upload, and sleep for these
// number of seconds between each attempt.
// We don't want to backoff too much to avoid wasting action
// minutes, but just waiting a little bit could maybe help.
const backoffPeriods = [1, 5, 15];
const client = api.getApiClient(githubAuth, githubApiUrl);
for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
const response = await api.getApiClient().request("PUT /repos/:owner/:repo/code-scanning/analysis", ({
owner: owner,
repo: repo,
const reqURL = mode === 'actions'
? 'PUT /repos/:owner/:repo/code-scanning/analysis'
: 'POST /repos/:owner/:repo/code-scanning/sarifs';
const response = await client.request(reqURL, ({
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
data: payload,
}));
core.debug('response status: ' + response.status);
logger.debug('response status: ' + response.status);
const statusCode = response.status;
if (statusCode === 202) {
core.info("Successfully uploaded results");
logger.info("Successfully uploaded results");
return;
}
@ -77,7 +89,7 @@ async function uploadPayload(payload) {
// On a 5xx status code we may retry the request
if (attempt < backoffPeriods.length) {
// Log the failure as a warning but don't mark the action as failed yet
core.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
logger.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
' seconds: (' + statusCode + ') ' + JSON.stringify(response.data));
// Sleep for the backoff period
@ -109,18 +121,48 @@ export interface UploadStatusReport {
// Uploads a single sarif file or a directory of sarif files
// depending on what the path happens to refer to.
// Returns true iff the upload occurred and succeeded
export async function upload(input: string): Promise<UploadStatusReport> {
if (fs.lstatSync(input).isDirectory()) {
const sarifFiles = fs.readdirSync(input)
export async function upload(
sarifFile: string,
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string,
analysisName: string,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
logger: Logger): Promise<UploadStatusReport> {
const sarifFiles: string[] = [];
if (fs.lstatSync(sarifFile).isDirectory()) {
fs.readdirSync(sarifFile)
.filter(f => f.endsWith(".sarif"))
.map(f => path.resolve(input, f));
.map(f => path.resolve(sarifFile, f))
.forEach(f => sarifFiles.push(f));
if (sarifFiles.length === 0) {
throw new Error("No SARIF files found to upload in \"" + input + "\".");
throw new Error("No SARIF files found to upload in \"" + sarifFile + "\".");
}
return await uploadFiles(sarifFiles);
} else {
return await uploadFiles([input]);
sarifFiles.push(sarifFile);
}
return await uploadFiles(
sarifFiles,
repositoryNwo,
commitOid,
ref,
analysisKey,
analysisName,
workflowRunID,
checkoutPath,
environment,
githubAuth,
githubApiUrl,
mode,
logger);
}
// Counts the number of results in the given SARIF file
@ -134,17 +176,17 @@ export function countResultsInSarif(sarif: string): number {
// Validates that the given file path refers to a valid SARIF file.
// Throws an error if the file is invalid.
export function validateSarifFileSchema(sarifFilePath: string) {
export function validateSarifFileSchema(sarifFilePath: string, logger: Logger) {
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, 'utf8'));
const schema = JSON.parse(fs.readFileSync(__dirname + '/../src/sarif_v2.1.0_schema.json', 'utf8'));
const schema = require('../src/sarif_v2.1.0_schema.json');
const result = new jsonschema.Validator().validate(sarif, schema);
if (!result.valid) {
// Output the more verbose error messages in groups as these may be very large.
for (const error of result.errors) {
core.startGroup("Error details: " + error.stack);
core.info(JSON.stringify(error, null, 2));
core.endGroup();
logger.startGroup("Error details: " + error.stack);
logger.info(JSON.stringify(error, null, 2));
logger.endGroup();
}
// Set the main error message to the stacks of all the errors.
@ -156,75 +198,69 @@ export function validateSarifFileSchema(sarifFilePath: string) {
// Uploads the given set of sarif files.
// Returns true iff the upload occurred and succeeded
async function uploadFiles(sarifFiles: string[]): Promise<UploadStatusReport> {
core.startGroup("Uploading results");
core.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
async function uploadFiles(
sarifFiles: string[],
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string,
analysisName: string,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
logger: Logger): Promise<UploadStatusReport> {
const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
if (process.env[sentinelEnvVar]) {
throw new Error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
logger.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
// Validate that the files we were asked to upload are all valid SARIF files
for (const file of sarifFiles) {
validateSarifFileSchema(file);
validateSarifFileSchema(file, logger);
}
const commitOid = await util.getCommitOid();
const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID');
const ref = util.getRef();
const analysisKey = await util.getAnalysisKey();
const analysisName = util.getRequiredEnvParam('GITHUB_WORKFLOW');
const startedAt = process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT];
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutPath = core.getInput('checkout_path');
let checkoutURI = fileUrl(checkoutPath);
const workflowRunID = parseInt(workflowRunIDStr, 10);
if (Number.isNaN(workflowRunID)) {
throw new Error('GITHUB_RUN_ID must define a non NaN workflow run ID');
}
let matrix: string | undefined = core.getInput('matrix');
if (matrix === "null" || matrix === "") {
matrix = undefined;
}
const toolNames = util.getToolNames(sarifPayload);
const payload = JSON.stringify({
"commit_oid": commitOid,
"ref": ref,
"analysis_key": analysisKey,
"analysis_name": analysisName,
"sarif": zipped_sarif,
"workflow_run_id": workflowRunID,
"checkout_uri": checkoutURI,
"environment": matrix,
"started_at": startedAt,
"tool_names": toolNames,
});
let payload: string;
if (mode === 'actions') {
payload = JSON.stringify({
"commit_oid": commitOid,
"ref": ref,
"analysis_key": analysisKey,
"analysis_name": analysisName,
"sarif": zipped_sarif,
"workflow_run_id": workflowRunID,
"checkout_uri": checkoutURI,
"environment": environment,
"tool_names": toolNames,
});
} else {
payload = JSON.stringify({
"commit_sha": commitOid,
"ref": ref,
"sarif": zipped_sarif,
"checkout_uri": checkoutURI,
"tool_name": toolNames[0],
});
}
// Log some useful debug info about the info
const rawUploadSizeBytes = sarifPayload.length;
core.debug("Raw upload size: " + rawUploadSizeBytes + " bytes");
console.debug("Raw upload size: " + rawUploadSizeBytes + " bytes");
const zippedUploadSizeBytes = zipped_sarif.length;
core.debug("Base64 zipped upload size: " + zippedUploadSizeBytes + " bytes");
console.debug("Base64 zipped upload size: " + zippedUploadSizeBytes + " bytes");
const numResultInSarif = countResultsInSarif(sarifPayload);
core.debug("Number of results in upload: " + numResultInSarif);
console.debug("Number of results in upload: " + numResultInSarif);
if (!util.isLocalRun()) {
// Make the upload
await uploadPayload(payload);
} else {
core.debug("Not uploading because this is a local run.");
}
core.endGroup();
// Make the upload
await uploadPayload(payload, repositoryNwo, githubAuth, githubApiUrl, mode, logger);
return {
raw_upload_size_bytes: rawUploadSizeBytes,