robojerk/codeql-action

Upload much more data in status reports

Browse source
This commit is contained in:
Robert 2020-07-20 16:33:37 +01:00
parent 9769e4a6df
commit 87758a1402
33 changed files with 537 additions and 339 deletions
Download patch file Download diff file Expand all files Collapse all files

65
lib/upload-lib.js generated
View file

@ -48,7 +48,7 @@ async function uploadPayload(payload) {
// If in test mode we don't want to upload the results
const testMode = process.env['TEST_MODE'] === 'true' || false;
if (testMode) {
return true;
return;
}
const [owner, repo] = util.getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
// Make up to 4 attempts to upload, and sleep for these
@ -66,13 +66,12 @@ async function uploadPayload(payload) {
const statusCode = response.status;
if (statusCode === 202) {
core.info("Successfully uploaded results");
return true;
return;
}
const requestID = response.headers["x-github-request-id"];
// On any other status code that's not 5xx mark the upload as failed
if (!statusCode || statusCode < 500 || statusCode >= 600) {
core.setFailed('Upload failed (' + requestID + '): (' + statusCode + ') ' + JSON.stringify(response.data));
return false;
throw new Error('Upload failed (' + requestID + '): (' + statusCode + ') ' + JSON.stringify(response.data));
}
// On a 5xx status code we may retry the request
if (attempt < backoffPeriods.length) {
@ -88,11 +87,12 @@ async function uploadPayload(payload) {
// If the upload fails with 5xx then we assume it is a temporary problem
// and not an error that the user has caused or can fix.
// We avoid marking the job as failed to avoid breaking CI workflows.
core.error('Upload failed (' + requestID + '): (' + statusCode + ') ' + JSON.stringify(response.data));
return false;
throw new Error('Upload failed (' + requestID + '): (' + statusCode + ') ' + JSON.stringify(response.data));
}
}
return false;
// This case shouldn't ever happen as the final iteration of the loop
// will always throw an error instead of exiting to here.
throw new Error('Upload failed');
}
// Uploads a single sarif file or a directory of sarif files
// depending on what the path happens to refer to.
@ -103,8 +103,7 @@ async function upload(input) {
.filter(f => f.endsWith(".sarif"))
.map(f => path.resolve(input, f));
if (sarifFiles.length === 0) {
core.setFailed("No SARIF files found to upload in \"" + input + "\".");
return false;
throw new Error("No SARIF files found to upload in \"" + input + "\".");
}
return await uploadFiles(sarifFiles);
}
@ -123,27 +122,22 @@ function countResultsInSarif(sarif) {
}
exports.countResultsInSarif = countResultsInSarif;
// Validates that the given file path refers to a valid SARIF file.
// Returns a non-empty list of error message if the file is invalid,
// otherwise returns the empty list if the file is valid.
// Throws an error if the file is invalid.
function validateSarifFileSchema(sarifFilePath) {
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, 'utf8'));
const schema = JSON.parse(fs.readFileSync(__dirname + '/../src/sarif_v2.1.0_schema.json', 'utf8'));
const result = new jsonschema.Validator().validate(sarif, schema);
if (result.valid) {
return true;
}
else {
// Set the failure message to the stacks of all the errors.
// This should be of a manageable size and may even give enough to fix the error.
const errorMessages = result.errors.map(e => "- " + e.stack);
core.setFailed("Unable to upload \"" + sarifFilePath + "\" as it is not valid SARIF:\n" + errorMessages.join("\n"));
// Also output the more verbose error messages in groups as these may be very large.
if (!result.valid) {
// Output the more verbose error messages in groups as these may be very large.
for (const error of result.errors) {
core.startGroup("Error details: " + error.stack);
core.info(JSON.stringify(error, null, 2));
core.endGroup();
}
return false;
// Set the main error message to the stacks of all the errors.
// This should be of a manageable size and may even give enough to fix the error.
const sarifErrors = result.errors.map(e => "- " + e.stack);
throw new Error("Unable to upload \"" + sarifFilePath + "\" as it is not valid SARIF:\n" + sarifErrors.join("\n"));
}
}
exports.validateSarifFileSchema = validateSarifFileSchema;
@ -154,22 +148,19 @@ async function uploadFiles(sarifFiles) {
core.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
if (process.env[sentinelEnvVar]) {
core.error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
return false;
throw new Error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
// Validate that the files we were asked to upload are all valid SARIF files
for (const file of sarifFiles) {
if (!validateSarifFileSchema(file)) {
return false;
}
validateSarifFileSchema(file);
}
const commitOid = await util.getCommitOid();
const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID');
const ref = util.getRef();
const analysisKey = await util.getAnalysisKey();
const analysisName = util.getRequiredEnvParam('GITHUB_WORKFLOW');
const startedAt = process.env[sharedEnv.CODEQL_ACTION_STARTED_AT];
const startedAt = process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT];
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload);
const zipped_sarif = zlib_1.default.gzipSync(sarifPayload).toString('base64');
@ -177,8 +168,7 @@ async function uploadFiles(sarifFiles) {
let checkoutURI = file_url_1.default(checkoutPath);
const workflowRunID = parseInt(workflowRunIDStr, 10);
if (Number.isNaN(workflowRunID)) {
core.setFailed('GITHUB_RUN_ID must define a non NaN workflow run ID');
return false;
throw new Error('GITHUB_RUN_ID must define a non NaN workflow run ID');
}
let matrix = core.getInput('matrix');
if (matrix === "null" || matrix === "") {
@ -198,12 +188,19 @@ async function uploadFiles(sarifFiles) {
"tool_names": toolNames,
});
// Log some useful debug info about the info
core.debug("Raw upload size: " + sarifPayload.length + " bytes");
core.debug("Base64 zipped upload size: " + zipped_sarif.length + " bytes");
core.debug("Number of results in upload: " + countResultsInSarif(sarifPayload));
const rawUploadSizeBytes = sarifPayload.length;
core.debug("Raw upload size: " + rawUploadSizeBytes + " bytes");
const zippedUploadSizeBytes = zipped_sarif.length;
core.debug("Base64 zipped upload size: " + zippedUploadSizeBytes + " bytes");
const numResultInSarif = countResultsInSarif(sarifPayload);
core.debug("Number of results in upload: " + numResultInSarif);
// Make the upload
const succeeded = await uploadPayload(payload);
await uploadPayload(payload);
core.endGroup();
return succeeded;
return {
raw_upload_size_bytes: rawUploadSizeBytes,
zipped_upload_size_bytes: zippedUploadSizeBytes,
num_results_in_sarif: numResultInSarif,
};
}
//# sourceMappingURL=upload-lib.js.map