Merge branch 'main' into split-upload-method

This commit is contained in:
Sam Partington 2020-12-22 11:23:49 +00:00
commit 82a8fa443e
76 changed files with 8064 additions and 489 deletions

View file

@ -5,6 +5,7 @@ import zlib from "zlib";
import * as core from "@actions/core";
import fileUrl from "file-url";
import * as jsonschema from "jsonschema";
import * as semver from "semver";
import * as api from "./api-client";
import * as fingerprints from "./fingerprints";
@ -55,7 +56,7 @@ async function uploadPayload(
return;
}
const client = api.getApiClient(apiDetails, mode, logger);
const client = api.getApiClient(apiDetails);
const reqURL =
mode === "actions"
@ -93,6 +94,7 @@ export async function uploadFromActions(
workflowRunID: number,
checkoutPath: string,
environment: string,
gitHubVersion: util.GitHubVersion,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<UploadStatusReport> {
@ -106,6 +108,7 @@ export async function uploadFromActions(
workflowRunID,
checkoutPath,
environment,
gitHubVersion,
apiDetails,
"actions",
logger
@ -121,6 +124,7 @@ export async function uploadFromRunner(
commitOid: string,
ref: string,
checkoutPath: string,
gitHubVersion: util.GitHubVersion,
apiDetails: api.GitHubApiDetails,
logger: Logger
): Promise<UploadStatusReport> {
@ -134,6 +138,7 @@ export async function uploadFromRunner(
undefined,
checkoutPath,
undefined,
gitHubVersion,
apiDetails,
"runner",
logger
@ -197,6 +202,65 @@ export function validateSarifFileSchema(sarifFilePath: string, logger: Logger) {
}
}
// buildPayload constructs a map ready to be uploaded to the API from the given
// parameters, respecting the current mode and target GitHub instance version.
export function buildPayload(
commitOid: string,
ref: string,
analysisKey: string | undefined,
analysisName: string | undefined,
zippedSarif: string,
workflowRunID: number | undefined,
checkoutURI: string,
environment: string | undefined,
toolNames: string[],
gitHubVersion: util.GitHubVersion,
mode: util.Mode
) {
if (mode === "actions") {
const payloadObj = {
commit_oid: commitOid,
ref,
analysis_key: analysisKey,
analysis_name: analysisName,
sarif: zippedSarif,
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined as undefined | string,
base_sha: undefined as undefined | string,
};
// This behaviour can be made the default when support for GHES 3.0 is discontinued.
if (
gitHubVersion.type === "dotcom" ||
semver.satisfies(gitHubVersion.version, `>=3.1`)
) {
if (
process.env.GITHUB_EVENT_NAME === "pull_request" &&
process.env.GITHUB_EVENT_PATH
) {
const githubEvent = JSON.parse(
fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")
);
payloadObj.base_ref = `refs/heads/$githubEvent.pull_request.base.ref`;
payloadObj.base_sha = githubEvent.pull_request.base.sha;
}
}
return payloadObj;
} else {
return {
commit_sha: commitOid,
ref,
sarif: zippedSarif,
checkout_uri: checkoutURI,
tool_name: toolNames[0],
};
}
}
// Uploads the given set of sarif files.
// Returns true iff the upload occurred and succeeded
async function uploadFiles(
@ -209,6 +273,7 @@ async function uploadFiles(
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
gitHubVersion: util.GitHubVersion,
apiDetails: api.GitHubApiDetails,
mode: util.Mode,
logger: Logger
@ -238,39 +303,29 @@ async function uploadFiles(
logger
);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString("base64");
const zippedSarif = zlib.gzipSync(sarifPayload).toString("base64");
const checkoutURI = fileUrl(checkoutPath);
const toolNames = util.getToolNames(sarifPayload);
let payload: string;
if (mode === "actions") {
payload = JSON.stringify({
commit_oid: commitOid,
ref,
analysis_key: analysisKey,
analysis_name: analysisName,
sarif: zipped_sarif,
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
});
} else {
payload = JSON.stringify({
commit_sha: commitOid,
ref,
sarif: zipped_sarif,
checkout_uri: checkoutURI,
tool_name: toolNames[0],
});
}
const payload = buildPayload(
commitOid,
ref,
analysisKey,
analysisName,
zippedSarif,
workflowRunID,
checkoutURI,
environment,
toolNames,
gitHubVersion,
mode
);
// Log some useful debug info about the info
const rawUploadSizeBytes = sarifPayload.length;
logger.debug(`Raw upload size: ${rawUploadSizeBytes} bytes`);
const zippedUploadSizeBytes = zipped_sarif.length;
const zippedUploadSizeBytes = zippedSarif.length;
logger.debug(`Base64 zipped upload size: ${zippedUploadSizeBytes} bytes`);
const numResultInSarif = countResultsInSarif(sarifPayload);
logger.debug(`Number of results in upload: ${numResultInSarif}`);