robojerk/codeql-action

Merge pull request #1420 from github/henrymercer/failed-runs-fix-action-not-found

Browse source 
Fix failed SARIF upload behavior when the workflow doesn't call the CodeQL Action
This commit is contained in:
Henry Mercer 2022-12-07 08:48:11 +00:00 • committed by GitHub
commit 79166d0788
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 219 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

114
src/init-action-post-helper.test.ts
View file

@ -1,4 +1,4 @@
import test from "ava";
import test, { ExecutionContext } from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
@ -21,6 +21,7 @@ setupTests(test);
test("post: init action with debug mode off", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
@ -54,6 +55,7 @@ test("post: init action with debug mode off", async (t) => {
test("post: init action with debug mode on", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
@ -86,23 +88,45 @@ test("post: init action with debug mode on", async (t) => {
});
test("uploads failed SARIF run for typical workflow", async (t) => {
const config = {
codeQLCmd: "codeql",
debugMode: true,
languages: [],
packs: [],
} as unknown as configUtils.Config;
const messages = [];
process.env["GITHUB_JOB"] = "analyze";
process.env["GITHUB_WORKSPACE"] =
"/home/runner/work/codeql-action/codeql-action";
sinon.stub(actionsUtil, "getRequiredInput").withArgs("matrix").returns("{}");
const actionsWorkflow = createTestWorkflow([
{
name: "Checkout repository",
uses: "actions/checkout@v3",
},
{
name: "Initialize CodeQL",
uses: "github/codeql-action/init@v2",
with: {
languages: "javascript",
},
},
{
name: "Perform CodeQL Analysis",
uses: "github/codeql-action/analyze@v2",
with: {
category: "my-category",
},
},
]);
await testFailedSarifUpload(t, actionsWorkflow, { category: "my-category" });
});
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeql, "getCodeQL").resolves(codeqlObject);
const diagnosticsExportStub = sinon.stub(codeqlObject, "diagnosticsExport");
test("uploading failed SARIF run fails when workflow does not reference github/codeql-action", async (t) => {
const actionsWorkflow = createTestWorkflow([
{
name: "Checkout repository",
uses: "actions/checkout@v3",
},
]);
await t.throwsAsync(
async () => await testFailedSarifUpload(t, actionsWorkflow)
);
});
sinon.stub(workflow, "getWorkflow").resolves({
function createTestWorkflow(
steps: workflow.WorkflowJobStep[]
): workflow.Workflow {
return {
name: "CodeQL",
on: {
push: {
@ -116,29 +140,35 @@ test("uploads failed SARIF run for typical workflow", async (t) => {
analyze: {
name: "CodeQL Analysis",
"runs-on": "ubuntu-latest",
steps: [
{
name: "Checkout repository",
uses: "actions/checkout@v3",
},
{
name: "Initialize CodeQL",
uses: "github/codeql-action/init@v2",
with: {
languages: "javascript",
},
},
{
name: "Perform CodeQL Analysis",
uses: "github/codeql-action/analyze@v2",
with: {
category: "my-category",
},
},
],
steps,
},
},
});
};
}
async function testFailedSarifUpload(
t: ExecutionContext<unknown>,
actionsWorkflow: workflow.Workflow,
{ category }: { category?: string } = {}
): Promise<void> {
const config = {
codeQLCmd: "codeql",
debugMode: true,
languages: [],
packs: [],
} as unknown as configUtils.Config;
const messages = [];
process.env["GITHUB_JOB"] = "analyze";
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
process.env["GITHUB_WORKSPACE"] =
"/home/runner/work/codeql-action/codeql-action";
sinon.stub(actionsUtil, "getRequiredInput").withArgs("matrix").returns("{}");
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeql, "getCodeQL").resolves(codeqlObject);
const diagnosticsExportStub = sinon.stub(codeqlObject, "diagnosticsExport");
sinon.stub(workflow, "getWorkflow").resolves(actionsWorkflow);
const uploadFromActions = sinon.stub(uploadLib, "uploadFromActions");
uploadFromActions.resolves({ sarifID: "42" } as uploadLib.UploadResult);
@ -152,19 +182,21 @@ test("uploads failed SARIF run for typical workflow", async (t) => {
);
t.deepEqual(messages, []);
t.true(
diagnosticsExportStub.calledOnceWith(sinon.match.string, "my-category")
diagnosticsExportStub.calledOnceWith(sinon.match.string, category),
`Actual args were: ${diagnosticsExportStub.args}`
);
t.true(
uploadFromActions.calledOnceWith(
sinon.match.string,
sinon.match.string,
"my-category",
category,
sinon.match.any
)
),
`Actual args were: ${uploadFromActions.args}`
);
t.true(
waitForProcessing.calledOnceWith(sinon.match.any, "42", sinon.match.any, {
isUnsuccessfulExecution: true,
})
);
});
}

2
src/init-action-post-helper.ts
View file

@ -101,7 +101,7 @@ export async function run(
`the following error: ${e}`
);
}
logger.warning(
logger.info(
`Failed to upload a SARIF file for the failed run. Error: ${e}`
);
}

12
src/workflow.test.ts
View file

@ -525,6 +525,7 @@ test("getWorkflowErrors() should not report an error if PRs are totally unconfig
});
test("getCategoryInputOrThrow returns category for simple workflow with category", (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
t.is(
getCategoryInputOrThrow(
yaml.load(`
@ -546,6 +547,7 @@ test("getCategoryInputOrThrow returns category for simple workflow with category
});
test("getCategoryInputOrThrow returns undefined for simple workflow without category", (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
t.is(
getCategoryInputOrThrow(
yaml.load(`
@ -565,6 +567,7 @@ test("getCategoryInputOrThrow returns undefined for simple workflow without cate
});
test("getCategoryInputOrThrow returns category for workflow with multiple jobs", (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
t.is(
getCategoryInputOrThrow(
yaml.load(`
@ -596,6 +599,7 @@ test("getCategoryInputOrThrow returns category for workflow with multiple jobs",
});
test("getCategoryInputOrThrow finds category for workflow with language matrix", (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
t.is(
getCategoryInputOrThrow(
yaml.load(`
@ -622,6 +626,7 @@ test("getCategoryInputOrThrow finds category for workflow with language matrix",
});
test("getCategoryInputOrThrow throws error for workflow with dynamic category", (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
t.throws(
() =>
getCategoryInputOrThrow(
@ -646,7 +651,8 @@ test("getCategoryInputOrThrow throws error for workflow with dynamic category",
);
});
test("getCategoryInputOrThrow throws error for workflow with multiple categories", (t) => {
test("getCategoryInputOrThrow throws error for workflow with multiple calls to analyze", (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
t.throws(
() =>
getCategoryInputOrThrow(
@ -669,8 +675,8 @@ test("getCategoryInputOrThrow throws error for workflow with multiple categories
),
{
message:
"Could not get category input to github/codeql-action/analyze since there were multiple steps " +
"calling github/codeql-action/analyze with different values for category.",
"Could not get category input to github/codeql-action/analyze since the analysis job " +
"calls github/codeql-action/analyze multiple times.",
}
);
});

63
src/workflow.ts
View file

@ -7,7 +7,7 @@ import * as yaml from "js-yaml";
import * as api from "./api-client";
import { getRequiredEnvParam } from "./util";
interface WorkflowJobStep {
export interface WorkflowJobStep {
name?: string;
run?: any;
uses?: string;
@ -327,35 +327,32 @@ function getInputOrThrow(
inputName: string,
matrixVars: { [key: string]: string } | undefined
) {
const preamble = `Could not get ${inputName} input to ${actionName} since`;
if (!workflow.jobs) {
throw new Error(
`Could not get ${inputName} input to ${actionName} since the workflow has no jobs.`
);
throw new Error(`${preamble} the workflow has no jobs.`);
}
if (!workflow.jobs[jobName]) {
throw new Error(`${preamble} the workflow has no job named ${jobName}.`);
}
const stepsCallingAction = getStepsCallingAction(
workflow.jobs[jobName],
actionName
);
if (stepsCallingAction.length === 0) {
throw new Error(
`Could not get ${inputName} input to ${actionName} since the workflow has no job named ${jobName}.`
`${preamble} the ${jobName} job does not call ${actionName}.`
);
} else if (stepsCallingAction.length > 1) {
throw new Error(
`${preamble} the ${jobName} job calls ${actionName} multiple times.`
);
}
const inputs = getStepsCallingAction(workflow.jobs[jobName], actionName)
.map((step) => step.with?.[inputName])
.filter((input) => input !== undefined)
.map((input) => input!);
let input = stepsCallingAction[0].with?.[inputName];
if (inputs.length === 0) {
return undefined;
}
if (!inputs.every((input) => input === inputs[0])) {
throw new Error(
`Could not get ${inputName} input to ${actionName} since there were multiple steps calling ` +
`${actionName} with different values for ${inputName}.`
);
}
let input = inputs[0];
if (matrixVars !== undefined) {
if (input !== undefined && matrixVars !== undefined) {
// Make a basic attempt to substitute matrix variables
// First normalize by removing whitespace
input = input.replace(/\${{\s+/, "${{").replace(/\s+}}/, "}}");
@ -363,8 +360,7 @@ function getInputOrThrow(
input = input.replace(`\${{matrix.${key}}}`, value);
}
}
if (input.includes("${{")) {
if (input !== undefined && input.includes("${{")) {
throw new Error(
`Could not get ${inputName} input to ${actionName} since it contained an unrecognized dynamic value.`
);
@ -372,6 +368,19 @@ function getInputOrThrow(
return input;
}
/**
* Get the expected name of the analyze Action.
*
* This allows us to test workflow parsing functionality as a CodeQL Action PR check.
*/
function getAnalyzeActionName() {
if (getRequiredEnvParam("GITHUB_REPOSITORY") === "github/codeql-action") {
return "./analyze";
} else {
return "github/codeql-action/analyze";
}
}
/**
* Makes a best effort attempt to retrieve the category input for the particular job,
* given a set of matrix variables.
@ -389,7 +398,7 @@ export function getCategoryInputOrThrow(
return getInputOrThrow(
workflow,
jobName,
"github/codeql-action/analyze",
getAnalyzeActionName(),
"category",
matrixVars
);
@ -413,7 +422,7 @@ export function getUploadInputOrThrow(
getInputOrThrow(
workflow,
jobName,
"github/codeql-action/analyze",
getAnalyzeActionName(),
"upload",
matrixVars
) || "true" // if unspecified, upload defaults to true
@ -438,7 +447,7 @@ export function getCheckoutPathInputOrThrow(
getInputOrThrow(
workflow,
jobName,
"github/codeql-action/analyze",
getAnalyzeActionName(),
"checkout_path",
matrixVars
) || getRequiredEnvParam("GITHUB_WORKSPACE") // if unspecified, checkout_path defaults to ${{ github.workspace }}