Merge remote-tracking branch 'upstream/main' into aeisenberg/unrevert-query-filters

This commit is contained in:
Andrew Eisenberg 2022-08-11 10:00:12 -07:00
commit 072cd929a3
73 changed files with 4750 additions and 246 deletions

View file

@ -753,13 +753,3 @@ test("isAnalyzingDefaultBranch()", async (t) => {
t.deepEqual(await actionsutil.isAnalyzingDefaultBranch(), false);
});
});
test("sanitizeArifactName", (t) => {
t.deepEqual(actionsutil.sanitizeArifactName("hello-world_"), "hello-world_");
t.deepEqual(actionsutil.sanitizeArifactName("hello`world`"), "helloworld");
t.deepEqual(actionsutil.sanitizeArifactName("hello===123"), "hello123");
t.deepEqual(
actionsutil.sanitizeArifactName("*m)a&n^y%i££n+v!a:l[i]d"),
"manyinvalid"
);
});

View file

@ -8,9 +8,12 @@ import * as safeWhich from "@chrisgavin/safe-which";
import * as yaml from "js-yaml";
import * as api from "./api-client";
import { Config } from "./config-utils";
import * as sharedEnv from "./shared-environment";
import {
doesDirectoryExist,
getCachedCodeQlVersion,
getCodeQLDatabasePath,
getRequiredEnvParam,
GITHUB_DOTCOM_URL,
isGitHubGhesVersionBelow,
@ -867,6 +870,33 @@ export async function isAnalyzingDefaultBranch(): Promise<boolean> {
return currentRef === defaultBranch;
}
export function sanitizeArifactName(name: string): string {
return name.replace(/[^a-zA-Z0-9_\\-]+/g, "");
export async function printDebugLogs(config: Config) {
for (const language of config.languages) {
const databaseDirectory = getCodeQLDatabasePath(config, language);
const logsDirectory = path.join(databaseDirectory, "log");
if (!doesDirectoryExist(logsDirectory)) {
core.info(`Directory ${logsDirectory} does not exist.`);
continue; // Skip this language database.
}
const walkLogFiles = (dir: string) => {
const entries = fs.readdirSync(dir, { withFileTypes: true });
if (entries.length === 0) {
core.info(`No debug logs found at directory ${logsDirectory}.`);
}
for (const entry of entries) {
if (entry.isFile()) {
const absolutePath = path.resolve(dir, entry.name);
core.startGroup(
`CodeQL Debug Logs - ${language} - ${entry.name} from file at path ${absolutePath}`
);
process.stdout.write(fs.readFileSync(absolutePath));
core.endGroup();
} else if (entry.isDirectory()) {
walkLogFiles(path.resolve(dir, entry.name));
}
}
};
walkLogFiles(logsDirectory);
}
}

View file

@ -0,0 +1,57 @@
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as analyzeActionPostHelper from "./analyze-action-post-helper";
import * as configUtils from "./config-utils";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test("post: analyze action with debug mode off", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: false,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
const uploadSarifSpy = sinon.spy();
await analyzeActionPostHelper.run(uploadSarifSpy);
t.assert(uploadSarifSpy.notCalled);
});
});
test("post: analyze action with debug mode on", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: true,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
requiredInputStub.withArgs("output").returns("fake-output-dir");
const uploadSarifSpy = sinon.spy();
await analyzeActionPostHelper.run(uploadSarifSpy);
t.assert(uploadSarifSpy.called);
});
});

View file

@ -0,0 +1,25 @@
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
export async function run(uploadSarifDebugArtifact: Function) {
const logger = getActionsLogger();
const config = await getConfig(actionsUtil.getTemporaryDirectory(), logger);
if (config === undefined) {
throw new Error(
"Config file could not be found at expected location. Did the 'init' action fail to start?"
);
}
// Upload Actions SARIF artifacts for debugging
if (config?.debugMode) {
core.info(
"Debug mode is on. Uploading available SARIF files as Actions debugging artifact..."
);
const outputDir = actionsUtil.getRequiredInput("output");
await uploadSarifDebugArtifact(config, outputDir);
}
}

View file

@ -0,0 +1,20 @@
/**
* This file is the entry point for the `post:` hook of `analyze-action.yml`.
* It will run after the all steps in this job, in reverse order in relation to
* other `post:` hooks.
*/
import * as core from "@actions/core";
import * as analyzeActionPostHelper from "./analyze-action-post-helper";
import * as debugArtifacts from "./debug-artifacts";
async function runWrapper() {
try {
await analyzeActionPostHelper.run(debugArtifacts.uploadSarifDebugArtifact);
} catch (error) {
core.setFailed(`analyze post-action step failed: ${error}`);
console.log(error);
}
}
void runWrapper();

View file

@ -1,7 +1,3 @@
import * as fs from "fs";
import * as path from "path";
import * as artifact from "@actions/artifact";
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
@ -13,7 +9,7 @@ import {
runQueries,
} from "./analyze";
import { getGitHubVersionActionsOnly } from "./api-client";
import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { GitHubFeatureFlags } from "./feature-flags";
@ -23,7 +19,6 @@ import { uploadTrapCaches } from "./trap-caching";
import * as upload_lib from "./upload-lib";
import { UploadResult } from "./upload-lib";
import * as util from "./util";
import { bundleDb, codeQlVersionAbove } from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
@ -135,50 +130,6 @@ async function run() {
config,
logger
);
if (config.debugMode) {
// Upload the SARIF files as an Actions artifact for debugging
await uploadDebugArtifacts(
config.languages.map((lang) =>
path.resolve(outputDir, `${lang}.sarif`)
),
outputDir,
config.debugArtifactName
);
}
}
const codeql = await getCodeQL(config.codeQLCmd);
if (config.debugMode) {
// Upload the logs as an Actions artifact for debugging
let toUpload: string[] = [];
for (const language of config.languages) {
toUpload = toUpload.concat(
listFolder(
path.resolve(util.getCodeQLDatabasePath(config, language), "log")
)
);
}
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Multilanguage tracing: there are additional logs in the root of the cluster
toUpload = toUpload.concat(
listFolder(path.resolve(config.dbLocation, "log"))
);
}
await uploadDebugArtifacts(
toUpload,
config.dbLocation,
config.debugArtifactName
);
if (!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))) {
// Before multi-language tracing, we wrote a compound-build-tracer.log in the temp dir
await uploadDebugArtifacts(
[path.resolve(config.tempDir, "compound-build-tracer.log")],
config.tempDir,
config.debugArtifactName
);
}
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
@ -211,6 +162,7 @@ async function run() {
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
const codeql = await getCodeQL(config.codeQLCmd);
await uploadTrapCaches(codeql, config, logger);
// We don't upload results in test mode, so don't wait for processing
@ -241,56 +193,6 @@ async function run() {
}
return;
} finally {
if (config?.debugMode) {
try {
// Upload the database bundles as an Actions artifact for debugging
const toUpload: string[] = [];
for (const language of config.languages) {
toUpload.push(
await bundleDb(
config,
language,
await getCodeQL(config.codeQLCmd),
`${config.debugDatabaseName}-${language}`
)
);
}
await uploadDebugArtifacts(
toUpload,
config.dbLocation,
config.debugArtifactName
);
} catch (error) {
console.log(`Failed to upload database debug bundles: ${error}`);
}
}
if (config?.debugMode) {
core.info("Debug mode is on. Printing CodeQL debug logs...");
for (const language of config.languages) {
const databaseDirectory = util.getCodeQLDatabasePath(config, language);
const logsDirectory = path.join(databaseDirectory, "log");
const walkLogFiles = (dir: string) => {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.isFile()) {
core.startGroup(
`CodeQL Debug Logs - ${language} - ${entry.name}`
);
process.stdout.write(
fs.readFileSync(path.resolve(dir, entry.name))
);
core.endGroup();
} else if (entry.isDirectory()) {
walkLogFiles(path.resolve(dir, entry.name));
}
}
};
walkLogFiles(logsDirectory);
}
}
}
if (runStats && uploadResult) {
@ -305,37 +207,6 @@ async function run() {
}
}
async function uploadDebugArtifacts(
toUpload: string[],
rootDir: string,
artifactName: string
) {
let suffix = "";
const matrix = actionsUtil.getRequiredInput("matrix");
if (matrix !== undefined && matrix !== "null") {
for (const entry of Object.entries(JSON.parse(matrix)).sort())
suffix += `-${entry[1]}`;
}
await artifact.create().uploadArtifact(
actionsUtil.sanitizeArifactName(`${artifactName}${suffix}`),
toUpload.map((file) => path.normalize(file)),
path.normalize(rootDir)
);
}
function listFolder(dir: string): string[] {
const entries = fs.readdirSync(dir, { withFileTypes: true });
let files: string[] = [];
for (const entry of entries) {
if (entry.isFile()) {
files.push(path.resolve(dir, entry.name));
} else if (entry.isDirectory()) {
files = files.concat(listFolder(path.resolve(dir, entry.name)));
}
}
return files;
}
export const runPromise = run();
async function runWrapper() {

View file

@ -143,7 +143,7 @@ export async function createdDBForScannedLanguages(
}
}
function dbIsFinalized(
export function dbIsFinalized(
config: configUtils.Config,
language: Language,
logger: Logger

View file

@ -0,0 +1,23 @@
import test from "ava";
import * as debugArtifacts from "./debug-artifacts";
test("sanitizeArifactName", (t) => {
t.deepEqual(
debugArtifacts.sanitizeArifactName("hello-world_"),
"hello-world_"
);
t.deepEqual(debugArtifacts.sanitizeArifactName("hello`world`"), "helloworld");
t.deepEqual(debugArtifacts.sanitizeArifactName("hello===123"), "hello123");
t.deepEqual(
debugArtifacts.sanitizeArifactName("*m)a&n^y%i££n+v!a:l[i]d"),
"manyinvalid"
);
});
test("uploadDebugArtifacts", async (t) => {
// Test that no error is thrown if artifacts list is empty.
await t.notThrowsAsync(
debugArtifacts.uploadDebugArtifacts([], "rootDir", "artifactName")
);
});

186
src/debug-artifacts.ts Normal file
View file

@ -0,0 +1,186 @@
import * as fs from "fs";
import * as path from "path";
import * as artifact from "@actions/artifact";
import * as core from "@actions/core";
import AdmZip from "adm-zip";
import del from "del";
import { getRequiredInput } from "./actions-util";
import { dbIsFinalized } from "./analyze";
import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import {
bundleDb,
codeQlVersionAbove,
doesDirectoryExist,
getCodeQLDatabasePath,
listFolder,
} from "./util";
export function sanitizeArifactName(name: string): string {
return name.replace(/[^a-zA-Z0-9_\\-]+/g, "");
}
export async function uploadDebugArtifacts(
toUpload: string[],
rootDir: string,
artifactName: string
) {
if (toUpload.length === 0) {
return;
}
let suffix = "";
const matrix = getRequiredInput("matrix");
if (matrix) {
try {
for (const [, matrixVal] of Object.entries(JSON.parse(matrix)).sort())
suffix += `-${matrixVal}`;
} catch (e) {
core.info(
"Could not parse user-specified `matrix` input into JSON. The debug artifact will not be named with the user's `matrix` input."
);
}
}
await artifact.create().uploadArtifact(
sanitizeArifactName(`${artifactName}${suffix}`),
toUpload.map((file) => path.normalize(file)),
path.normalize(rootDir)
);
}
export async function uploadSarifDebugArtifact(
config: Config,
outputDir: string
) {
if (!doesDirectoryExist(outputDir)) {
return;
}
let toUpload: string[] = [];
for (const lang of config.languages) {
const sarifFile = path.resolve(outputDir, `${lang}.sarif`);
if (fs.existsSync(sarifFile)) {
toUpload = toUpload.concat(sarifFile);
}
}
await uploadDebugArtifacts(toUpload, outputDir, config.debugArtifactName);
}
export async function uploadLogsDebugArtifact(config: Config) {
const codeql = await getCodeQL(config.codeQLCmd);
let toUpload: string[] = [];
for (const language of config.languages) {
const databaseDirectory = getCodeQLDatabasePath(config, language);
const logsDirectory = path.resolve(databaseDirectory, "log");
if (doesDirectoryExist(logsDirectory)) {
toUpload = toUpload.concat(listFolder(logsDirectory));
}
}
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Multilanguage tracing: there are additional logs in the root of the cluster
const multiLanguageTracingLogsDirectory = path.resolve(
config.dbLocation,
"log"
);
if (doesDirectoryExist(multiLanguageTracingLogsDirectory)) {
toUpload = toUpload.concat(listFolder(multiLanguageTracingLogsDirectory));
}
}
await uploadDebugArtifacts(
toUpload,
config.dbLocation,
config.debugArtifactName
);
// Before multi-language tracing, we wrote a compound-build-tracer.log in the temp dir
if (!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))) {
const compoundBuildTracerLogDirectory = path.resolve(
config.tempDir,
"compound-build-tracer.log"
);
if (doesDirectoryExist(compoundBuildTracerLogDirectory)) {
await uploadDebugArtifacts(
[compoundBuildTracerLogDirectory],
config.tempDir,
config.debugArtifactName
);
}
}
}
/**
* If a database has not been finalized, we cannot run the `codeql database bundle`
* command in the CLI because it will return an error. Instead we directly zip
* all files in the database folder and return the path.
*/
async function createPartialDatabaseBundle(
config: Config,
language: Language
): Promise<string> {
const databasePath = getCodeQLDatabasePath(config, language);
const databaseBundlePath = path.resolve(
config.dbLocation,
`${config.debugDatabaseName}-${language}-partial.zip`
);
core.info(
`${config.debugDatabaseName}-${language} is not finalized. Uploading partial database bundle at ${databaseBundlePath}...`
);
// See `bundleDb` for explanation behind deleting existing db bundle.
if (fs.existsSync(databaseBundlePath)) {
await del(databaseBundlePath, { force: true });
}
const zip = new AdmZip();
zip.addLocalFolder(databasePath);
zip.writeZip(databaseBundlePath);
return databaseBundlePath;
}
/**
* Runs `codeql database bundle` command and returns the path.
*/
async function createDatabaseBundleCli(
config: Config,
language: Language
): Promise<string> {
// Otherwise run `codeql database bundle` command.
const databaseBundlePath = await bundleDb(
config,
language,
await getCodeQL(config.codeQLCmd),
`${config.debugDatabaseName}-${language}`
);
return databaseBundlePath;
}
export async function uploadDatabaseBundleDebugArtifact(
config: Config,
logger: Logger
) {
for (const language of config.languages) {
try {
let databaseBundlePath;
if (!dbIsFinalized(config, language, logger)) {
databaseBundlePath = await createPartialDatabaseBundle(
config,
language
);
} else {
databaseBundlePath = await createDatabaseBundleCli(config, language);
}
await uploadDebugArtifacts(
[databaseBundlePath],
config.dbLocation,
config.debugArtifactName
);
} catch (error) {
core.info(
`Failed to upload database debug bundle for ${config.debugDatabaseName}-${language}: ${error}`
);
}
}
}

View file

@ -0,0 +1,69 @@
import test from "ava";
import * as sinon from "sinon";
import * as configUtils from "./config-utils";
import * as initActionPostHelper from "./init-action-post-helper";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test("post: init action with debug mode off", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: false,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
const uploadDatabaseBundleSpy = sinon.spy();
const uploadLogsSpy = sinon.spy();
const printDebugLogsSpy = sinon.spy();
await initActionPostHelper.run(
uploadDatabaseBundleSpy,
uploadLogsSpy,
printDebugLogsSpy
);
t.assert(uploadDatabaseBundleSpy.notCalled);
t.assert(uploadLogsSpy.notCalled);
t.assert(printDebugLogsSpy.notCalled);
});
});
test("post: init action with debug mode on", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: true,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
const uploadDatabaseBundleSpy = sinon.spy();
const uploadLogsSpy = sinon.spy();
const printDebugLogsSpy = sinon.spy();
await initActionPostHelper.run(
uploadDatabaseBundleSpy,
uploadLogsSpy,
printDebugLogsSpy
);
t.assert(uploadDatabaseBundleSpy.called);
t.assert(uploadLogsSpy.called);
t.assert(printDebugLogsSpy.called);
});
});

View file

@ -0,0 +1,31 @@
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
export async function run(
uploadDatabaseBundleDebugArtifact: Function,
uploadLogsDebugArtifact: Function,
printDebugLogs: Function
) {
const logger = getActionsLogger();
const config = await getConfig(actionsUtil.getTemporaryDirectory(), logger);
if (config === undefined) {
throw new Error(
"Config file could not be found at expected location. Did the 'init' action fail to start?"
);
}
// Upload appropriate Actions artifacts for debugging
if (config?.debugMode) {
core.info(
"Debug mode is on. Uploading available database bundles and logs as Actions debugging artifacts..."
);
await uploadDatabaseBundleDebugArtifact(config, logger);
await uploadLogsDebugArtifact(config);
await printDebugLogs(config);
}
}

26
src/init-action-post.ts Normal file
View file

@ -0,0 +1,26 @@
/**
* This file is the entry point for the `post:` hook of `init-action.yml`.
* It will run after the all steps in this job, in reverse order in relation to
* other `post:` hooks.
*/
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import * as debugArtifacts from "./debug-artifacts";
import * as initActionPostHelper from "./init-action-post-helper";
async function runWrapper() {
try {
await initActionPostHelper.run(
debugArtifacts.uploadDatabaseBundleDebugArtifact,
debugArtifacts.uploadLogsDebugArtifact,
actionsUtil.printDebugLogs
);
} catch (error) {
core.setFailed(`init post-action step failed: ${error}`);
console.log(error);
}
}
void runWrapper();

View file

@ -1,5 +1,6 @@
import * as fs from "fs";
import * as os from "os";
import path from "path";
import * as stream from "stream";
import * as core from "@actions/core";
@ -440,3 +441,45 @@ for (const [
isActionsStub.restore();
});
}
test("doesDirectoryExist", async (t) => {
// Returns false if no file/dir of this name exists
t.false(util.doesDirectoryExist("non-existent-file.txt"));
await util.withTmpDir(async (tmpDir: string) => {
// Returns false if file
const testFile = `${tmpDir}/test-file.txt`;
fs.writeFileSync(testFile, "");
t.false(util.doesDirectoryExist(testFile));
// Returns true if directory
fs.writeFileSync(`${tmpDir}/nested-test-file.txt`, "");
t.true(util.doesDirectoryExist(tmpDir));
});
});
test("listFolder", async (t) => {
// Returns empty if not a directory
t.deepEqual(util.listFolder("not-a-directory"), []);
// Returns empty if directory is empty
await util.withTmpDir(async (emptyTmpDir: string) => {
t.deepEqual(util.listFolder(emptyTmpDir), []);
});
// Returns all file names in directory
await util.withTmpDir(async (tmpDir: string) => {
const nestedDir = fs.mkdtempSync(path.join(tmpDir, "nested-"));
fs.writeFileSync(path.resolve(nestedDir, "nested-test-file.txt"), "");
fs.writeFileSync(path.resolve(tmpDir, "test-file-1.txt"), "");
fs.writeFileSync(path.resolve(tmpDir, "test-file-2.txt"), "");
fs.writeFileSync(path.resolve(tmpDir, "test-file-3.txt"), "");
t.deepEqual(util.listFolder(tmpDir), [
path.resolve(nestedDir, "nested-test-file.txt"),
path.resolve(tmpDir, "test-file-1.txt"),
path.resolve(tmpDir, "test-file-2.txt"),
path.resolve(tmpDir, "test-file-3.txt"),
]);
});
});

View file

@ -762,3 +762,34 @@ export async function checkActionVersion(version: string) {
export function isInTestMode(): boolean {
return process.env["TEST_MODE"] === "true" || false;
}
/*
* Returns whether the path in the argument represents an existing directory.
*/
export function doesDirectoryExist(dirPath: string): boolean {
try {
const stats = fs.lstatSync(dirPath);
return stats.isDirectory();
} catch (e) {
return false;
}
}
/**
* Returns a recursive list of files in a given directory.
*/
export function listFolder(dir: string): string[] {
if (!doesDirectoryExist(dir)) {
return [];
}
const entries = fs.readdirSync(dir, { withFileTypes: true });
let files: string[] = [];
for (const entry of entries) {
if (entry.isFile()) {
files.push(path.resolve(dir, entry.name));
} else if (entry.isDirectory()) {
files = files.concat(listFolder(path.resolve(dir, entry.name)));
}
}
return files;
}