robojerk/codeql-action

Merge branch 'main' into nickfyson/error_wrapper

Browse source 
# Conflicts:
#	lib/codeql.js
#	lib/codeql.js.map
#	src/codeql.ts
This commit is contained in:
Nick Fyson 2020-09-14 13:30:37 +01:00
commit e5e9aad174
5595 changed files with 493483 additions and 64057 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/analysis-paths.test.ts
View file

@ -1,13 +1,13 @@
import test from 'ava';
import test from "ava";
import * as analysisPaths from './analysis-paths';
import {setupTests} from './testing-utils';
import * as util from './util';
import * as analysisPaths from "./analysis-paths";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test("emptyPaths", async t => {
return await util.withTmpDir(async tmpDir => {
test("emptyPaths", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const config = {
languages: [],
queries: {},
@ -16,30 +16,33 @@ test("emptyPaths", async t => {
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: '',
codeQLCmd: "",
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env['LGTM_INDEX_INCLUDE'], undefined);
t.is(process.env['LGTM_INDEX_EXCLUDE'], undefined);
t.is(process.env['LGTM_INDEX_FILTERS'], undefined);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
t.is(process.env["LGTM_INDEX_EXCLUDE"], undefined);
t.is(process.env["LGTM_INDEX_FILTERS"], undefined);
});
});
test("nonEmptyPaths", async t => {
return await util.withTmpDir(async tmpDir => {
test("nonEmptyPaths", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const config = {
languages: [],
queries: {},
paths: ['path1', 'path2', '**/path3'],
pathsIgnore: ['path4', 'path5', 'path6/**'],
paths: ["path1", "path2", "**/path3"],
pathsIgnore: ["path4", "path5", "path6/**"],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: '',
codeQLCmd: "",
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env['LGTM_INDEX_INCLUDE'], 'path1\npath2');
t.is(process.env['LGTM_INDEX_EXCLUDE'], 'path4\npath5');
t.is(process.env['LGTM_INDEX_FILTERS'], 'include:path1\ninclude:path2\ninclude:**/path3\nexclude:path4\nexclude:path5\nexclude:path6/**');
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
t.is(process.env["LGTM_INDEX_EXCLUDE"], "path4\npath5");
t.is(
process.env["LGTM_INDEX_FILTERS"],
"include:path1\ninclude:path2\ninclude:**/path3\nexclude:path4\nexclude:path5\nexclude:path6/**"
);
});
});

43
src/analysis-paths.ts
View file

@ -1,8 +1,8 @@
import * as configUtils from './config-utils';
import { Logger } from './logging';
import * as configUtils from "./config-utils";
import { Logger } from "./logging";
function isInterpretedLanguage(language): boolean {
return language === 'javascript' || language === 'python';
return language === "javascript" || language === "python";
}
// Matches a string containing only characters that are legal to include in paths on windows.
@ -11,24 +11,29 @@ export const legalWindowsPathCharactersRegex = /^[^<>:"\|?]*$/;
// Builds an environment variable suitable for LGTM_INDEX_INCLUDE or LGTM_INDEX_EXCLUDE
function buildIncludeExcludeEnvVar(paths: string[]): string {
// Ignore anything containing a *
paths = paths.filter(p => p.indexOf('*') === -1);
paths = paths.filter((p) => p.indexOf("*") === -1);
// Some characters are illegal in path names in windows
if (process.platform === 'win32') {
paths = paths.filter(p => p.match(legalWindowsPathCharactersRegex));
if (process.platform === "win32") {
paths = paths.filter((p) => p.match(legalWindowsPathCharactersRegex));
}
return paths.join('\n');
return paths.join("\n");
}
export function printPathFiltersWarning(config: configUtils.Config, logger: Logger) {
export function printPathFiltersWarning(
config: configUtils.Config,
logger: Logger
) {
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
logger.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
if (
(config.paths.length !== 0 || config.pathsIgnore.length !== 0) &&
!config.languages.every(isInterpretedLanguage)
) {
logger.warning(
'The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python'
);
}
}
@ -41,19 +46,21 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// traverse the entire file tree to determine which files are matched.
// Any paths containing "*" are not included in these.
if (config.paths.length !== 0) {
process.env['LGTM_INDEX_INCLUDE'] = buildIncludeExcludeEnvVar(config.paths);
process.env["LGTM_INDEX_INCLUDE"] = buildIncludeExcludeEnvVar(config.paths);
}
if (config.pathsIgnore.length !== 0) {
process.env['LGTM_INDEX_EXCLUDE'] = buildIncludeExcludeEnvVar(config.pathsIgnore);
process.env["LGTM_INDEX_EXCLUDE"] = buildIncludeExcludeEnvVar(
config.pathsIgnore
);
}
// The 'LGTM_INDEX_FILTERS' environment variable controls which files are
// extracted or ignored. It does not control which directories are traversed.
// This does understand the glob and double-glob syntax.
const filters: string[] = [];
filters.push(...config.paths.map(p => 'include:' + p));
filters.push(...config.pathsIgnore.map(p => 'exclude:' + p));
filters.push(...config.paths.map((p) => `include:${p}`));
filters.push(...config.pathsIgnore.map((p) => `exclude:${p}`));
if (filters.length !== 0) {
process.env['LGTM_INDEX_FILTERS'] = filters.join('\n');
process.env["LGTM_INDEX_FILTERS"] = filters.join("\n");
}
}

81
src/analyze-action.ts
View file

@ -1,20 +1,31 @@
import * as core from '@actions/core';
import * as core from "@actions/core";
import { AnalysisStatusReport, runAnalyze } from './analyze';
import { getConfig } from './config-utils';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as util from './util';
import { AnalysisStatusReport, runAnalyze } from "./analyze";
import { getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as util from "./util";
interface FinishStatusReport extends util.StatusReportBase, AnalysisStatusReport {}
interface FinishStatusReport
extends util.StatusReportBase,
AnalysisStatusReport {}
async function sendStatusReport(
startedAt: Date,
stats: AnalysisStatusReport | undefined,
error?: Error) {
const status = stats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase('finish', status, startedAt, error?.message, error?.stack);
error?: Error
) {
const status =
stats?.analyze_failure_language !== undefined || error !== undefined
? "failure"
: "success";
const statusReportBase = await util.createStatusReportBase(
"finish",
status,
startedAt,
error?.message,
error?.stack
);
const statusReport: FinishStatusReport = {
...statusReportBase,
...(stats || {}),
@ -27,34 +38,44 @@ async function run() {
let stats: AnalysisStatusReport | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
if (
!(await util.sendStatusReport(
await util.createStatusReportBase("finish", "starting", startedAt),
true
))
) {
return;
}
const logger = getActionsLogger();
const config = await getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
const config = await getConfig(
util.getRequiredEnvParam("RUNNER_TEMP"),
logger
);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
throw new Error(
"Config file could not be found at expected location. Has the 'init' action been called?"
);
}
stats = await runAnalyze(
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY")),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getRequiredEnvParam("GITHUB_WORKFLOW"),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
core.getInput('upload') === 'true',
'actions',
core.getInput('output'),
util.getMemoryFlag(core.getInput('ram')),
util.getAddSnippetsFlag(core.getInput('add-snippets')),
util.getThreadsFlag(core.getInput('threads'), logger),
core.getInput("checkout_path"),
core.getInput("matrix"),
core.getInput("token"),
util.getRequiredEnvParam("GITHUB_SERVER_URL"),
core.getInput("upload") === "true",
"actions",
core.getInput("output"),
util.getMemoryFlag(core.getInput("ram")),
util.getAddSnippetsFlag(core.getInput("add-snippets")),
util.getThreadsFlag(core.getInput("threads"), logger),
config,
logger);
logger
);
} catch (error) {
core.setFailed(error.message);
console.log(error);
@ -65,7 +86,7 @@ async function run() {
await sendStatusReport(startedAt, stats);
}
run().catch(e => {
core.setFailed("analyze action failed: " + e);
run().catch((e) => {
core.setFailed(`analyze action failed: ${e}`);
console.log(e);
});

105
src/analyze.ts
View file

@ -1,15 +1,15 @@
import * as fs from 'fs';
import * as path from 'path';
import * as fs from "fs";
import * as path from "path";
import * as analysisPaths from './analysis-paths';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
import * as analysisPaths from "./analysis-paths";
import { getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import * as sharedEnv from "./shared-environment";
import * as upload_lib from "./upload-lib";
import * as util from "./util";
export interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
@ -40,12 +40,14 @@ export interface QueriesStatusReport {
analyze_failure_language?: string;
}
export interface AnalysisStatusReport extends upload_lib.UploadStatusReport, QueriesStatusReport {}
export interface AnalysisStatusReport
extends upload_lib.UploadStatusReport,
QueriesStatusReport {}
async function createdDBForScannedLanguages(
config: configUtils.Config,
logger: Logger) {
logger: Logger
) {
// Insert the LGTM_INDEX_X env vars at this point so they are set when
// we extract any scanned languages.
analysisPaths.includeAndExcludeAnalysisPaths(config);
@ -53,8 +55,11 @@ async function createdDBForScannedLanguages(
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
logger.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
logger.startGroup(`Extracting ${language}`);
await codeql.extractScannedLanguage(
util.getCodeQLDatabasePath(config.tempDir, language),
language
);
logger.endGroup();
}
}
@ -62,14 +67,16 @@ async function createdDBForScannedLanguages(
async function finalizeDatabaseCreation(
config: configUtils.Config,
logger: Logger) {
logger: Logger
) {
await createdDBForScannedLanguages(config, logger);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
logger.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
logger.startGroup(`Finalizing ${language}`);
await codeql.finalizeDatabase(
util.getCodeQLDatabasePath(config.tempDir, language)
);
logger.endGroup();
}
}
@ -81,33 +88,45 @@ async function runQueries(
addSnippetsFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<QueriesStatusReport> {
logger: Logger
): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of config.languages) {
logger.startGroup('Analyzing ' + language);
for (const language of config.languages) {
logger.startGroup(`Analyzing ${language}`);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
throw new Error(
`Unable to analyse ${language} as no queries were selected for this language`
);
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
const querySuite = `${databasePath}-queries.qls`;
const querySuiteContents = queries.map((q) => `- query: ${q}`).join("\n");
fs.writeFileSync(querySuite, querySuiteContents);
logger.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
logger.debug(
`Query suite file for ${language}...\n${querySuiteContents}`
);
const sarifFile = path.join(sarifFolder, language + '.sarif');
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, addSnippetsFlag, threadsFlag);
await codeql.databaseAnalyze(
databasePath,
sarifFile,
querySuite,
memoryFlag,
addSnippetsFlag,
threadsFlag
);
logger.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
logger.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
@ -137,21 +156,28 @@ export async function runAnalyze(
addSnippetsFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<AnalysisStatusReport> {
logger: Logger
): Promise<AnalysisStatusReport> {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
logger.info('Finalizing database creation');
logger.info("Finalizing database creation");
await finalizeDatabaseCreation(config, logger);
logger.info('Analyzing database');
const queriesStats = await runQueries(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, config, logger);
logger.info("Analyzing database");
const queriesStats = await runQueries(
outputDir,
memoryFlag,
addSnippetsFlag,
threadsFlag,
config,
logger
);
if (!doUpload) {
logger.info('Not uploading results');
logger.info("Not uploading results");
return { ...queriesStats };
}
@ -168,7 +194,8 @@ export async function runAnalyze(
githubAuth,
githubUrl,
mode,
logger);
logger
);
return { ...queriesStats, ...uploadStats };
}

36
src/api-client.ts
View file

@ -1,21 +1,24 @@
import * as core from "@actions/core";
import * as github from "@actions/github";
import consoleLogLevel from "console-log-level";
import * as path from 'path';
import * as path from "path";
import { getRequiredEnvParam, isLocalRun } from "./util";
export const getApiClient = function(githubAuth: string, githubUrl: string, allowLocalRun = false) {
export const getApiClient = function (
githubAuth: string,
githubUrl: string,
allowLocalRun = false
) {
if (isLocalRun() && !allowLocalRun) {
throw new Error('Invalid API call in local run');
throw new Error("Invalid API call in local run");
}
return new github.GitHub(
{
auth: githubAuth,
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
log: consoleLogLevel({ level: "debug" })
});
return new github.GitHub({
auth: githubAuth,
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
log: consoleLogLevel({ level: "debug" }),
});
};
function getApiUrl(githubUrl: string): string {
@ -23,12 +26,12 @@ function getApiUrl(githubUrl: string): string {
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
if (url.hostname === "github.com" || url.hostname === "api.github.com") {
return "https://api.github.com";
}
// Add the /api/v3 API prefix
url.pathname = path.join(url.pathname, 'api', 'v3');
url.pathname = path.join(url.pathname, "api", "v3");
return url.toString();
}
@ -37,7 +40,8 @@ function getApiUrl(githubUrl: string): string {
// and called only from the action entrypoints.
export function getActionsApiClient(allowLocalRun = false) {
return getApiClient(
core.getInput('token'),
getRequiredEnvParam('GITHUB_SERVER_URL'),
allowLocalRun);
core.getInput("token"),
getRequiredEnvParam("GITHUB_SERVER_URL"),
allowLocalRun
);
}

60
src/autobuild-action.ts
View file

@ -1,10 +1,10 @@
import * as core from '@actions/core';
import * as core from "@actions/core";
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import * as config_utils from './config-utils';
import { Language } from './languages';
import { getActionsLogger } from './logging';
import * as util from './util';
import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
import * as config_utils from "./config-utils";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import * as util from "./util";
interface AutobuildStatusReport extends util.StatusReportBase {
// Comma-separated set of languages being autobuilt
@ -17,18 +17,22 @@ async function sendCompletedStatusReport(
startedAt: Date,
allLanguages: string[],
failingLanguage?: string,
cause?: Error) {
const status = failingLanguage !== undefined || cause !== undefined ? 'failure' : 'success';
cause?: Error
) {
const status =
failingLanguage !== undefined || cause !== undefined
? "failure"
: "success";
const statusReportBase = await util.createStatusReportBase(
'autobuild',
"autobuild",
status,
startedAt,
cause?.message,
cause?.stack);
cause?.stack
);
const statusReport: AutobuildStatusReport = {
...statusReportBase,
autobuild_languages: allLanguages.join(','),
autobuild_languages: allLanguages.join(","),
autobuild_failure: failingLanguage,
};
await util.sendStatusReport(statusReport);
@ -40,30 +44,46 @@ async function run() {
let language: Language | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('autobuild', 'starting', startedAt), true)) {
if (
!(await util.sendStatusReport(
await util.createStatusReportBase("autobuild", "starting", startedAt),
true
))
) {
return;
}
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
const config = await config_utils.getConfig(
util.getRequiredEnvParam("RUNNER_TEMP"),
logger
);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
throw new Error(
"Config file could not be found at expected location. Has the 'init' action been called?"
);
}
language = determineAutobuildLanguage(config, logger);
if (language !== undefined) {
await runAutobuild(language, config, logger);
}
} catch (error) {
core.setFailed("We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. " + error.message);
core.setFailed(
`We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. ${error.message}`
);
console.log(error);
await sendCompletedStatusReport(startedAt, language ? [language] : [], language, error);
await sendCompletedStatusReport(
startedAt,
language ? [language] : [],
language,
error
);
return;
}
await sendCompletedStatusReport(startedAt, language ? [language] : []);
}
run().catch(e => {
core.setFailed("autobuild action failed. " + e);
run().catch((e) => {
core.setFailed(`autobuild action failed. ${e}`);
console.log(e);
});

23
src/autobuild.ts
View file

@ -1,13 +1,12 @@
import { getCodeQL } from './codeql';
import * as config_utils from './config-utils';
import { isTracedLanguage, Language } from './languages';
import { Logger } from './logging';
import { getCodeQL } from "./codeql";
import * as config_utils from "./config-utils";
import { Language, isTracedLanguage } from "./languages";
import { Logger } from "./logging";
export function determineAutobuildLanguage(
config: config_utils.Config,
logger: Logger
): Language | undefined {
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
@ -16,14 +15,20 @@ export function determineAutobuildLanguage(
const language = autobuildLanguages[0];
if (!language) {
logger.info("None of the languages in this project require extra build steps");
logger.info(
"None of the languages in this project require extra build steps"
);
return undefined;
}
logger.debug(`Detected dominant traced language: ${language}`);
if (autobuildLanguages.length > 1) {
logger.warning(`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages.slice(1).join(' and ')}, you must replace this call with custom build steps.`);
logger.warning(
`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages
.slice(1)
.join(" and ")}, you must replace this call with custom build steps.`
);
}
return language;
@ -32,8 +37,8 @@ export function determineAutobuildLanguage(
export async function runAutobuild(
language: Language,
config: config_utils.Config,
logger: Logger) {
logger: Logger
) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = getCodeQL(config.codeQLCmd);
await codeQL.runAutobuild(language);

108
src/codeql.test.ts
View file

@ -1,61 +1,66 @@
import * as toolcache from '@actions/tool-cache';
import test from 'ava';
import nock from 'nock';
import * as path from 'path';
import * as toolcache from "@actions/tool-cache";
import test from "ava";
import nock from "nock";
import * as path from "path";
import * as codeql from './codeql';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
import * as codeql from "./codeql";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test('download codeql bundle cache', async t => {
await util.withTmpDir(async tmpDir => {
const versions = ['20200601', '20200610'];
test("download codeql bundle cache", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const versions = ["20200601", "20200610"];
for (let i = 0; i < versions.length; i++) {
const version = versions[i];
nock('https://example.com')
nock("https://example.com")
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(200, path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`));
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`)
);
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
'token',
'https://github.example.com',
"token",
"https://github.example.com",
tmpDir,
tmpDir,
'runner',
getRunnerLogger(true));
"runner",
getRunnerLogger(true)
);
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
}
const cachedVersions = toolcache.findAllVersions('CodeQL');
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
});
test('parse codeql bundle url version', t => {
test("parse codeql bundle url version", (t) => {
const tests = {
'20200601': '0.0.0-20200601',
'20200601.0': '0.0.0-20200601.0',
'20200601.0.0': '20200601.0.0',
'1.2.3': '1.2.3',
'1.2.3-alpha': '1.2.3-alpha',
'1.2.3-beta.1': '1.2.3-beta.1',
"20200601": "0.0.0-20200601",
"20200601.0": "0.0.0-20200601.0",
"20200601.0.0": "20200601.0.0",
"1.2.3": "1.2.3",
"1.2.3-alpha": "1.2.3-alpha",
"1.2.3-beta.1": "1.2.3-beta.1",
};
for (const [version, expectedVersion] of Object.entries(tests)) {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = codeql.getCodeQLURLVersion(url, getRunnerLogger(true));
const parsedVersion = codeql.getCodeQLURLVersion(
url,
getRunnerLogger(true)
);
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e.message);
@ -63,34 +68,43 @@ test('parse codeql bundle url version', t => {
}
});
test('getExtraOptions works for explicit paths', t => {
t.deepEqual(codeql.getExtraOptions({}, ['foo'], []), []);
test("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);
t.deepEqual(codeql.getExtraOptions({foo: [42]}, ['foo'], []), ['42']);
t.deepEqual(codeql.getExtraOptions({ foo: [42] }, ["foo"], []), ["42"]);
t.deepEqual(codeql.getExtraOptions({foo: {bar: [42]}}, ['foo', 'bar'], []), ['42']);
t.deepEqual(
codeql.getExtraOptions({ foo: { bar: [42] } }, ["foo", "bar"], []),
["42"]
);
});
test('getExtraOptions works for wildcards', t => {
t.deepEqual(codeql.getExtraOptions({'*': [42]}, ['foo'], []), ['42']);
test("getExtraOptions works for wildcards", (t) => {
t.deepEqual(codeql.getExtraOptions({ "*": [42] }, ["foo"], []), ["42"]);
});
test('getExtraOptions works for wildcards and explicit paths', t => {
let o1 = {'*': [42], foo: [87]};
t.deepEqual(codeql.getExtraOptions(o1, ['foo'], []), ['42', '87']);
test("getExtraOptions works for wildcards and explicit paths", (t) => {
const o1 = { "*": [42], foo: [87] };
t.deepEqual(codeql.getExtraOptions(o1, ["foo"], []), ["42", "87"]);
let o2 = {'*': [42], foo: [87]};
t.deepEqual(codeql.getExtraOptions(o2, ['foo', 'bar'], []), ['42']);
const o2 = { "*": [42], foo: [87] };
t.deepEqual(codeql.getExtraOptions(o2, ["foo", "bar"], []), ["42"]);
let o3 = {'*': [42], foo: { '*': [87], bar: [99]}};
let p = ['foo', 'bar'];
t.deepEqual(codeql.getExtraOptions(o3, p, []), ['42', '87', '99']);
const o3 = { "*": [42], foo: { "*": [87], bar: [99] } };
const p = ["foo", "bar"];
t.deepEqual(codeql.getExtraOptions(o3, p, []), ["42", "87", "99"]);
});
test('getExtraOptions throws for bad content', t => {
t.throws(() => codeql.getExtraOptions({'*': 42}, ['foo'], []));
test("getExtraOptions throws for bad content", (t) => {
t.throws(() => codeql.getExtraOptions({ "*": 42 }, ["foo"], []));
t.throws(() => codeql.getExtraOptions({foo: 87}, ['foo'], []));
t.throws(() => codeql.getExtraOptions({ foo: 87 }, ["foo"], []));
t.throws(() => codeql.getExtraOptions({'*': [42], foo: { '*': 87, bar: [99]}}, ['foo', 'bar'], []));
t.throws(() =>
codeql.getExtraOptions(
{ "*": [42], foo: { "*": 87, bar: [99] } },
["foo", "bar"],
[]
)
);
});

449
src/codeql.ts
View file

@ -1,40 +1,40 @@
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as http from '@actions/http-client';
import { IHeaders } from '@actions/http-client/interfaces';
import * as toolcache from '@actions/tool-cache';
import * as fs from 'fs';
import * as path from 'path';
import * as semver from 'semver';
import * as stream from 'stream';
import * as globalutil from 'util';
import uuidV4 from 'uuid/v4';
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import * as http from "@actions/http-client";
import { IHeaders } from "@actions/http-client/interfaces";
import * as toolcache from "@actions/tool-cache";
import * as fs from "fs";
import * as path from "path";
import * as semver from "semver";
import * as stream from "stream";
import * as globalutil from "util";
import uuidV4 from "uuid/v4";
import * as api from './api-client';
import * as defaults from './defaults.json'; // Referenced from codeql-action-sync-tool!
import { errorMatchers} from './error-matcher';
import { Language } from './languages';
import { Logger } from './logging';
import { toolrunnerErrorCatcher } from './toolrunner-error-catcher';
import * as util from './util';
import * as api from "./api-client";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { Language } from "./languages";
import { Logger } from "./logging";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import * as util from "./util";
type Options = (string|number|boolean)[];
type Options = Array<string | number | boolean>;
/**
* Extra command line options for the codeql commands.
*/
interface ExtraOptions {
'*'?: Options;
"*"?: Options;
database?: {
'*'?: Options,
init?: Options,
'trace-command'?: Options,
analyze?: Options,
finalize?: Options
"*"?: Options;
init?: Options;
"trace-command"?: Options;
analyze?: Options;
finalize?: Options;
};
resolve?: {
'*'?: Options,
extractor?: Options,
queries?: Options
"*"?: Options;
extractor?: Options;
queries?: Options;
};
}
@ -55,7 +55,11 @@ export interface CodeQL {
/**
* Run 'codeql database init'.
*/
databaseInit(databasePath: string, language: Language, sourceRoot: string): Promise<void>;
databaseInit(
databasePath: string,
language: Language,
sourceRoot: string
): Promise<void>;
/**
* Runs the autobuilder for the given language.
*/
@ -72,7 +76,10 @@ export interface CodeQL {
/**
* Run 'codeql resolve queries'.
*/
resolveQueries(queries: string[], extraSearchPath: string | undefined): Promise<ResolveQueriesOutput>;
resolveQueries(
queries: string[],
extraSearchPath: string | undefined
): Promise<ResolveQueriesOutput>;
/**
* Run 'codeql database analyze'.
*/
@ -82,20 +89,21 @@ export interface CodeQL {
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string): Promise<void>;
threadsFlag: string
): Promise<void>;
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
[queryPath: string]: {}
}
[queryPath: string]: {};
};
};
noDeclaredLanguage: {
[queryPath: string]: {}
[queryPath: string]: {};
};
multipleDeclaredLanguages: {
[queryPath: string]: {}
[queryPath: string]: {};
};
}
@ -110,7 +118,7 @@ const CODEQL_BUNDLE_NAME = "codeql-bundle.tar.gz";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
function getCodeQLActionRepository(mode: util.Mode): string {
if (mode !== 'actions') {
if (mode !== "actions") {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
@ -122,19 +130,22 @@ function getCodeQLActionRepository(mode: util.Mode): string {
const relativeScriptPath = path.relative(actionsDirectory, __filename);
// This handles the case where the Action does not come from an Action repository,
// e.g. our integration tests which use the Action code from the current checkout.
if (relativeScriptPath.startsWith("..") || path.isAbsolute(relativeScriptPath)) {
if (
relativeScriptPath.startsWith("..") ||
path.isAbsolute(relativeScriptPath)
) {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
const relativeScriptPathParts = relativeScriptPath.split(path.sep);
return relativeScriptPathParts[0] + "/" + relativeScriptPathParts[1];
return `${relativeScriptPathParts[0]}/${relativeScriptPathParts[1]}`;
}
async function getCodeQLBundleDownloadURL(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<string> {
logger: Logger
): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(mode);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
@ -146,28 +157,39 @@ async function getCodeQLBundleDownloadURL(
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
const uniqueDownloadSources = potentialDownloadSources.filter((url, index, self) => index === self.indexOf(url));
for (let downloadSource of uniqueDownloadSources) {
let [apiURL, repository] = downloadSource;
const uniqueDownloadSources = potentialDownloadSources.filter(
(url, index, self) => index === self.indexOf(url)
);
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (apiURL === util.GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
if (
apiURL === util.GITHUB_DOTCOM_URL &&
repository === CODEQL_DEFAULT_ACTION_REPOSITORY
) {
break;
}
let [repositoryOwner, repositoryName] = repository.split("/");
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getApiClient(githubAuth, githubUrl).repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION
});
for (let asset of release.data.assets) {
const release = await api
.getApiClient(githubAuth, githubUrl)
.repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION,
});
for (const asset of release.data.assets) {
if (asset.name === CODEQL_BUNDLE_NAME) {
logger.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
logger.info(
`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`
);
return asset.url;
}
}
} catch (e) {
logger.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
logger.info(
`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`
);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${CODEQL_BUNDLE_NAME}`;
@ -179,13 +201,15 @@ async function toolcacheDownloadTool(
url: string,
headers: IHeaders | undefined,
tempDir: string,
logger: Logger): Promise<string> {
const client = new http.HttpClient('CodeQL Action');
logger: Logger
): Promise<string> {
const client = new http.HttpClient("CodeQL Action");
const dest = path.join(tempDir, uuidV4());
const response: http.HttpClientResponse = await client.get(url, headers);
if (response.message.statusCode !== 200) {
logger.info(`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`);
logger.info(
`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`
);
throw new Error(`Unexpected HTTP response: ${response.message.statusCode}`);
}
const pipeline = globalutil.promisify(stream.pipeline);
@ -201,53 +225,71 @@ export async function setupCodeQL(
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
logger: Logger
): Promise<CodeQL> {
// Setting these two env vars makes the toolcache code safe to use outside,
// of actions but this is obviously not a great thing we're doing and it would
// be better to write our own implementation to use outside of actions.
process.env['RUNNER_TEMP'] = tempDir;
process.env['RUNNER_TOOL_CACHE'] = toolsDir;
process.env["RUNNER_TEMP"] = tempDir;
process.env["RUNNER_TOOL_CACHE"] = toolsDir;
try {
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`, logger);
const codeqlURLVersion = getCodeQLURLVersion(
codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`,
logger
);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
let codeqlFolder = toolcache.find("CodeQL", codeqlURLVersion);
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL(githubAuth, githubUrl, mode, logger);
codeqlURL = await getCodeQLBundleDownloadURL(
githubAuth,
githubUrl,
mode,
logger
);
}
const headers: IHeaders = {accept: 'application/octet-stream'};
const headers: IHeaders = { accept: "application/octet-stream" };
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
if (codeqlURL.startsWith(githubUrl + "/")) {
logger.debug('Downloading CodeQL bundle with token.');
if (codeqlURL.startsWith(`${githubUrl}/`)) {
logger.debug("Downloading CodeQL bundle with token.");
headers.authorization = `token ${githubAuth}`;
} else {
logger.debug('Downloading CodeQL bundle without token.');
logger.debug("Downloading CodeQL bundle without token.");
}
logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers, tempDir, logger);
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const codeqlPath = await toolcacheDownloadTool(
codeqlURL,
headers,
tempDir,
logger
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
codeqlFolder = await toolcache.cacheDir(
codeqlExtracted,
"CodeQL",
codeqlURLVersion
);
}
let codeqlCmd = path.join(codeqlFolder, 'codeql', 'codeql');
if (process.platform === 'win32') {
let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
if (process.platform === "win32") {
codeqlCmd += ".exe";
} else if (process.platform !== 'linux' && process.platform !== 'darwin') {
throw new Error("Unsupported platform: " + process.platform);
} else if (process.platform !== "linux" && process.platform !== "darwin") {
throw new Error(`Unsupported platform: ${process.platform}`);
}
cachedCodeQL = getCodeQLForCmd(codeqlCmd);
return cachedCodeQL;
} catch (e) {
logger.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
@ -255,22 +297,27 @@ export async function setupCodeQL(
}
export function getCodeQLURLVersion(url: string, logger: Logger): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Version could not be inferred`);
throw new Error(
`Malformed tools url: ${url}. Version could not be inferred`
);
}
let version = match[1];
if (!semver.valid(version)) {
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
logger.debug(
`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`
);
version = `0.0.0-${version}`;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`);
throw new Error(
`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`
);
}
return s;
@ -289,13 +336,14 @@ export function getCodeQL(cmd: string): CodeQL {
function resolveFunction<T>(
partialCodeql: Partial<CodeQL>,
methodName: string,
defaultImplementation?: T): T {
if (typeof partialCodeql[methodName] !== 'function') {
defaultImplementation?: T
): T {
if (typeof partialCodeql[methodName] !== "function") {
if (defaultImplementation !== undefined) {
return defaultImplementation;
}
const dummyMethod = () => {
throw new Error('CodeQL ' + methodName + ' method not correctly defined');
throw new Error(`CodeQL ${methodName} method not correctly defined`);
};
return dummyMethod as any;
}
@ -310,15 +358,18 @@ function resolveFunction<T>(
*/
export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
cachedCodeQL = {
getPath: resolveFunction(partialCodeql, 'getPath', () => '/tmp/dummy-path'),
printVersion: resolveFunction(partialCodeql, 'printVersion'),
getTracerEnv: resolveFunction(partialCodeql, 'getTracerEnv'),
databaseInit: resolveFunction(partialCodeql, 'databaseInit'),
runAutobuild: resolveFunction(partialCodeql, 'runAutobuild'),
extractScannedLanguage: resolveFunction(partialCodeql, 'extractScannedLanguage'),
finalizeDatabase: resolveFunction(partialCodeql, 'finalizeDatabase'),
resolveQueries: resolveFunction(partialCodeql, 'resolveQueries'),
databaseAnalyze: resolveFunction(partialCodeql, 'databaseAnalyze')
getPath: resolveFunction(partialCodeql, "getPath", () => "/tmp/dummy-path"),
printVersion: resolveFunction(partialCodeql, "printVersion"),
getTracerEnv: resolveFunction(partialCodeql, "getTracerEnv"),
databaseInit: resolveFunction(partialCodeql, "databaseInit"),
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(
partialCodeql,
"extractScannedLanguage"
),
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
};
return cachedCodeQL;
}
@ -332,27 +383,33 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
export function getCachedCodeQL(): CodeQL {
if (cachedCodeQL === undefined) {
// Should never happen as setCodeQL is called by testing-utils.setupTests
throw new Error('cachedCodeQL undefined');
throw new Error("cachedCodeQL undefined");
}
return cachedCodeQL;
}
function getCodeQLForCmd(cmd: string): CodeQL {
return {
getPath: function() {
getPath() {
return cmd;
},
printVersion: async function() {
async printVersion() {
await new toolrunnner.ToolRunner(cmd, [
'version',
'--format=json'
"version",
"--format=json",
]).exec();
},
getTracerEnv: async function(databasePath: string) {
async getTracerEnv(databasePath: string) {
// Write tracer-env.js to a temp location.
const tracerEnvJs = path.resolve(databasePath, 'working', 'tracer-env.js');
fs.mkdirSync(path.dirname(tracerEnvJs), {recursive: true});
fs.writeFileSync(tracerEnvJs, `
const tracerEnvJs = path.resolve(
databasePath,
"working",
"tracer-env.js"
);
fs.mkdirSync(path.dirname(tracerEnvJs), { recursive: true });
fs.writeFileSync(
tracerEnvJs,
`
const fs = require('fs');
const env = {};
for (let entry of Object.entries(process.env)) {
@ -363,134 +420,164 @@ function getCodeQLForCmd(cmd: string): CodeQL {
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');`);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');`
);
const envFile = path.resolve(databasePath, 'working', 'env.tmp');
const envFile = path.resolve(databasePath, "working", "env.tmp");
await new toolrunnner.ToolRunner(cmd, [
'database',
'trace-command',
"database",
"trace-command",
databasePath,
...getExtraOptionsFromEnv(['database', 'trace-command']),
...getExtraOptionsFromEnv(["database", "trace-command"]),
process.execPath,
tracerEnvJs,
envFile
envFile,
]).exec();
return JSON.parse(fs.readFileSync(envFile, 'utf-8'));
return JSON.parse(fs.readFileSync(envFile, "utf-8"));
},
databaseInit: async function(databasePath: string, language: Language, sourceRoot: string) {
async databaseInit(
databasePath: string,
language: Language,
sourceRoot: string
) {
await new toolrunnner.ToolRunner(cmd, [
'database',
'init',
"database",
"init",
databasePath,
'--language=' + language,
'--source-root=' + sourceRoot,
...getExtraOptionsFromEnv(['database', 'init']),
`--language=${language}`,
`--source-root=${sourceRoot}`,
...getExtraOptionsFromEnv(["database", "init"]),
]).exec();
},
runAutobuild: async function(language: Language) {
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
const autobuildCmd = path.join(path.dirname(cmd), language, 'tools', cmdName);
async runAutobuild(language: Language) {
const cmdName =
process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
const autobuildCmd = path.join(
path.dirname(cmd),
language,
"tools",
cmdName
);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
const javaToolOptions = process.env["JAVA_TOOL_OPTIONS"] || "";
process.env["JAVA_TOOL_OPTIONS"] = [
...javaToolOptions.split(/\s+/),
"-Dhttp.keepAlive=false",
"-Dmaven.wagon.http.pool=false",
].join(" ");
await new toolrunnner.ToolRunner(autobuildCmd).exec();
},
extractScannedLanguage: async function(databasePath: string, language: Language) {
async extractScannedLanguage(databasePath: string, language: Language) {
// Get extractor location
let extractorPath = '';
let extractorPath = "";
await new toolrunnner.ToolRunner(
cmd,
[
'resolve',
'extractor',
'--format=json',
'--language=' + language,
...getExtraOptionsFromEnv(['resolve', 'extractor']),
"resolve",
"extractor",
"--format=json",
`--language=${language}`,
...getExtraOptionsFromEnv(["resolve", "extractor"]),
],
{
silent: true,
listeners: {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
}).exec();
stdout: (data) => {
extractorPath += data.toString();
},
stderr: (data) => {
process.stderr.write(data);
},
},
}
).exec();
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
const traceCommand = path.resolve(JSON.parse(extractorPath), 'tools', 'autobuild' + ext);
const ext = process.platform === "win32" ? ".cmd" : ".sh";
const traceCommand = path.resolve(
JSON.parse(extractorPath),
"tools",
`autobuild${ext}`
);
// Run trace command
await toolrunnerErrorCatcher(
cmd, [
'database',
'trace-command',
...getExtraOptionsFromEnv(['database', 'trace-command']),
cmd,
[
"database",
"trace-command",
...getExtraOptionsFromEnv(["database", "trace-command"]),
databasePath,
'--',
traceCommand
"--",
traceCommand,
],
errorMatchers
);
},
finalizeDatabase: async function(databasePath: string) {
async finalizeDatabase(databasePath: string) {
await toolrunnerErrorCatcher(
cmd, [
'database',
'finalize',
...getExtraOptionsFromEnv(['database', 'finalize']),
databasePath
cmd,
[
"database",
"finalize",
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
],
errorMatchers);
errorMatchers
);
},
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined
) {
const codeqlArgs = [
'resolve',
'queries',
"resolve",
"queries",
...queries,
'--format=bylanguage',
...getExtraOptionsFromEnv(['resolve', 'queries'])
"--format=bylanguage",
...getExtraOptionsFromEnv(["resolve", "queries"]),
];
if (extraSearchPath !== undefined) {
codeqlArgs.push('--search-path', extraSearchPath);
codeqlArgs.push("--search-path", extraSearchPath);
}
let output = '';
let output = "";
await new toolrunnner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
}
}
},
},
}).exec();
return JSON.parse(output);
},
databaseAnalyze: async function(
async databaseAnalyze(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string) {
threadsFlag: string
) {
await new toolrunnner.ToolRunner(cmd, [
'database',
'analyze',
"database",
"analyze",
memoryFlag,
threadsFlag,
databasePath,
'--format=sarif-latest',
'--output=' + sarifFile,
"--format=sarif-latest",
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(['database', 'analyze']),
querySuite
...getExtraOptionsFromEnv(["database", "analyze"]),
querySuite,
]).exec();
}
},
};
}
@ -498,7 +585,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
* Gets the options for `path` of `options` as an array of extra option strings.
*/
function getExtraOptionsFromEnv(path: string[]) {
let options: ExtraOptions = util.getExtraOptionsEnvParam();
const options: ExtraOptions = util.getExtraOptionsEnvParam();
return getExtraOptions(options, path, []);
}
@ -511,7 +598,8 @@ function getExtraOptionsFromEnv(path: string[]) {
export /* exported for testing */ function getExtraOptions(
options: any,
path: string[],
pathInfo: string[]): string[] {
pathInfo: string[]
): string[] {
/**
* Gets `options` as an array of extra option strings.
*
@ -522,23 +610,30 @@ export /* exported for testing */ function getExtraOptions(
return [];
}
if (!Array.isArray(options)) {
const msg =
`The extra options for '${pathInfo.join('.')}' ('${JSON.stringify(options)}') are not in an array.`;
const msg = `The extra options for '${pathInfo.join(
"."
)}' ('${JSON.stringify(options)}') are not in an array.`;
throw new Error(msg);
}
return options.map(o => {
return options.map((o) => {
const t = typeof o;
if (t !== 'string' && t !== 'number' && t !== 'boolean') {
const msg =
`The extra option for '${pathInfo.join('.')}' ('${JSON.stringify(o)}') is not a primitive value.`;
if (t !== "string" && t !== "number" && t !== "boolean") {
const msg = `The extra option for '${pathInfo.join(
"."
)}' ('${JSON.stringify(o)}') is not a primitive value.`;
throw new Error(msg);
}
return o + '';
return `${o}`;
});
}
let all = asExtraOptions(options?.['*'], pathInfo.concat('*'));
let specific = path.length === 0 ?
asExtraOptions(options, pathInfo) :
getExtraOptions(options?.[path[0]], path?.slice(1), pathInfo.concat(path[0]));
const all = asExtraOptions(options?.["*"], pathInfo.concat("*"));
const specific =
path.length === 0
? asExtraOptions(options, pathInfo)
: getExtraOptions(
options?.[path[0]],
path?.slice(1),
pathInfo.concat(path[0])
);
return all.concat(specific);
}

676
src/codeql.ts.orig Normal file
View file

@ -0,0 +1,676 @@
<<<<<<< HEAD
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as http from '@actions/http-client';
import { IHeaders } from '@actions/http-client/interfaces';
import * as toolcache from '@actions/tool-cache';
import * as fs from 'fs';
import * as path from 'path';
import * as semver from 'semver';
import * as stream from 'stream';
import * as globalutil from 'util';
import uuidV4 from 'uuid/v4';
import * as api from './api-client';
import * as defaults from './defaults.json'; // Referenced from codeql-action-sync-tool!
import { errorMatchers} from './error-matcher';
import { Language } from './languages';
import { Logger } from './logging';
import { toolrunnerErrorCatcher } from './toolrunner-error-catcher';
import * as util from './util';
type Options = (string|number|boolean)[];
=======
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import * as http from "@actions/http-client";
import { IHeaders } from "@actions/http-client/interfaces";
import * as toolcache from "@actions/tool-cache";
import * as fs from "fs";
import * as path from "path";
import * as semver from "semver";
import * as stream from "stream";
import * as globalutil from "util";
import uuidV4 from "uuid/v4";
import * as api from "./api-client";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { Language } from "./languages";
import { Logger } from "./logging";
import * as util from "./util";
type Options = Array<string | number | boolean>;
>>>>>>> main
/**
* Extra command line options for the codeql commands.
*/
interface ExtraOptions {
"*"?: Options;
database?: {
"*"?: Options;
init?: Options;
"trace-command"?: Options;
analyze?: Options;
finalize?: Options;
};
resolve?: {
"*"?: Options;
extractor?: Options;
queries?: Options;
};
}
export interface CodeQL {
/**
* Get the path of the CodeQL executable.
*/
getPath(): string;
/**
* Print version information about CodeQL.
*/
printVersion(): Promise<void>;
/**
* Run 'codeql database trace-command' on 'tracer-env.js' and parse
* the result to get environment variables set by CodeQL.
*/
getTracerEnv(databasePath: string): Promise<{ [key: string]: string }>;
/**
* Run 'codeql database init'.
*/
databaseInit(
databasePath: string,
language: Language,
sourceRoot: string
): Promise<void>;
/**
* Runs the autobuilder for the given language.
*/
runAutobuild(language: Language): Promise<void>;
/**
* Extract code for a scanned language using 'codeql database trace-command'
* and running the language extracter.
*/
extractScannedLanguage(database: string, language: Language): Promise<void>;
/**
* Finalize a database using 'codeql database finalize'.
*/
finalizeDatabase(databasePath: string): Promise<void>;
/**
* Run 'codeql resolve queries'.
*/
resolveQueries(
queries: string[],
extraSearchPath: string | undefined
): Promise<ResolveQueriesOutput>;
/**
* Run 'codeql database analyze'.
*/
databaseAnalyze(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string
): Promise<void>;
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
[queryPath: string]: {};
};
};
noDeclaredLanguage: {
[queryPath: string]: {};
};
multipleDeclaredLanguages: {
[queryPath: string]: {};
};
}
/**
* Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
* Can be overridden in tests using `setCodeQL`.
*/
let cachedCodeQL: CodeQL | undefined = undefined;
const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
const CODEQL_BUNDLE_NAME = "codeql-bundle.tar.gz";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
function getCodeQLActionRepository(mode: util.Mode): string {
if (mode !== "actions") {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
// Actions do not know their own repository name,
// so we currently use this hack to find the name based on where our files are.
// This can be removed once the change to the runner in https://github.com/actions/runner/pull/585 is deployed.
const runnerTemp = util.getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
const relativeScriptPath = path.relative(actionsDirectory, __filename);
// This handles the case where the Action does not come from an Action repository,
// e.g. our integration tests which use the Action code from the current checkout.
if (
relativeScriptPath.startsWith("..") ||
path.isAbsolute(relativeScriptPath)
) {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
const relativeScriptPathParts = relativeScriptPath.split(path.sep);
return `${relativeScriptPathParts[0]}/${relativeScriptPathParts[1]}`;
}
async function getCodeQLBundleDownloadURL(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger
): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(mode);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[githubUrl, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
[githubUrl, CODEQL_DEFAULT_ACTION_REPOSITORY],
// GitHub.com, and the canonical Action.
[util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
const uniqueDownloadSources = potentialDownloadSources.filter(
(url, index, self) => index === self.indexOf(url)
);
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (
apiURL === util.GITHUB_DOTCOM_URL &&
repository === CODEQL_DEFAULT_ACTION_REPOSITORY
) {
break;
}
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api
.getApiClient(githubAuth, githubUrl)
.repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION,
});
for (const asset of release.data.assets) {
if (asset.name === CODEQL_BUNDLE_NAME) {
logger.info(
`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`
);
return asset.url;
}
}
} catch (e) {
logger.info(
`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`
);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${CODEQL_BUNDLE_NAME}`;
}
// We have to download CodeQL manually because the toolcache doesn't support Accept headers.
// This can be removed once https://github.com/actions/toolkit/pull/530 is merged and released.
async function toolcacheDownloadTool(
url: string,
headers: IHeaders | undefined,
tempDir: string,
logger: Logger
): Promise<string> {
const client = new http.HttpClient("CodeQL Action");
const dest = path.join(tempDir, uuidV4());
const response: http.HttpClientResponse = await client.get(url, headers);
if (response.message.statusCode !== 200) {
logger.info(
`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`
);
throw new Error(`Unexpected HTTP response: ${response.message.statusCode}`);
}
const pipeline = globalutil.promisify(stream.pipeline);
fs.mkdirSync(path.dirname(dest), { recursive: true });
await pipeline(response.message, fs.createWriteStream(dest));
return dest;
}
export async function setupCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger
): Promise<CodeQL> {
// Setting these two env vars makes the toolcache code safe to use outside,
// of actions but this is obviously not a great thing we're doing and it would
// be better to write our own implementation to use outside of actions.
process.env["RUNNER_TEMP"] = tempDir;
process.env["RUNNER_TOOL_CACHE"] = toolsDir;
try {
const codeqlURLVersion = getCodeQLURLVersion(
codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`,
logger
);
let codeqlFolder = toolcache.find("CodeQL", codeqlURLVersion);
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL(
githubAuth,
githubUrl,
mode,
logger
);
}
const headers: IHeaders = { accept: "application/octet-stream" };
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
if (codeqlURL.startsWith(`${githubUrl}/`)) {
logger.debug("Downloading CodeQL bundle with token.");
headers.authorization = `token ${githubAuth}`;
} else {
logger.debug("Downloading CodeQL bundle without token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const codeqlPath = await toolcacheDownloadTool(
codeqlURL,
headers,
tempDir,
logger
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(
codeqlExtracted,
"CodeQL",
codeqlURLVersion
);
}
let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
if (process.platform === "win32") {
codeqlCmd += ".exe";
} else if (process.platform !== "linux" && process.platform !== "darwin") {
throw new Error(`Unsupported platform: ${process.platform}`);
}
cachedCodeQL = getCodeQLForCmd(codeqlCmd);
return cachedCodeQL;
} catch (e) {
logger.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string, logger: Logger): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(
`Malformed tools url: ${url}. Version could not be inferred`
);
}
let version = match[1];
if (!semver.valid(version)) {
logger.debug(
`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`
);
version = `0.0.0-${version}`;
}
const s = semver.clean(version);
if (!s) {
throw new Error(
`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`
);
}
return s;
}
/**
* Use the CodeQL executable located at the given path.
*/
export function getCodeQL(cmd: string): CodeQL {
if (cachedCodeQL === undefined) {
cachedCodeQL = getCodeQLForCmd(cmd);
}
return cachedCodeQL;
}
function resolveFunction<T>(
partialCodeql: Partial<CodeQL>,
methodName: string,
defaultImplementation?: T
): T {
if (typeof partialCodeql[methodName] !== "function") {
if (defaultImplementation !== undefined) {
return defaultImplementation;
}
const dummyMethod = () => {
throw new Error(`CodeQL ${methodName} method not correctly defined`);
};
return dummyMethod as any;
}
return partialCodeql[methodName];
}
/**
* Set the functionality for CodeQL methods. Only for use in tests.
*
* Accepts a partial object and any undefined methods will be implemented
* to immediately throw an exception indicating which method is missing.
*/
export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
cachedCodeQL = {
getPath: resolveFunction(partialCodeql, "getPath", () => "/tmp/dummy-path"),
printVersion: resolveFunction(partialCodeql, "printVersion"),
getTracerEnv: resolveFunction(partialCodeql, "getTracerEnv"),
databaseInit: resolveFunction(partialCodeql, "databaseInit"),
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(
partialCodeql,
"extractScannedLanguage"
),
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
};
return cachedCodeQL;
}
/**
* Get the cached CodeQL object. Should only be used from tests.
*
* TODO: Work out a good way for tests to get this from the test context
* instead of having to have this method.
*/
export function getCachedCodeQL(): CodeQL {
if (cachedCodeQL === undefined) {
// Should never happen as setCodeQL is called by testing-utils.setupTests
throw new Error("cachedCodeQL undefined");
}
return cachedCodeQL;
}
function getCodeQLForCmd(cmd: string): CodeQL {
return {
getPath() {
return cmd;
},
async printVersion() {
await new toolrunnner.ToolRunner(cmd, [
"version",
"--format=json",
]).exec();
},
async getTracerEnv(databasePath: string) {
// Write tracer-env.js to a temp location.
const tracerEnvJs = path.resolve(
databasePath,
"working",
"tracer-env.js"
);
fs.mkdirSync(path.dirname(tracerEnvJs), { recursive: true });
fs.writeFileSync(
tracerEnvJs,
`
const fs = require('fs');
const env = {};
for (let entry of Object.entries(process.env)) {
const key = entry[0];
const value = entry[1];
if (typeof value !== 'undefined' && key !== '_' && !key.startsWith('JAVA_MAIN_CLASS_')) {
env[key] = value;
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');`
);
const envFile = path.resolve(databasePath, "working", "env.tmp");
await new toolrunnner.ToolRunner(cmd, [
"database",
"trace-command",
databasePath,
...getExtraOptionsFromEnv(["database", "trace-command"]),
process.execPath,
tracerEnvJs,
envFile,
]).exec();
return JSON.parse(fs.readFileSync(envFile, "utf-8"));
},
async databaseInit(
databasePath: string,
language: Language,
sourceRoot: string
) {
await new toolrunnner.ToolRunner(cmd, [
"database",
"init",
databasePath,
`--language=${language}`,
`--source-root=${sourceRoot}`,
...getExtraOptionsFromEnv(["database", "init"]),
]).exec();
},
async runAutobuild(language: Language) {
const cmdName =
process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
const autobuildCmd = path.join(
path.dirname(cmd),
language,
"tools",
cmdName
);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
const javaToolOptions = process.env["JAVA_TOOL_OPTIONS"] || "";
process.env["JAVA_TOOL_OPTIONS"] = [
...javaToolOptions.split(/\s+/),
"-Dhttp.keepAlive=false",
"-Dmaven.wagon.http.pool=false",
].join(" ");
await new toolrunnner.ToolRunner(autobuildCmd).exec();
},
async extractScannedLanguage(databasePath: string, language: Language) {
// Get extractor location
let extractorPath = "";
await new toolrunnner.ToolRunner(
cmd,
[
"resolve",
"extractor",
"--format=json",
`--language=${language}`,
...getExtraOptionsFromEnv(["resolve", "extractor"]),
],
{
silent: true,
listeners: {
stdout: (data) => {
extractorPath += data.toString();
},
stderr: (data) => {
process.stderr.write(data);
},
},
}
).exec();
// Set trace command
const ext = process.platform === "win32" ? ".cmd" : ".sh";
const traceCommand = path.resolve(
JSON.parse(extractorPath),
"tools",
`autobuild${ext}`
);
// Run trace command
<<<<<<< HEAD
await toolrunnerErrorCatcher(
cmd, [
'database',
'trace-command',
...getExtraOptionsFromEnv(['database', 'trace-command']),
databasePath,
'--',
traceCommand
],
errorMatchers
);
},
finalizeDatabase: async function(databasePath: string) {
await toolrunnerErrorCatcher(
cmd, [
'database',
'finalize',
...getExtraOptionsFromEnv(['database', 'finalize']),
databasePath
],
errorMatchers);
=======
await new toolrunnner.ToolRunner(cmd, [
"database",
"trace-command",
...getExtraOptionsFromEnv(["database", "trace-command"]),
databasePath,
"--",
traceCommand,
]).exec();
},
async finalizeDatabase(databasePath: string) {
await new toolrunnner.ToolRunner(cmd, [
"database",
"finalize",
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
]).exec();
>>>>>>> main
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined
) {
const codeqlArgs = [
"resolve",
"queries",
...queries,
"--format=bylanguage",
...getExtraOptionsFromEnv(["resolve", "queries"]),
];
if (extraSearchPath !== undefined) {
codeqlArgs.push("--search-path", extraSearchPath);
}
let output = "";
await new toolrunnner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
}).exec();
return JSON.parse(output);
},
async databaseAnalyze(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string
) {
await new toolrunnner.ToolRunner(cmd, [
"database",
"analyze",
memoryFlag,
threadsFlag,
databasePath,
"--format=sarif-latest",
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "analyze"]),
querySuite,
]).exec();
},
};
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*/
function getExtraOptionsFromEnv(path: string[]) {
const options: ExtraOptions = util.getExtraOptionsEnvParam();
return getExtraOptions(options, path, []);
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*
* - the special terminal step name '*' in `options` matches all path steps
* - throws an exception if this conversion is impossible.
*/
export /* exported for testing */ function getExtraOptions(
options: any,
path: string[],
pathInfo: string[]
): string[] {
/**
* Gets `options` as an array of extra option strings.
*
* - throws an exception mentioning `pathInfo` if this conversion is impossible.
*/
function asExtraOptions(options: any, pathInfo: string[]): string[] {
if (options === undefined) {
return [];
}
if (!Array.isArray(options)) {
const msg = `The extra options for '${pathInfo.join(
"."
)}' ('${JSON.stringify(options)}') are not in an array.`;
throw new Error(msg);
}
return options.map((o) => {
const t = typeof o;
if (t !== "string" && t !== "number" && t !== "boolean") {
const msg = `The extra option for '${pathInfo.join(
"."
)}' ('${JSON.stringify(o)}') is not a primitive value.`;
throw new Error(msg);
}
return `${o}`;
});
}
const all = asExtraOptions(options?.["*"], pathInfo.concat("*"));
const specific =
path.length === 0
? asExtraOptions(options, pathInfo)
: getExtraOptions(
options?.[path[0]],
path?.slice(1),
pathInfo.concat(path[0])
);
return all.concat(specific);
}

788
src/config-utils.test.ts
View file

File diff suppressed because it is too large Load diff

535
src/config-utils.ts
View file

@ -1,33 +1,33 @@
import * as fs from 'fs';
import * as yaml from 'js-yaml';
import * as path from 'path';
import * as fs from "fs";
import * as yaml from "js-yaml";
import * as path from "path";
import * as api from './api-client';
import { CodeQL, ResolveQueriesOutput } from './codeql';
import * as api from "./api-client";
import { CodeQL, ResolveQueriesOutput } from "./codeql";
import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
// Property names from the user-supplied config file.
const NAME_PROPERTY = 'name';
const DISABLE_DEFAULT_QUERIES_PROPERTY = 'disable-default-queries';
const QUERIES_PROPERTY = 'queries';
const QUERIES_USES_PROPERTY = 'uses';
const PATHS_IGNORE_PROPERTY = 'paths-ignore';
const PATHS_PROPERTY = 'paths';
const NAME_PROPERTY = "name";
const DISABLE_DEFAULT_QUERIES_PROPERTY = "disable-default-queries";
const QUERIES_PROPERTY = "queries";
const QUERIES_USES_PROPERTY = "uses";
const PATHS_IGNORE_PROPERTY = "paths-ignore";
const PATHS_PROPERTY = "paths";
/**
* Format of the config file supplied by the user.
*/
export interface UserConfig {
name?: string;
'disable-default-queries'?: boolean;
queries?: {
"disable-default-queries"?: boolean;
queries?: Array<{
name?: string;
uses: string;
}[];
'paths-ignore'?: string[];
}>;
"paths-ignore"?: string[];
paths?: string[];
}
@ -86,16 +86,17 @@ export interface Config {
*
* Format is a map from language to an array of path suffixes of .ql files.
*/
const DISABLED_BUILTIN_QUERIES: {[language: string]: string[]} = {
'csharp': [
'ql/src/Security Features/CWE-937/VulnerablePackage.ql',
'ql/src/Security Features/CWE-451/MissingXFrameOptions.ql',
]
const DISABLED_BUILTIN_QUERIES: { [language: string]: string[] } = {
csharp: [
"ql/src/Security Features/CWE-937/VulnerablePackage.ql",
"ql/src/Security Features/CWE-451/MissingXFrameOptions.ql",
],
};
function queryIsDisabled(language, query): boolean {
return (DISABLED_BUILTIN_QUERIES[language] || [])
.some(disabledQuery => query.endsWith(disabledQuery));
return (DISABLED_BUILTIN_QUERIES[language] || []).some((disabledQuery) =>
query.endsWith(disabledQuery)
);
}
/**
@ -106,17 +107,25 @@ function validateQueries(resolvedQueries: ResolveQueriesOutput) {
const noDeclaredLanguage = resolvedQueries.noDeclaredLanguage;
const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
if (noDeclaredLanguageQueries.length !== 0) {
throw new Error('The following queries do not declare a language. ' +
'Their qlpack.yml files are either missing or is invalid.\n' +
noDeclaredLanguageQueries.join('\n'));
throw new Error(
`${
"The following queries do not declare a language. " +
"Their qlpack.yml files are either missing or is invalid.\n"
}${noDeclaredLanguageQueries.join("\n")}`
);
}
const multipleDeclaredLanguages = resolvedQueries.multipleDeclaredLanguages;
const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
const multipleDeclaredLanguagesQueries = Object.keys(
multipleDeclaredLanguages
);
if (multipleDeclaredLanguagesQueries.length !== 0) {
throw new Error('The following queries declare multiple languages. ' +
'Their qlpack.yml files are either missing or is invalid.\n' +
multipleDeclaredLanguagesQueries.join('\n'));
throw new Error(
`${
"The following queries declare multiple languages. " +
"Their qlpack.yml files are either missing or is invalid.\n"
}${multipleDeclaredLanguagesQueries.join("\n")}`
);
}
}
@ -128,15 +137,22 @@ async function runResolveQueries(
resultMap: { [language: string]: string[] },
toResolve: string[],
extraSearchPath: string | undefined,
errorOnInvalidQueries: boolean) {
errorOnInvalidQueries: boolean
) {
const resolvedQueries = await codeQL.resolveQueries(
toResolve,
extraSearchPath
);
const resolvedQueries = await codeQL.resolveQueries(toResolve, extraSearchPath);
for (const [language, queries] of Object.entries(resolvedQueries.byLanguage)) {
for (const [language, queries] of Object.entries(
resolvedQueries.byLanguage
)) {
if (resultMap[language] === undefined) {
resultMap[language] = [];
}
resultMap[language].push(...Object.keys(queries).filter(q => !queryIsDisabled(language, q)));
resultMap[language].push(
...Object.keys(queries).filter((q) => !queryIsDisabled(language, q))
);
}
if (errorOnInvalidQueries) {
@ -147,13 +163,17 @@ async function runResolveQueries(
/**
* Get the set of queries included by default.
*/
async function addDefaultQueries(codeQL: CodeQL, languages: string[], resultMap: { [language: string]: string[] }) {
const suites = languages.map(l => l + '-code-scanning.qls');
async function addDefaultQueries(
codeQL: CodeQL,
languages: string[],
resultMap: { [language: string]: string[] }
) {
const suites = languages.map((l) => `${l}-code-scanning.qls`);
await runResolveQueries(codeQL, resultMap, suites, undefined, false);
}
// The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = ['security-extended', 'security-and-quality'] as const;
const builtinSuites = ["security-extended", "security-and-quality"] as const;
/**
* Determine the set of queries associated with suiteName's suites and add them to resultMap.
@ -164,14 +184,14 @@ async function addBuiltinSuiteQueries(
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
suiteName: string,
configFile?: string) {
configFile?: string
) {
const suite = builtinSuites.find((suite) => suite === suiteName);
if (!suite) {
throw new Error(getQueryUsesInvalid(configFile, suiteName));
}
const suites = languages.map(l => l + '-' + suiteName + '.qls');
const suites = languages.map((l) => `${l}-${suiteName}.qls`);
await runResolveQueries(codeQL, resultMap, suites, undefined, false);
}
@ -183,8 +203,8 @@ async function addLocalQueries(
resultMap: { [language: string]: string[] },
localQueryPath: string,
checkoutPath: string,
configFile?: string) {
configFile?: string
) {
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
let absoluteQueryPath = path.join(checkoutPath, localQueryPath);
@ -198,11 +218,23 @@ async function addLocalQueries(
absoluteQueryPath = fs.realpathSync(absoluteQueryPath);
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (!(absoluteQueryPath + path.sep).startsWith(fs.realpathSync(checkoutPath) + path.sep)) {
throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
if (
!(absoluteQueryPath + path.sep).startsWith(
fs.realpathSync(checkoutPath) + path.sep
)
) {
throw new Error(
getLocalPathOutsideOfRepository(configFile, localQueryPath)
);
}
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath, true);
await runResolveQueries(
codeQL,
resultMap,
[absoluteQueryPath],
checkoutPath,
true
);
}
/**
@ -215,16 +247,16 @@ async function addRemoteQueries(
tempDir: string,
githubUrl: string,
logger: Logger,
configFile?: string) {
let tok = queryUses.split('@');
configFile?: string
) {
let tok = queryUses.split("@");
if (tok.length !== 2) {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
}
const ref = tok[1];
tok = tok[0].split('/');
tok = tok[0].split("/");
// The first token is the owner
// The second token is the repo
// The rest is a path, if there is more than one token combine them to form the full path
@ -232,10 +264,10 @@ async function addRemoteQueries(
throw new Error(getQueryUsesInvalid(configFile, queryUses));
}
// Check none of the parts of the repository name are empty
if (tok[0].trim() === '' || tok[1].trim() === '') {
if (tok[0].trim() === "" || tok[1].trim() === "") {
throw new Error(getQueryUsesInvalid(configFile, queryUses));
}
const nwo = tok[0] + '/' + tok[1];
const nwo = `${tok[0]}/${tok[1]}`;
// Checkout the external repository
const checkoutPath = await externalQueries.checkoutExternalRepository(
@ -243,11 +275,13 @@ async function addRemoteQueries(
ref,
githubUrl,
tempDir,
logger);
logger
);
const queryPath = tok.length > 2
? path.join(checkoutPath, tok.slice(2).join('/'))
: checkoutPath;
const queryPath =
tok.length > 2
? path.join(checkoutPath, tok.slice(2).join("/"))
: checkoutPath;
await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath, true);
}
@ -269,8 +303,8 @@ async function parseQueryUses(
checkoutPath: string,
githubUrl: string,
logger: Logger,
configFile?: string) {
configFile?: string
) {
queryUses = queryUses.trim();
if (queryUses === "") {
throw new Error(getQueryUsesInvalid(configFile));
@ -278,18 +312,38 @@ async function parseQueryUses(
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), checkoutPath, configFile);
await addLocalQueries(
codeQL,
resultMap,
queryUses.slice(2),
checkoutPath,
configFile
);
return;
}
// Check for one of the builtin suites
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
await addBuiltinSuiteQueries(languages, codeQL, resultMap, queryUses, configFile);
if (queryUses.indexOf("/") === -1 && queryUses.indexOf("@") === -1) {
await addBuiltinSuiteQueries(
languages,
codeQL,
resultMap,
queryUses,
configFile
);
return;
}
// Otherwise, must be a reference to another repo
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, githubUrl, logger, configFile);
await addRemoteQueries(
codeQL,
resultMap,
queryUses,
tempDir,
githubUrl,
logger,
configFile
);
}
// Regex validating stars in paths or paths-ignore entries.
@ -307,58 +361,70 @@ export function validateAndSanitisePath(
originalPath: string,
propertyName: string,
configFile: string,
logger: Logger): string {
logger: Logger
): string {
// Take a copy so we don't modify the original path, so we can still construct error messages
let path = originalPath;
// All paths are relative to the src root, so strip off leading slashes.
while (path.charAt(0) === '/') {
while (path.charAt(0) === "/") {
path = path.substring(1);
}
// Trailing ** are redundant, so strip them off
if (path.endsWith('/**')) {
if (path.endsWith("/**")) {
path = path.substring(0, path.length - 2);
}
// An empty path is not allowed as it's meaningless
if (path === '') {
throw new Error(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" is not an invalid path. ' +
'It is not necessary to include it, and it is not allowed to exclude it.'));
if (path === "") {
throw new Error(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" is not an invalid path. ` +
`It is not necessary to include it, and it is not allowed to exclude it.`
)
);
}
// Check for illegal uses of **
if (path.match(pathStarsRegex)) {
throw new Error(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" contains an invalid "**" wildcard. ' +
'They must be immediately preceeded and followed by a slash as in "/**/", or come at the start or end.'));
throw new Error(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" contains an invalid "**" wildcard. ` +
`They must be immediately preceeded and followed by a slash as in "/**/", or come at the start or end.`
)
);
}
// Check for other regex characters that we don't support.
// Output a warning so the user knows, but otherwise continue normally.
if (path.match(filterPatternCharactersRegex)) {
logger.warning(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" contains an unsupported character. ' +
'The filter pattern characters ?, +, [, ], ! are not supported and will be matched literally.'));
logger.warning(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" contains an unsupported character. ` +
`The filter pattern characters ?, +, [, ], ! are not supported and will be matched literally.`
)
);
}
// Ban any uses of backslash for now.
// This may not play nicely with project layouts.
// This restriction can be lifted later if we determine they are ok.
if (path.indexOf('\\') !== -1) {
throw new Error(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" contains an "\\" character. These are not allowed in filters. ' +
'If running on windows we recommend using "/" instead for path filters.'));
if (path.indexOf("\\") !== -1) {
throw new Error(
getConfigFilePropertyError(
configFile,
propertyName,
`"${originalPath}" contains an "\\" character. These are not allowed in filters. ` +
`If running on windows we recommend using "/" instead for path filters.`
)
);
}
return path;
@ -368,86 +434,132 @@ export function validateAndSanitisePath(
// the property was in a workflow file, not a config file
export function getNameInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, NAME_PROPERTY, 'must be a non-empty string');
return getConfigFilePropertyError(
configFile,
NAME_PROPERTY,
"must be a non-empty string"
);
}
export function getDisableDefaultQueriesInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, DISABLE_DEFAULT_QUERIES_PROPERTY, 'must be a boolean');
return getConfigFilePropertyError(
configFile,
DISABLE_DEFAULT_QUERIES_PROPERTY,
"must be a boolean"
);
}
export function getQueriesInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY, 'must be an array');
}
export function getQueryUsesInvalid(configFile: string | undefined, queryUses?: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'must be a built-in suite (' + builtinSuites.join(' or ') +
'), a relative path, or be of the form "owner/repo[/path]@ref"' +
(queryUses !== undefined ? '\n Found: ' + queryUses : ''));
QUERIES_PROPERTY,
"must be an array"
);
}
export function getQueryUsesInvalid(
configFile: string | undefined,
queryUses?: string
): string {
return getConfigFilePropertyError(
configFile,
`${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`,
`must be a built-in suite (${builtinSuites.join(
" or "
)}), a relative path, or be of the form "owner/repo[/path]@ref"${
queryUses !== undefined ? `\n Found: ${queryUses}` : ""
}`
);
}
export function getPathsIgnoreInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, PATHS_IGNORE_PROPERTY, 'must be an array of non-empty strings');
return getConfigFilePropertyError(
configFile,
PATHS_IGNORE_PROPERTY,
"must be an array of non-empty strings"
);
}
export function getPathsInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, 'must be an array of non-empty strings');
}
export function getLocalPathOutsideOfRepository(configFile: string | undefined, localPath: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'is invalid as the local path "' + localPath + '" is outside of the repository');
PATHS_PROPERTY,
"must be an array of non-empty strings"
);
}
export function getLocalPathDoesNotExist(configFile: string | undefined, localPath: string): string {
export function getLocalPathOutsideOfRepository(
configFile: string | undefined,
localPath: string
): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'is invalid as the local path "' + localPath + '" does not exist in the repository');
`${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`,
`is invalid as the local path "${localPath}" is outside of the repository`
);
}
export function getConfigFileOutsideWorkspaceErrorMessage(configFile: string): string {
return 'The configuration file "' + configFile + '" is outside of the workspace';
export function getLocalPathDoesNotExist(
configFile: string | undefined,
localPath: string
): string {
return getConfigFilePropertyError(
configFile,
`${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`,
`is invalid as the local path "${localPath}" does not exist in the repository`
);
}
export function getConfigFileDoesNotExistErrorMessage(configFile: string): string {
return 'The configuration file "' + configFile + '" does not exist';
export function getConfigFileOutsideWorkspaceErrorMessage(
configFile: string
): string {
return `The configuration file "${configFile}" is outside of the workspace`;
}
export function getConfigFileRepoFormatInvalidMessage(configFile: string): string {
let error = 'The configuration file "' + configFile + '" is not a supported remote file reference.';
error += ' Expected format <owner>/<repository>/<file-path>@<ref>';
export function getConfigFileDoesNotExistErrorMessage(
configFile: string
): string {
return `The configuration file "${configFile}" does not exist`;
}
export function getConfigFileRepoFormatInvalidMessage(
configFile: string
): string {
let error = `The configuration file "${configFile}" is not a supported remote file reference.`;
error += " Expected format <owner>/<repository>/<file-path>@<ref>";
return error;
}
export function getConfigFileFormatInvalidMessage(configFile: string): string {
return 'The configuration file "' + configFile + '" could not be read';
return `The configuration file "${configFile}" could not be read`;
}
export function getConfigFileDirectoryGivenMessage(configFile: string): string {
return 'The configuration file "' + configFile + '" looks like a directory, not a file';
return `The configuration file "${configFile}" looks like a directory, not a file`;
}
function getConfigFilePropertyError(configFile: string | undefined, property: string, error: string): string {
function getConfigFilePropertyError(
configFile: string | undefined,
property: string,
error: string
): string {
if (configFile === undefined) {
return 'The workflow property "' + property + '" is invalid: ' + error;
return `The workflow property "${property}" is invalid: ${error}`;
} else {
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
return `The configuration file "${configFile}" is invalid: property "${property}" ${error}`;
}
}
export function getNoLanguagesError(): string {
return "Did not detect any languages to analyze. " +
"Please update input in workflow or check that GitHub detects the correct languages in your repository.";
return (
"Did not detect any languages to analyze. " +
"Please update input in workflow or check that GitHub detects the correct languages in your repository."
);
}
export function getUnknownLanguagesError(languages: string[]): string {
return "Did not recognise the following languages: " + languages.join(', ');
return `Did not recognise the following languages: ${languages.join(", ")}`;
}
/**
@ -457,23 +569,25 @@ async function getLanguagesInRepo(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
logger: Logger
): Promise<Language[]> {
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.listLanguages({
owner: repository.owner,
repo: repository.repo
});
const response = await api
.getApiClient(githubAuth, githubUrl, true)
.repos.listLanguages({
owner: repository.owner,
repo: repository.repo,
});
logger.debug("Languages API response: " + JSON.stringify(response));
logger.debug(`Languages API response: ${JSON.stringify(response)}`);
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
let languages: Set<Language> = new Set();
for (let lang of Object.keys(response.data)) {
let parsedLang = parseLanguage(lang);
const languages: Set<Language> = new Set();
for (const lang of Object.keys(response.data)) {
const parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
@ -496,14 +610,14 @@ async function getLanguages(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
logger: Logger
): Promise<Language[]> {
// Obtain from action input 'languages' if set
let languages = (languagesInput || "")
.split(',')
.map(x => x.trim())
.filter(x => x.length > 0);
logger.info("Languages from configuration: " + JSON.stringify(languages));
.split(",")
.map((x) => x.trim())
.filter((x) => x.length > 0);
logger.info(`Languages from configuration: ${JSON.stringify(languages)}`);
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
@ -511,8 +625,11 @@ async function getLanguages(
repository,
githubAuth,
githubUrl,
logger);
logger.info("Automatically detected languages: " + JSON.stringify(languages));
logger
);
logger.info(
`Automatically detected languages: ${JSON.stringify(languages)}`
);
}
// If the languages parameter was not given and no languages were
@ -524,7 +641,7 @@ async function getLanguages(
// Make sure they are supported
const parsedLanguages: Language[] = [];
const unknownLanguages: string[] = [];
for (let language of languages) {
for (const language of languages) {
const parsedLanguage = parseLanguage(language);
if (parsedLanguage === undefined) {
unknownLanguages.push(language);
@ -547,13 +664,13 @@ async function addQueriesFromWorkflow(
tempDir: string,
checkoutPath: string,
githubUrl: string,
logger: Logger) {
logger: Logger
) {
queriesInput = queriesInput.trim();
// "+" means "don't override config file" - see shouldAddConfigFileQueries
queriesInput = queriesInput.replace(/^\+/, '');
queriesInput = queriesInput.replace(/^\+/, "");
for (const query of queriesInput.split(',')) {
for (const query of queriesInput.split(",")) {
await parseQueryUses(
languages,
codeQL,
@ -562,7 +679,8 @@ async function addQueriesFromWorkflow(
tempDir,
checkoutPath,
githubUrl,
logger);
logger
);
}
}
@ -572,7 +690,7 @@ async function addQueriesFromWorkflow(
// should instead be added in addition
function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
if (queriesInput) {
return queriesInput.trimStart().substr(0, 1) === '+';
return queriesInput.trimStart().substr(0, 1) === "+";
}
return true;
@ -591,14 +709,15 @@ export async function getDefaultConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
logger: Logger
): Promise<Config> {
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
logger
);
const queries = {};
await addDefaultQueries(codeQL, languages, queries);
if (queriesInput) {
@ -610,12 +729,13 @@ export async function getDefaultConfig(
tempDir,
checkoutPath,
githubUrl,
logger);
logger
);
}
return {
languages: languages,
queries: queries,
languages,
queries,
pathsIgnore: [],
paths: [],
originalUserInput: {},
@ -639,8 +759,8 @@ async function loadConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
logger: Logger
): Promise<Config> {
let parsedYAML: UserConfig;
if (isLocal(configFile)) {
@ -648,10 +768,7 @@ async function loadConfig(
configFile = path.resolve(checkoutPath, configFile);
parsedYAML = getLocalConfig(configFile, checkoutPath);
} else {
parsedYAML = await getRemoteConfig(
configFile,
githubAuth,
githubUrl);
parsedYAML = await getRemoteConfig(configFile, githubAuth, githubUrl);
}
// Validate that the 'name' property is syntactically correct,
@ -670,7 +787,8 @@ async function loadConfig(
repository,
githubAuth,
githubUrl,
logger);
logger
);
const queries = {};
const pathsIgnore: string[] = [];
@ -700,14 +818,21 @@ async function loadConfig(
tempDir,
checkoutPath,
githubUrl,
logger);
logger
);
}
if (shouldAddConfigFileQueries(queriesInput) && QUERIES_PROPERTY in parsedYAML) {
if (
shouldAddConfigFileQueries(queriesInput) &&
QUERIES_PROPERTY in parsedYAML
) {
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
throw new Error(getQueriesInvalid(configFile));
}
for (const query of parsedYAML[QUERIES_PROPERTY]!) {
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
if (
!(QUERIES_USES_PROPERTY in query) ||
typeof query[QUERIES_USES_PROPERTY] !== "string"
) {
throw new Error(getQueryUsesInvalid(configFile));
}
await parseQueryUses(
@ -719,7 +844,8 @@ async function loadConfig(
checkoutPath,
githubUrl,
logger,
configFile);
configFile
);
}
}
@ -727,11 +853,13 @@ async function loadConfig(
if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
throw new Error(getPathsIgnoreInvalid(configFile));
}
parsedYAML[PATHS_IGNORE_PROPERTY]!.forEach(path => {
if (typeof path !== "string" || path === '') {
parsedYAML[PATHS_IGNORE_PROPERTY]!.forEach((path) => {
if (typeof path !== "string" || path === "") {
throw new Error(getPathsIgnoreInvalid(configFile));
}
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile, logger));
pathsIgnore.push(
validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile, logger)
);
});
}
@ -739,11 +867,13 @@ async function loadConfig(
if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
throw new Error(getPathsInvalid(configFile));
}
parsedYAML[PATHS_PROPERTY]!.forEach(path => {
if (typeof path !== "string" || path === '') {
parsedYAML[PATHS_PROPERTY]!.forEach((path) => {
if (typeof path !== "string" || path === "") {
throw new Error(getPathsInvalid(configFile));
}
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile, logger));
paths.push(
validateAndSanitisePath(path, PATHS_PROPERTY, configFile, logger)
);
});
}
@ -751,8 +881,10 @@ async function loadConfig(
// it is a user configuration error.
for (const language of languages) {
if (queries[language] === undefined || queries[language].length === 0) {
throw new Error(`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
throw new Error(
`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run."
);
}
}
@ -785,13 +917,13 @@ export async function initConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
logger: Logger
): Promise<Config> {
let config: Config;
// If no config file was provided create an empty one
if (!configFile) {
logger.debug('No configuration file was provided');
logger.debug("No configuration file was provided");
config = await getDefaultConfig(
languagesInput,
queriesInput,
@ -802,7 +934,8 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
logger);
logger
);
} else {
config = await loadConfig(
languagesInput,
@ -815,7 +948,8 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
logger);
logger
);
}
// Save the config so we can easily access it again in the future
@ -829,7 +963,7 @@ function isLocal(configPath: string): boolean {
return true;
}
return (configPath.indexOf("@") === -1);
return configPath.indexOf("@") === -1;
}
function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
@ -843,28 +977,32 @@ function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
throw new Error(getConfigFileDoesNotExistErrorMessage(configFile));
}
return yaml.safeLoad(fs.readFileSync(configFile, 'utf8'));
return yaml.safeLoad(fs.readFileSync(configFile, "utf8"));
}
async function getRemoteConfig(
configFile: string,
githubAuth: string,
githubUrl: string): Promise<UserConfig> {
githubUrl: string
): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp('(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)');
const format = new RegExp(
"(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)"
);
const pieces = format.exec(configFile);
// 5 = 4 groups + the whole expression
if (pieces === null || pieces.groups === undefined || pieces.length < 5) {
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
}
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.getContents({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
ref: pieces.groups.ref,
});
const response = await api
.getApiClient(githubAuth, githubUrl, true)
.repos.getContents({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
ref: pieces.groups.ref,
});
let fileContents: string;
if ("content" in response.data && response.data.content !== undefined) {
@ -875,14 +1013,14 @@ async function getRemoteConfig(
throw new Error(getConfigFileFormatInvalidMessage(configFile));
}
return yaml.safeLoad(Buffer.from(fileContents, 'base64').toString('binary'));
return yaml.safeLoad(Buffer.from(fileContents, "base64").toString("binary"));
}
/**
* Get the file path where the parsed config will be stored.
*/
export function getPathToParsedConfigFile(tempDir: string): string {
return path.join(tempDir, 'config');
return path.join(tempDir, "config");
}
/**
@ -892,8 +1030,8 @@ async function saveConfig(config: Config, logger: Logger) {
const configString = JSON.stringify(config);
const configFile = getPathToParsedConfigFile(config.tempDir);
fs.mkdirSync(path.dirname(configFile), { recursive: true });
fs.writeFileSync(configFile, configString, 'utf8');
logger.debug('Saved config:');
fs.writeFileSync(configFile, configString, "utf8");
logger.debug("Saved config:");
logger.debug(configString);
}
@ -901,13 +1039,16 @@ async function saveConfig(config: Config, logger: Logger) {
* Get the config that has been saved to the given temp dir.
* If the config could not be found then returns undefined.
*/
export async function getConfig(tempDir: string, logger: Logger): Promise<Config | undefined> {
export async function getConfig(
tempDir: string,
logger: Logger
): Promise<Config | undefined> {
const configFile = getPathToParsedConfigFile(tempDir);
if (!fs.existsSync(configFile)) {
return undefined;
}
const configString = fs.readFileSync(configFile, 'utf8');
logger.debug('Loaded config:');
const configString = fs.readFileSync(configFile, "utf8");
logger.debug("Loaded config:");
logger.debug(configString);
return JSON.parse(configString);
}

17
src/error-matcher.test.ts
View file

@ -1,23 +1,26 @@
import test from "ava";
import test from 'ava';
import { namedMatchersForTesting } from './error-matcher';
import { namedMatchersForTesting } from "./error-matcher";
/*
NB We test the regexes for all the matchers against example log output snippets.
*/
test('noSourceCodeFound matches against example javascript output', async t => {
t.assert(testErrorMatcher("noSourceCodeFound", `
test("noSourceCodeFound matches against example javascript output", async (t) => {
t.assert(
testErrorMatcher(
"noSourceCodeFound",
`
2020-09-07T17:39:53.9050522Z [2020-09-07 17:39:53] [build] Done extracting /opt/hostedtoolcache/CodeQL/0.0.0-20200630/x64/codeql/javascript/tools/data/externs/web/ie_vml.js (3 ms)
2020-09-07T17:39:53.9051849Z [2020-09-07 17:39:53] [build-err] No JavaScript or TypeScript code found.
2020-09-07T17:39:53.9052444Z [2020-09-07 17:39:53] [build-err] No JavaScript or TypeScript code found.
2020-09-07T17:39:53.9251124Z [2020-09-07 17:39:53] [ERROR] Spawned process exited abnormally (code 255; tried to run: [/opt/hostedtoolcache/CodeQL/0.0.0-20200630/x64/codeql/javascript/tools/autobuild.sh])
`));
`
)
);
});
function testErrorMatcher(matcherName: string, logSample: string): boolean {
if (!(matcherName in namedMatchersForTesting)) {
throw new Error(`Unknown matcher ${matcherName}`);
}

12
src/error-matcher.ts
View file

@ -1,10 +1,9 @@
// defines properties to match against the result of executed commands,
// and a custom error to return when a match is found
export interface ErrorMatcher {
exitCode?: number; // exit code of the run process
outputRegex?: RegExp; // pattern to match against either stdout or stderr
message: string; // the error message that will be thrown for a matching process
exitCode?: number; // exit code of the run process
outputRegex?: RegExp; // pattern to match against either stdout or stderr
message: string; // the error message that will be thrown for a matching process
}
// exported only for testing purposes
@ -15,8 +14,9 @@ export const namedMatchersForTesting: { [key: string]: ErrorMatcher } = {
noSourceCodeFound: {
exitCode: 32,
outputRegex: new RegExp("No JavaScript or TypeScript code found\\."),
message: "No code found during the build. Please see:\n" +
"https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning#no-code-found-during-the-build"
message:
"No code found during the build. Please see:\n" +
"https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning#no-code-found-during-the-build",
},
};

92
src/external-queries.test.ts
View file

@ -1,45 +1,53 @@
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import test from 'ava';
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import test from "ava";
import * as fs from "fs";
import * as path from "path";
import * as externalQueries from "./external-queries";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test("checkoutExternalQueries", async t => {
await util.withTmpDir(async tmpDir => {
test("checkoutExternalQueries", async (t) => {
await util.withTmpDir(async (tmpDir) => {
// Create a test repo in a subdir of the temp dir.
// It should have a default branch with two commits after the initial commit, where
// - the first commit contains files 'a' and 'b'
// - the second commit contains only 'a'
// Place the repo in a subdir because we're going to checkout a copy in tmpDir
const testRepoBaseDir = path.join(tmpDir, 'test-repo-dir');
const repoName = 'some/repo';
const testRepoBaseDir = path.join(tmpDir, "test-repo-dir");
const repoName = "some/repo";
const repoPath = path.join(testRepoBaseDir, repoName);
const repoGitDir = path.join(repoPath, '.git');
const repoGitDir = path.join(repoPath, ".git");
// Run the given git command, and return the output.
// Passes --git-dir and --work-tree.
// Any stderr output is suppressed until the command fails.
const runGit = async function(command: string[]): Promise<string> {
let stdout = '';
let stderr = '';
command = [`--git-dir=${repoGitDir}`, `--work-tree=${repoPath}`, ...command];
console.log('Running: git ' + command.join(' '));
const runGit = async function (command: string[]): Promise<string> {
let stdout = "";
let stderr = "";
command = [
`--git-dir=${repoGitDir}`,
`--work-tree=${repoPath}`,
...command,
];
console.log(`Running: git ${command.join(" ")}`);
try {
await new toolrunnner.ToolRunner('git', command, {
await new toolrunnner.ToolRunner("git", command, {
silent: true,
listeners: {
stdout: (data) => { stdout += data.toString(); },
stderr: (data) => { stderr += data.toString(); },
}
stdout: (data) => {
stdout += data.toString();
},
stderr: (data) => {
stderr += data.toString();
},
},
}).exec();
} catch (e) {
console.log('Command failed: git ' + command.join(' '));
console.log(`Command failed: git ${command.join(" ")}`);
process.stderr.write(stderr);
throw e;
}
@ -47,25 +55,23 @@ test("checkoutExternalQueries", async t => {
};
fs.mkdirSync(repoPath, { recursive: true });
await runGit(['init', repoPath]);
await runGit(['config', 'user.email', 'test@github.com']);
await runGit(['config', 'user.name', 'Test Test']);
await runGit(["init", repoPath]);
await runGit(["config", "user.email", "test@github.com"]);
await runGit(["config", "user.name", "Test Test"]);
fs.writeFileSync(path.join(repoPath, 'a'), 'a content');
await runGit(['add', 'a']);
await runGit(['commit', '-m', 'commit1']);
fs.writeFileSync(path.join(repoPath, 'b'), 'b content');
await runGit(['add', 'b']);
await runGit(['commit', '-m', 'commit1']);
const commit1Sha = await runGit(['rev-parse', 'HEAD']);
fs.unlinkSync(path.join(repoPath, 'b'));
await runGit(['add', 'b']);
await runGit(['commit', '-m', 'commit2']);
const commit2Sha = await runGit(['rev-parse', 'HEAD']);
fs.writeFileSync(path.join(repoPath, "a"), "a content");
await runGit(["add", "a"]);
await runGit(["commit", "-m", "commit1"]);
fs.writeFileSync(path.join(repoPath, "b"), "b content");
await runGit(["add", "b"]);
await runGit(["commit", "-m", "commit1"]);
const commit1Sha = await runGit(["rev-parse", "HEAD"]);
fs.unlinkSync(path.join(repoPath, "b"));
await runGit(["add", "b"]);
await runGit(["commit", "-m", "commit2"]);
const commit2Sha = await runGit(["rev-parse", "HEAD"]);
// Checkout the first commit, which should contain 'a' and 'b'
t.false(fs.existsSync(path.join(tmpDir, repoName)));
@ -74,13 +80,12 @@ test("checkoutExternalQueries", async t => {
commit1Sha,
`file://${testRepoBaseDir}`,
tmpDir,
getRunnerLogger(true));
getRunnerLogger(true)
);
t.true(fs.existsSync(path.join(tmpDir, repoName)));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha)));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha, 'a')));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha, 'b')));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha, "a")));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha, "b")));
// Checkout the second commit as well, which should only contain 'a'
t.false(fs.existsSync(path.join(tmpDir, repoName, commit2Sha)));
@ -89,9 +94,10 @@ test("checkoutExternalQueries", async t => {
commit2Sha,
`file://${testRepoBaseDir}`,
tmpDir,
getRunnerLogger(true));
getRunnerLogger(true)
);
t.true(fs.existsSync(path.join(tmpDir, repoName, commit2Sha)));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit2Sha, 'a')));
t.false(fs.existsSync(path.join(tmpDir, repoName, commit2Sha, 'b')));
t.true(fs.existsSync(path.join(tmpDir, repoName, commit2Sha, "a")));
t.false(fs.existsSync(path.join(tmpDir, repoName, commit2Sha, "b")));
});
});

35
src/external-queries.ts
View file

@ -1,8 +1,8 @@
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from 'fs';
import * as path from 'path';
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import * as fs from "fs";
import * as path from "path";
import { Logger } from './logging';
import { Logger } from "./logging";
/**
* Check out repository at the given ref, and return the directory of the checkout.
@ -12,24 +12,31 @@ export async function checkoutExternalRepository(
ref: string,
githubUrl: string,
tempDir: string,
logger: Logger): Promise<string> {
logger.info('Checking out ' + repository);
logger: Logger
): Promise<string> {
logger.info(`Checking out ${repository}`);
const checkoutLocation = path.join(tempDir, repository, ref);
if (!checkoutLocation.startsWith(tempDir)) {
// this still permits locations that mess with sibling repositories in `tempDir`, but that is acceptable
throw new Error(`'${repository}@${ref}' is not a valid repository and reference.`);
throw new Error(
`'${repository}@${ref}' is not a valid repository and reference.`
);
}
if (!fs.existsSync(checkoutLocation)) {
const repoURL = githubUrl + '/' + repository;
await new toolrunnner.ToolRunner('git', ['clone', repoURL, checkoutLocation]).exec();
await new toolrunnner.ToolRunner('git', [
'--work-tree=' + checkoutLocation,
'--git-dir=' + checkoutLocation + '/.git',
'checkout', ref,
const repoURL = `${githubUrl}/${repository}`;
await new toolrunnner.ToolRunner("git", [
"clone",
repoURL,
checkoutLocation,
]).exec();
await new toolrunnner.ToolRunner("git", [
`--work-tree=${checkoutLocation}`,
`--git-dir=${checkoutLocation}/.git`,
"checkout",
ref,
]).exec();
}

259
src/fingerprints.test.ts
View file

@ -1,17 +1,17 @@
import test from 'ava';
import test from "ava";
import * as ava from "ava";
import * as fs from 'fs';
import * as path from 'path';
import * as fs from "fs";
import * as path from "path";
import * as fingerprints from './fingerprints';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as fingerprints from "./fingerprints";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
setupTests(test);
function testHash(t: ava.Assertions, input: string, expectedHashes: string[]) {
let index = 0;
let callback = function (lineNumber: number, hash: string) {
const callback = function (lineNumber: number, hash: string) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
@ -20,187 +20,188 @@ function testHash(t: ava.Assertions, input: string, expectedHashes: string[]) {
t.is(index, input.split(/\r\n|\r|\n/).length);
}
test('hash', (t: ava.Assertions) => {
test("hash", (t: ava.Assertions) => {
// Try empty file
testHash(t, "", ["c129715d7a2bc9a3:1"]);
// Try various combinations of newline characters
testHash(
t,
" a\nb\n \t\tc\n d",
[
"271789c17abda88f:1",
"54703d4cd895b18:1",
"180aee12dab6264:1",
"a23a3dc5e078b07b:1"
]);
testHash(
t,
" hello; \t\nworld!!!\n\n\n \t\tGreetings\n End",
[
"8b7cf3e952e7aeb2:1",
"b1ae1287ec4718d9:1",
"bff680108adb0fcc:1",
"c6805c5e1288b612:1",
"b86d3392aea1be30:1",
"e6ceba753e1a442:1",
]);
testHash(
t,
" hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n",
[
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(
t,
" hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r",
[
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(
t,
" hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n",
[
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(
t,
" hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n",
[
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " a\nb\n \t\tc\n d", [
"271789c17abda88f:1",
"54703d4cd895b18:1",
"180aee12dab6264:1",
"a23a3dc5e078b07b:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
"8b7cf3e952e7aeb2:1",
"b1ae1287ec4718d9:1",
"bff680108adb0fcc:1",
"c6805c5e1288b612:1",
"b86d3392aea1be30:1",
"e6ceba753e1a442:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
// Try repeating line that will generate identical hashes
testHash(
t,
"Lorem ipsum dolor sit amet.\n".repeat(10),
[
"a7f2ff13bc495cf2:1",
"a7f2ff13bc495cf2:2",
"a7f2ff13bc495cf2:3",
"a7f2ff13bc495cf2:4",
"a7f2ff13bc495cf2:5",
"a7f2ff13bc495cf2:6",
"a7f2ff1481e87703:1",
"a9cf91f7bbf1862b:1",
"55ec222b86bcae53:1",
"cc97dc7b1d7d8f7b:1",
"c129715d7a2bc9a3:1"
]);
testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
"a7f2ff13bc495cf2:1",
"a7f2ff13bc495cf2:2",
"a7f2ff13bc495cf2:3",
"a7f2ff13bc495cf2:4",
"a7f2ff13bc495cf2:5",
"a7f2ff13bc495cf2:6",
"a7f2ff1481e87703:1",
"a9cf91f7bbf1862b:1",
"55ec222b86bcae53:1",
"cc97dc7b1d7d8f7b:1",
"c129715d7a2bc9a3:1",
]);
testHash(
t,
"x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n",
[
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
"5c482a0f8b35ea28:1",
"54517377da7028d2:1",
"2c644846cb18d53e:1",
"f1b89f20de0d133:1",
"c129715d7a2bc9a3:1"
]);
testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
"5c482a0f8b35ea28:1",
"54517377da7028d2:1",
"2c644846cb18d53e:1",
"f1b89f20de0d133:1",
"c129715d7a2bc9a3:1",
]);
});
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
const location = { "uri": uri, "index": index };
const artifacts = artifactsURIs.map(uri => ({ "location": { "uri": uri } }));
return fingerprints.resolveUriToFile(location, artifacts, process.cwd(), getRunnerLogger(true));
const location = { uri, index };
const artifacts = artifactsURIs.map((uri) => ({ location: { uri } }));
return fingerprints.resolveUriToFile(
location,
artifacts,
process.cwd(),
getRunnerLogger(true)
);
}
test('resolveUriToFile', t => {
test("resolveUriToFile", (t) => {
// The resolveUriToFile method checks that the file exists and is in the right directory
// so we need to give it real files to look at. We will use this file as an example.
// For this to work we require the current working directory to be a parent, but this
// should generally always be the case so this is fine.
const cwd = process.cwd();
const filepath = __filename;
t.true(filepath.startsWith(cwd + '/'));
t.true(filepath.startsWith(`${cwd}/`));
const relativeFilepaht = filepath.substring(cwd.length + 1);
// Absolute paths are unmodified
t.is(testResolveUriToFile(filepath, undefined, []), filepath);
t.is(testResolveUriToFile('file://' + filepath, undefined, []), filepath);
t.is(testResolveUriToFile(`file://${filepath}`, undefined, []), filepath);
// Relative paths are made absolute
t.is(testResolveUriToFile(relativeFilepaht, undefined, []), filepath);
t.is(testResolveUriToFile('file://' + relativeFilepaht, undefined, []), filepath);
t.is(
testResolveUriToFile(`file://${relativeFilepaht}`, undefined, []),
filepath
);
// Absolute paths outside the src root are discarded
t.is(testResolveUriToFile('/src/foo/bar.js', undefined, []), undefined);
t.is(testResolveUriToFile('file:///src/foo/bar.js', undefined, []), undefined);
t.is(testResolveUriToFile("/src/foo/bar.js", undefined, []), undefined);
t.is(
testResolveUriToFile("file:///src/foo/bar.js", undefined, []),
undefined
);
// Other schemes are discarded
t.is(testResolveUriToFile('https://' + filepath, undefined, []), undefined);
t.is(testResolveUriToFile('ftp://' + filepath, undefined, []), undefined);
t.is(testResolveUriToFile(`https://${filepath}`, undefined, []), undefined);
t.is(testResolveUriToFile(`ftp://${filepath}`, undefined, []), undefined);
// Invalid URIs are discarded
t.is(testResolveUriToFile(1, undefined, []), undefined);
t.is(testResolveUriToFile(undefined, undefined, []), undefined);
// Non-existant files are discarded
t.is(testResolveUriToFile(filepath + '2', undefined, []), undefined);
t.is(testResolveUriToFile(`${filepath}2`, undefined, []), undefined);
// Index is resolved
t.is(testResolveUriToFile(undefined, 0, [filepath]), filepath);
t.is(testResolveUriToFile(undefined, 1, ['foo', filepath]), filepath);
t.is(testResolveUriToFile(undefined, 1, ["foo", filepath]), filepath);
// Invalid indexes are discarded
t.is(testResolveUriToFile(undefined, 1, [filepath]), undefined);
t.is(testResolveUriToFile(undefined, '0', [filepath]), undefined);
t.is(testResolveUriToFile(undefined, "0", [filepath]), undefined);
});
test('addFingerprints', t => {
test("addFingerprints", (t) => {
// Run an end-to-end test on a test file
let input = fs.readFileSync(__dirname + '/../src/testdata/fingerprinting.input.sarif').toString();
let expected = fs.readFileSync(__dirname + '/../src/testdata/fingerprinting.expected.sarif').toString();
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
.toString();
let expected = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.expected.sarif`)
.toString();
// The test files are stored prettified, but addFingerprints outputs condensed JSON
input = JSON.stringify(JSON.parse(input));
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)), expected);
t.deepEqual(
fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
expected
);
});
test('missingRegions', t => {
test("missingRegions", (t) => {
// Run an end-to-end test on a test file
let input = fs.readFileSync(__dirname + '/../src/testdata/fingerprinting2.input.sarif').toString();
let expected = fs.readFileSync(__dirname + '/../src/testdata/fingerprinting2.expected.sarif').toString();
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
.toString();
let expected = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.expected.sarif`)
.toString();
// The test files are stored prettified, but addFingerprints outputs condensed JSON
input = JSON.stringify(JSON.parse(input));
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)), expected);
t.deepEqual(
fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
expected
);
});

91
src/fingerprints.ts
View file

@ -1,12 +1,12 @@
import * as fs from 'fs';
import Long from 'long';
import * as fs from "fs";
import Long from "long";
import { Logger } from './logging';
import { Logger } from "./logging";
const tab = '\t'.charCodeAt(0);
const space = ' '.charCodeAt(0);
const lf = '\n'.charCodeAt(0);
const cr = '\r'.charCodeAt(0);
const tab = "\t".charCodeAt(0);
const space = " ".charCodeAt(0);
const lf = "\n".charCodeAt(0);
const cr = "\r".charCodeAt(0);
const BLOCK_SIZE = 100;
const MOD = Long.fromInt(37); // L
@ -46,7 +46,7 @@ export function hash(callback: hashCallback, input: string) {
// The current hash value, updated as we read each character
let hash = Long.ZERO;
let firstMod = computeFirstMod();
const firstMod = computeFirstMod();
// The current index in the window, will wrap around to zero when we reach BLOCK_SIZE
let index = 0;
@ -62,12 +62,12 @@ export function hash(callback: hashCallback, input: string) {
// Output the current hash and line number to the callback function
const outputHash = function () {
let hashValue = hash.toUnsigned().toString(16);
const hashValue = hash.toUnsigned().toString(16);
if (!hashCounts[hashValue]) {
hashCounts[hashValue] = 0;
}
hashCounts[hashValue]++;
callback(lineNumbers[index], hashValue + ":" + hashCounts[hashValue]);
callback(lineNumbers[index], `${hashValue}:${hashCounts[hashValue]}`);
lineNumbers[index] = -1;
};
@ -125,7 +125,11 @@ export function hash(callback: hashCallback, input: string) {
// Generate a hash callback function that updates the given result in-place
// when it recieves a hash for the correct line number. Ignores hashes for other lines.
function locationUpdateCallback(result: any, location: any, logger: Logger): hashCallback {
function locationUpdateCallback(
result: any,
location: any,
logger: Logger
): hashCallback {
let locationStartLine = location.physicalLocation?.region?.startLine;
if (locationStartLine === undefined) {
// We expect the region section to be present, but it can be absent if the
@ -142,17 +146,17 @@ function locationUpdateCallback(result: any, location: any, logger: Logger): has
if (!result.partialFingerprints) {
result.partialFingerprints = {};
}
const existingFingerprint = result.partialFingerprints.primaryLocationLineHash;
const existingFingerprint =
result.partialFingerprints.primaryLocationLineHash;
// If the hash doesn't match the existing fingerprint then
// output a warning and don't overwrite it.
if (!existingFingerprint) {
result.partialFingerprints.primaryLocationLineHash = hash;
} else if (existingFingerprint !== hash) {
logger.warning('Calculated fingerprint of ' + hash +
' for file ' + location.physicalLocation.artifactLocation.uri +
' line ' + lineNumber +
', but found existing inconsistent fingerprint value ' + existingFingerprint);
logger.warning(
`Calculated fingerprint of ${hash} for file ${location.physicalLocation.artifactLocation.uri} line ${lineNumber}, but found existing inconsistent fingerprint value ${existingFingerprint}`
);
}
};
}
@ -165,14 +169,16 @@ export function resolveUriToFile(
location: any,
artifacts: any[],
checkoutPath: string,
logger: Logger): string | undefined {
logger: Logger
): string | undefined {
// This may be referencing an artifact
if (!location.uri && location.index !== undefined) {
if (typeof location.index !== 'number' ||
if (
typeof location.index !== "number" ||
location.index < 0 ||
location.index >= artifacts.length ||
typeof artifacts[location.index].location !== 'object') {
typeof artifacts[location.index].location !== "object"
) {
logger.debug(`Ignoring location as URI "${location.index}" is invalid`);
return undefined;
}
@ -180,33 +186,37 @@ export function resolveUriToFile(
}
// Get the URI and decode
if (typeof location.uri !== 'string') {
if (typeof location.uri !== "string") {
logger.debug(`Ignoring location as index "${location.uri}" is invalid`);
return undefined;
}
let uri = decodeURIComponent(location.uri);
// Remove a file scheme, and abort if the scheme is anything else
const fileUriPrefix = 'file://';
const fileUriPrefix = "file://";
if (uri.startsWith(fileUriPrefix)) {
uri = uri.substring(fileUriPrefix.length);
}
if (uri.indexOf('://') !== -1) {
logger.debug(`Ignoring location URI "${uri}" as the scheme is not recognised`);
if (uri.indexOf("://") !== -1) {
logger.debug(
`Ignoring location URI "${uri}" as the scheme is not recognised`
);
return undefined;
}
// Discard any absolute paths that aren't in the src root
const srcRootPrefix = checkoutPath + '/';
if (uri.startsWith('/') && !uri.startsWith(srcRootPrefix)) {
logger.debug(`Ignoring location URI "${uri}" as it is outside of the src root`);
const srcRootPrefix = `${checkoutPath}/`;
if (uri.startsWith("/") && !uri.startsWith(srcRootPrefix)) {
logger.debug(
`Ignoring location URI "${uri}" as it is outside of the src root`
);
return undefined;
}
// Just assume a relative path is relative to the src root.
// This is not necessarily true but should be a good approximation
// and here we likely want to err on the side of handling more cases.
if (!uri.startsWith('/')) {
if (!uri.startsWith("/")) {
uri = srcRootPrefix + uri;
}
@ -221,21 +231,29 @@ export function resolveUriToFile(
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(sarifContents: string, checkoutPath: string, logger: Logger): string {
let sarif = JSON.parse(sarifContents);
export function addFingerprints(
sarifContents: string,
checkoutPath: string,
logger: Logger
): string {
const sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
// callbacks to accept hashes for that file and update the location
const callbacksByFile: { [filename: string]: hashCallback[] } = {};
for (const run of sarif.runs || []) {
// We may need the list of artifacts to resolve against
let artifacts = run.artifacts || [];
const artifacts = run.artifacts || [];
for (const result of run.results || []) {
// Check the primary location is defined correctly and is in the src root
const primaryLocation = (result.locations || [])[0];
if (!primaryLocation?.physicalLocation?.artifactLocation) {
logger.debug(`Unable to compute fingerprint for invalid location: ${JSON.stringify(primaryLocation)}`);
logger.debug(
`Unable to compute fingerprint for invalid location: ${JSON.stringify(
primaryLocation
)}`
);
continue;
}
@ -243,14 +261,17 @@ export function addFingerprints(sarifContents: string, checkoutPath: string, log
primaryLocation.physicalLocation.artifactLocation,
artifacts,
checkoutPath,
logger);
logger
);
if (!filepath) {
continue;
}
if (!callbacksByFile[filepath]) {
callbacksByFile[filepath] = [];
}
callbacksByFile[filepath].push(locationUpdateCallback(result, primaryLocation, logger));
callbacksByFile[filepath].push(
locationUpdateCallback(result, primaryLocation, logger)
);
}
}
@ -258,7 +279,7 @@ export function addFingerprints(sarifContents: string, checkoutPath: string, log
Object.entries(callbacksByFile).forEach(([filepath, callbacks]) => {
// A callback that forwards the hash to all other callbacks for that file
const teeCallback = function (lineNumber: number, hash: string) {
Object.values(callbacks).forEach(c => c(lineNumber, hash));
Object.values(callbacks).forEach((c) => c(lineNumber, hash));
};
const fileContents = fs.readFileSync(filepath).toString();
hash(teeCallback, fileContents);

146
src/init-action.ts
View file

@ -1,11 +1,11 @@
import * as core from '@actions/core';
import * as core from "@actions/core";
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from './init';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as util from './util';
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as util from "./util";
interface InitSuccessStatusReport extends util.StatusReportBase {
// Comma-separated list of languages that analysis was run for
@ -23,24 +23,39 @@ interface InitSuccessStatusReport extends util.StatusReportBase {
queries: string;
}
async function sendSuccessStatusReport(startedAt: Date, config: configUtils.Config) {
const statusReportBase = await util.createStatusReportBase('init', 'success', startedAt);
async function sendSuccessStatusReport(
startedAt: Date,
config: configUtils.Config
) {
const statusReportBase = await util.createStatusReportBase(
"init",
"success",
startedAt
);
const languages = config.languages.join(',');
const workflowLanguages = core.getInput('languages', { required: false });
const paths = (config.originalUserInput.paths || []).join(',');
const pathsIgnore = (config.originalUserInput['paths-ignore'] || []).join(',');
const disableDefaultQueries = config.originalUserInput['disable-default-queries'] ? languages : '';
const queries = (config.originalUserInput.queries || []).map(q => q.uses).join(',');
const languages = config.languages.join(",");
const workflowLanguages = core.getInput("languages", { required: false });
const paths = (config.originalUserInput.paths || []).join(",");
const pathsIgnore = (config.originalUserInput["paths-ignore"] || []).join(
","
);
const disableDefaultQueries = config.originalUserInput[
"disable-default-queries"
]
? languages
: "";
const queries = (config.originalUserInput.queries || [])
.map((q) => q.uses)
.join(",");
const statusReport: InitSuccessStatusReport = {
...statusReportBase,
languages: languages,
languages,
workflow_languages: workflowLanguages,
paths: paths,
paths,
paths_ignore: pathsIgnore,
disable_default_queries: disableDefaultQueries,
queries: queries,
queries,
};
await util.sendStatusReport(statusReport);
@ -54,75 +69,94 @@ async function run() {
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('init', 'starting', startedAt), true)) {
if (
!(await util.sendStatusReport(
await util.createStatusReportBase("init", "starting", startedAt),
true
))
) {
return;
}
codeql = await initCodeQL(
core.getInput('tools'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
'actions',
logger);
core.getInput("tools"),
core.getInput("token"),
util.getRequiredEnvParam("GITHUB_SERVER_URL"),
util.getRequiredEnvParam("RUNNER_TEMP"),
util.getRequiredEnvParam("RUNNER_TOOL_CACHE"),
"actions",
logger
);
config = await initConfig(
core.getInput('languages'),
core.getInput('queries'),
core.getInput('config-file'),
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
core.getInput("languages"),
core.getInput("queries"),
core.getInput("config-file"),
parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY")),
util.getRequiredEnvParam("RUNNER_TEMP"),
util.getRequiredEnvParam("RUNNER_TOOL_CACHE"),
codeql,
util.getRequiredEnvParam('GITHUB_WORKSPACE'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
logger);
util.getRequiredEnvParam("GITHUB_WORKSPACE"),
core.getInput("token"),
util.getRequiredEnvParam("GITHUB_SERVER_URL"),
logger
);
} catch (e) {
core.setFailed(e.message);
console.log(e);
await util.sendStatusReport(await util.createStatusReportBase('init', 'aborted', startedAt, e.message));
await util.sendStatusReport(
await util.createStatusReportBase("init", "aborted", startedAt, e.message)
);
return;
}
try {
// Forward Go flags
const goFlags = process.env['GOFLAGS'];
const goFlags = process.env["GOFLAGS"];
if (goFlags) {
core.exportVariable('GOFLAGS', goFlags);
core.warning("Passing the GOFLAGS env parameter to the init action is deprecated. Please move this to the analyze action.");
core.exportVariable("GOFLAGS", goFlags);
core.warning(
"Passing the GOFLAGS env parameter to the init action is deprecated. Please move this to the analyze action."
);
}
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
const codeqlRam = process.env["CODEQL_RAM"] || "6500";
core.exportVariable("CODEQL_RAM", codeqlRam);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig !== undefined) {
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
Object.entries(tracerConfig.env).forEach(([key, value]) =>
core.exportVariable(key, value)
);
if (process.platform === 'win32') {
await injectWindowsTracer('Runner.Worker.exe', undefined, config, codeql, tracerConfig);
if (process.platform === "win32") {
await injectWindowsTracer(
"Runner.Worker.exe",
undefined,
config,
codeql,
tracerConfig
);
}
}
} catch (error) {
core.setFailed(error.message);
console.log(error);
await util.sendStatusReport(await util.createStatusReportBase(
'init',
'failure',
startedAt,
error.message,
error.stack));
await util.sendStatusReport(
await util.createStatusReportBase(
"init",
"failure",
startedAt,
error.message,
error.stack
)
);
return;
}
await sendSuccessStatusReport(startedAt, config);
}
run().catch(e => {
core.setFailed("init action failed: " + e);
run().catch((e) => {
core.setFailed(`init action failed: ${e}`);
console.log(e);
});

74
src/init.ts
View file

@ -1,14 +1,14 @@
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from 'fs';
import * as path from 'path';
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import * as fs from "fs";
import * as path from "path";
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import { getCombinedTracerConfig, TracerConfig } from './tracer-config';
import * as util from './util';
import * as analysisPaths from "./analysis-paths";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
export async function initCodeQL(
codeqlURL: string | undefined,
@ -17,9 +17,9 @@ export async function initCodeQL(
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
logger.startGroup('Setup CodeQL tools');
logger: Logger
): Promise<CodeQL> {
logger.startGroup("Setup CodeQL tools");
const codeql = await setupCodeQL(
codeqlURL,
githubAuth,
@ -27,7 +27,8 @@ export async function initCodeQL(
tempDir,
toolsDir,
mode,
logger);
logger
);
await codeql.printVersion();
logger.endGroup();
return codeql;
@ -44,9 +45,9 @@ export async function initConfig(
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<configUtils.Config> {
logger.startGroup('Load language configuration');
logger: Logger
): Promise<configUtils.Config> {
logger.startGroup("Load language configuration");
const config = await configUtils.initConfig(
languagesInput,
queriesInput,
@ -58,7 +59,8 @@ export async function initConfig(
checkoutPath,
githubAuth,
githubUrl,
logger);
logger
);
analysisPaths.printPathFiltersWarning(config, logger);
logger.endGroup();
return config;
@ -66,16 +68,20 @@ export async function initConfig(
export async function runInit(
codeql: CodeQL,
config: configUtils.Config): Promise<TracerConfig | undefined> {
config: configUtils.Config
): Promise<TracerConfig | undefined> {
const sourceRoot = path.resolve();
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
for (const language of config.languages) {
// Init language database
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
await codeql.databaseInit(
util.getCodeQLDatabasePath(config.tempDir, language),
language,
sourceRoot
);
}
return await getCombinedTracerConfig(config, codeql);
@ -91,8 +97,8 @@ export async function injectWindowsTracer(
processLevel: number | undefined,
config: configUtils.Config,
codeql: CodeQL,
tracerConfig: TracerConfig) {
tracerConfig: TracerConfig
) {
let script: string;
if (processName !== undefined) {
script = `
@ -155,15 +161,23 @@ export async function injectWindowsTracer(
Invoke-Expression "&$tracer --inject=$id"`;
}
const injectTracerPath = path.join(config.tempDir, 'inject-tracer.ps1');
const injectTracerPath = path.join(config.tempDir, "inject-tracer.ps1");
fs.writeFileSync(injectTracerPath, script);
await new toolrunnner.ToolRunner(
'powershell',
"powershell",
[
'-ExecutionPolicy', 'Bypass',
'-file', injectTracerPath,
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
"-ExecutionPolicy",
"Bypass",
"-file",
injectTracerPath,
path.resolve(
path.dirname(codeql.getPath()),
"tools",
"win64",
"tracer.exe"
),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } }).exec();
{ env: { ODASA_TRACER_CONFIGURATION: tracerConfig.spec } }
).exec();
}

43
src/languages.test.ts
View file

@ -1,32 +1,37 @@
import test from 'ava';
import test from "ava";
import {isScannedLanguage, isTracedLanguage, Language, parseLanguage} from './languages';
import {setupTests} from './testing-utils';
import {
Language,
isScannedLanguage,
isTracedLanguage,
parseLanguage,
} from "./languages";
import { setupTests } from "./testing-utils";
setupTests(test);
test('parseLangauge', async t => {
test("parseLangauge", async (t) => {
// Exact matches
t.deepEqual(parseLanguage('csharp'), Language.csharp);
t.deepEqual(parseLanguage('cpp'), Language.cpp);
t.deepEqual(parseLanguage('go'), Language.go);
t.deepEqual(parseLanguage('java'), Language.java);
t.deepEqual(parseLanguage('javascript'), Language.javascript);
t.deepEqual(parseLanguage('python'), Language.python);
t.deepEqual(parseLanguage("csharp"), Language.csharp);
t.deepEqual(parseLanguage("cpp"), Language.cpp);
t.deepEqual(parseLanguage("go"), Language.go);
t.deepEqual(parseLanguage("java"), Language.java);
t.deepEqual(parseLanguage("javascript"), Language.javascript);
t.deepEqual(parseLanguage("python"), Language.python);
// Aliases
t.deepEqual(parseLanguage('c'), Language.cpp);
t.deepEqual(parseLanguage('c++'), Language.cpp);
t.deepEqual(parseLanguage('c#'), Language.csharp);
t.deepEqual(parseLanguage('typescript'), Language.javascript);
t.deepEqual(parseLanguage("c"), Language.cpp);
t.deepEqual(parseLanguage("c++"), Language.cpp);
t.deepEqual(parseLanguage("c#"), Language.csharp);
t.deepEqual(parseLanguage("typescript"), Language.javascript);
// Not matches
t.deepEqual(parseLanguage('foo'), undefined);
t.deepEqual(parseLanguage(' '), undefined);
t.deepEqual(parseLanguage(''), undefined);
t.deepEqual(parseLanguage("foo"), undefined);
t.deepEqual(parseLanguage(" "), undefined);
t.deepEqual(parseLanguage(""), undefined);
});
test('isTracedLanguage', async t => {
test("isTracedLanguage", async (t) => {
t.true(isTracedLanguage(Language.cpp));
t.true(isTracedLanguage(Language.java));
t.true(isTracedLanguage(Language.csharp));
@ -36,7 +41,7 @@ test('isTracedLanguage', async t => {
t.false(isTracedLanguage(Language.python));
});
test('isScannedLanguage', async t => {
test("isScannedLanguage", async (t) => {
t.false(isScannedLanguage(Language.cpp));
t.false(isScannedLanguage(Language.java));
t.false(isScannedLanguage(Language.csharp));

25
src/languages.ts
View file

@ -1,19 +1,19 @@
// All the languages supported by CodeQL
export enum Language {
csharp = 'csharp',
cpp = 'cpp',
go = 'go',
java = 'java',
javascript = 'javascript',
python = 'python',
csharp = "csharp",
cpp = "cpp",
go = "go",
java = "java",
javascript = "javascript",
python = "python",
}
// Additional names for languages
const LANGUAGE_ALIASES: {[lang: string]: Language} = {
'c': Language.cpp,
'c++': Language.cpp,
'c#': Language.csharp,
'typescript': Language.javascript,
const LANGUAGE_ALIASES: { [lang: string]: Language } = {
c: Language.cpp,
"c++": Language.cpp,
"c#": Language.csharp,
typescript: Language.javascript,
};
// Translate from user input or GitHub's API names for languages to CodeQL's names for languages
@ -34,9 +34,8 @@ export function parseLanguage(language: string): Language | undefined {
return undefined;
}
export function isTracedLanguage(language: Language): boolean {
return ['cpp', 'java', 'csharp'].includes(language);
return ["cpp", "java", "csharp"].includes(language);
}
export function isScannedLanguage(language: Language): boolean {

2
src/logging.ts
View file

@ -1,4 +1,4 @@
import * as core from '@actions/core';
import * as core from "@actions/core";
export interface Logger {
debug: (message: string) => void;

2
src/repository.ts
View file

@ -5,7 +5,7 @@ export interface RepositoryNwo {
}
export function parseRepositoryNwo(input: string): RepositoryNwo {
const parts = input.split('/');
const parts = input.split("/");
if (parts.length !== 2) {
throw new Error(`"${input}" is not a valid repository name`);
}

301
src/runner.ts
View file

@ -1,21 +1,21 @@
import { Command } from 'commander';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { Command } from "commander";
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import { runAnalyze } from './analyze';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import { CodeQL, getCodeQL } from './codeql';
import { Config, getConfig } from './config-utils';
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from './init';
import { Language, parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import { getAddSnippetsFlag, getMemoryFlag, getThreadsFlag } from './util';
import { runAnalyze } from "./analyze";
import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
import { CodeQL, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
import { Language, parseLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
import { getAddSnippetsFlag, getMemoryFlag, getThreadsFlag } from "./util";
const program = new Command();
program.version('0.0.1');
program.version("0.0.1");
function parseGithubUrl(inputUrl: string): string {
try {
@ -23,24 +23,23 @@ function parseGithubUrl(inputUrl: string): string {
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://github.com';
if (url.hostname === "github.com" || url.hostname === "api.github.com") {
return "https://github.com";
}
// Remove the API prefix if it's present
if (url.pathname.indexOf('/api/v3') !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf('/api/v3'));
if (url.pathname.indexOf("/api/v3") !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf("/api/v3"));
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
function getTempDir(userInput: string | undefined): string {
const tempDir = path.join(userInput || process.cwd(), 'codeql-runner');
const tempDir = path.join(userInput || process.cwd(), "codeql-runner");
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
@ -48,38 +47,38 @@ function getTempDir(userInput: string | undefined): string {
}
function getToolsDir(userInput: string | undefined): string {
const toolsDir = userInput || path.join(os.homedir(), 'codeql-runner-tools');
const toolsDir = userInput || path.join(os.homedir(), "codeql-runner-tools");
if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true });
}
return toolsDir;
}
const codeqlEnvJsonFilename = 'codeql-env.json';
const codeqlEnvJsonFilename = "codeql-env.json";
// Imports the environment from codeqlEnvJsonFilename if not already present
function importTracerEnvironment(config: Config) {
if (!('ODASA_TRACER_CONFIGURATION' in process.env)) {
if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString('utf-8'));
Object.keys(env).forEach(key => process.env[key] = env[key]);
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
Object.keys(env).forEach((key) => (process.env[key] = env[key]));
}
}
// Allow the user to specify refs in full refs/heads/branch format
// or just the short branch name and prepend "refs/heads/" to it.
function parseRef(userInput: string): string {
if (userInput.startsWith('refs/')) {
if (userInput.startsWith("refs/")) {
return userInput;
} else {
return 'refs/heads/' + userInput;
return `refs/heads/${userInput}`;
}
}
// Parses the --trace-process-name arg from process.argv, or returns undefined
function parseTraceProcessName(): string | undefined {
for (let i = 0; i < process.argv.length - 1; i++) {
if (process.argv[i] === '--trace-process-name') {
if (process.argv[i] === "--trace-process-name") {
return process.argv[i + 1];
}
}
@ -89,7 +88,7 @@ function parseTraceProcessName(): string | undefined {
// Parses the --trace-process-level arg from process.argv, or returns undefined
function parseTraceProcessLevel(): number | undefined {
for (let i = 0; i < process.argv.length - 1; i++) {
if (process.argv[i] === '--trace-process-level') {
if (process.argv[i] === "--trace-process-level") {
const v = parseInt(process.argv[i + 1], 10);
return isNaN(v) ? undefined : v;
}
@ -112,19 +111,40 @@ interface InitArgs {
}
program
.command('init')
.description('Initializes CodeQL')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--languages <languages>', 'Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo.')
.option('--queries <queries>', 'Comma-separated list of additional queries to run. This overrides the same setting in a configuration file.')
.option('--config-file <file>', 'Path to config file.')
.option('--codeql-path <path>', 'Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--tools-dir <dir>', 'Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory.')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.command("init")
.description("Initializes CodeQL")
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.requiredOption(
"--github-auth <auth>",
"GitHub Apps token or personal access token. (Required)"
)
.option(
"--languages <languages>",
"Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo."
)
.option(
"--queries <queries>",
"Comma-separated list of additional queries to run. This overrides the same setting in a configuration file."
)
.option("--config-file <file>", "Path to config file.")
.option(
"--codeql-path <path>",
"Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option(
"--tools-dir <dir>",
"Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory."
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--debug", "Print more verbose output", false)
// This prevents a message like: error: unknown option '--trace-process-level'
// Remove this if commander.js starts supporting hidden options.
.allowUnknownOption()
@ -149,8 +169,9 @@ program
parseGithubUrl(cmd.githubUrl),
tempDir,
toolsDir,
'runner',
logger);
"runner",
logger
);
}
const config = await initConfig(
@ -164,60 +185,64 @@ program
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
logger);
logger
);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig === undefined) {
return;
}
if (process.platform === 'win32') {
if (process.platform === "win32") {
await injectWindowsTracer(
parseTraceProcessName(),
parseTraceProcessLevel(),
config,
codeql,
tracerConfig);
tracerConfig
);
}
// Always output a json file of the env that can be consumed programatically
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
fs.writeFileSync(jsonEnvFile, JSON.stringify(tracerConfig.env));
if (process.platform === 'win32') {
const batEnvFile = path.join(config.tempDir, 'codeql-env.bat');
if (process.platform === "win32") {
const batEnvFile = path.join(config.tempDir, "codeql-env.bat");
const batEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `Set ${key}=${value}`)
.join('\n');
.join("\n");
fs.writeFileSync(batEnvFile, batEnvFileContents);
const powershellEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const powershellEnvFile = path.join(config.tempDir, "codeql-env.sh");
const powershellEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `$env:${key}="${value}"`)
.join('\n');
.join("\n");
fs.writeFileSync(powershellEnvFile, powershellEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`);
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`
);
} else {
// Assume that anything that's not windows is using a unix-style shell
const shEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const shEnvFile = path.join(config.tempDir, "codeql-env.sh");
const shEnvFileContents = Object.entries(tracerConfig.env)
// Some vars contain ${LIB} that we do not want to be expanded when executing this script
.map(([key, value]) => `export ${key}="${value.replace('$', '\\$')}"`)
.join('\n');
.map(([key, value]) => `export ${key}="${value.replace("$", "\\$")}"`)
.join("\n");
fs.writeFileSync(shEnvFile, shEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`for example by running ". ${shEnvFile}".`);
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`for example by running ". ${shEnvFile}".`
);
}
} catch (e) {
logger.error('Init failed');
logger.error("Init failed");
logger.error(e);
process.exitCode = 1;
}
@ -230,26 +255,38 @@ interface AutobuildArgs {
}
program
.command('autobuild')
.description('Attempts to automatically build code')
.option('--language <language>', 'The language to build. Otherwise will detect the dominant compiled language.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.command("autobuild")
.description("Attempts to automatically build code")
.option(
"--language <language>",
"The language to build. Otherwise will detect the dominant compiled language."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: AutobuildArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
throw new Error(
"Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
importTracerEnvironment(config);
let language: Language | undefined = undefined;
if (cmd.language !== undefined) {
language = parseLanguage(cmd.language);
if (language === undefined || !config.languages.includes(language)) {
throw new Error(`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(', ')}.`);
throw new Error(
`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(
", "
)}.`
);
}
} else {
language = determineAutobuildLanguage(config, logger);
@ -258,7 +295,7 @@ program
await runAutobuild(language, config, logger);
}
} catch (e) {
logger.error('Autobuild failed');
logger.error("Autobuild failed");
logger.error(e);
process.exitCode = 1;
}
@ -281,31 +318,57 @@ interface AnalyzeArgs {
}
program
.command('analyze')
.description('Finishes extracting code and runs CodeQL queries')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--no-upload', 'Do not upload results after analysis.')
.option('--output-dir <dir>', 'Directory to output SARIF files to. Default is in the temp directory.')
.option('--ram <ram>', 'Amount of memory to use when running queries. Default is to use all available memory.')
.option('--no-add-snippets', 'Specify whether to include code snippets in the sarif output.')
.option('--threads <threads>', 'Number of threads to use when running queries. ' +
'Default is to use all available cores.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.command("analyze")
.description("Finishes extracting code and runs CodeQL queries")
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption(
"--commit <commit>",
"SHA of commit that was analyzed. (Required)"
)
.requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.requiredOption(
"--github-auth <auth>",
"GitHub Apps token or personal access token. (Required)"
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--no-upload", "Do not upload results after analysis.")
.option(
"--output-dir <dir>",
"Directory to output SARIF files to. Default is in the temp directory."
)
.option(
"--ram <ram>",
"Amount of memory to use when running queries. Default is to use all available memory."
)
.option(
"--no-add-snippets",
"Specify whether to include code snippets in the sarif output."
)
.option(
"--threads <threads>",
"Number of threads to use when running queries. " +
"Default is to use all available cores."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: AnalyzeArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const tempDir = getTempDir(cmd.tempDir);
const outputDir = cmd.outputDir || path.join(tempDir, 'codeql-sarif');
const outputDir = cmd.outputDir || path.join(tempDir, "codeql-sarif");
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
throw new Error(
"Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
await runAnalyze(
parseRepositoryNwo(cmd.repository),
@ -319,15 +382,16 @@ program
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
cmd.upload,
'runner',
"runner",
outputDir,
getMemoryFlag(cmd.ram),
getAddSnippetsFlag(cmd.addSnippets),
getThreadsFlag(cmd.threads, logger),
config,
logger);
logger
);
} catch (e) {
logger.error('Analyze failed');
logger.error("Analyze failed");
logger.error(e);
process.exitCode = 1;
}
@ -345,16 +409,30 @@ interface UploadArgs {
}
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload, or a directory containing multiple SARIF files. (Required)')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.command("upload")
.description(
"Uploads a SARIF file, or all SARIF files from a directory, to code scanning"
)
.requiredOption(
"--sarif-file <file>",
"SARIF file to upload, or a directory containing multiple SARIF files. (Required)"
)
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption(
"--commit <commit>",
"SHA of commit that was analyzed. (Required)"
)
.requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.requiredOption(
"--github-auth <auth>",
"GitHub Apps token or personal access token. (Required)"
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: UploadArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
@ -370,10 +448,11 @@ program
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
'runner',
logger);
"runner",
logger
);
} catch (e) {
logger.error('Upload failed');
logger.error("Upload failed");
logger.error(e);
process.exitCode = 1;
}

4
src/shared-environment.ts
View file

@ -1,7 +1,7 @@
export const ODASA_TRACER_CONFIGURATION = 'ODASA_TRACER_CONFIGURATION';
export const ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION";
// The time at which the first action (normally init) started executing.
// If a workflow invokes a different action without first invoking the init
// action (i.e. the upload action is being used by a third-party integrator)
// then this variable will be assigned the start time of the action invoked
// rather that the init action.
export const CODEQL_WORKFLOW_STARTED_AT = 'CODEQL_WORKFLOW_STARTED_AT';
export const CODEQL_WORKFLOW_STARTED_AT = "CODEQL_WORKFLOW_STARTED_AT";

31
src/testing-utils.ts
View file

@ -1,31 +1,40 @@
import {TestInterface} from 'ava';
import sinon from 'sinon';
import { TestInterface } from "ava";
import sinon from "sinon";
import * as CodeQL from './codeql';
import * as CodeQL from "./codeql";
type TestContext = {stdoutWrite: any, stderrWrite: any, testOutput: string, env: NodeJS.ProcessEnv};
type TestContext = {
stdoutWrite: any;
stderrWrite: any;
testOutput: string;
env: NodeJS.ProcessEnv;
};
function wrapOutput(context: TestContext) {
// Function signature taken from Socket.write.
// Note there are two overloads:
// write(buffer: Uint8Array | string, cb?: (err?: Error) => void): boolean;
// write(str: Uint8Array | string, encoding?: string, cb?: (err?: Error) => void): boolean;
return (chunk: Uint8Array | string, encoding?: string, cb?: (err?: Error) => void): boolean => {
return (
chunk: Uint8Array | string,
encoding?: string,
cb?: (err?: Error) => void
): boolean => {
// Work out which method overload we are in
if (cb === undefined && typeof encoding === 'function') {
if (cb === undefined && typeof encoding === "function") {
cb = encoding;
encoding = undefined;
}
// Record the output
if (typeof chunk === 'string') {
if (typeof chunk === "string") {
context.testOutput += chunk;
} else {
context.testOutput += new TextDecoder(encoding || 'utf-8').decode(chunk);
context.testOutput += new TextDecoder(encoding || "utf-8").decode(chunk);
}
// Satisfy contract by calling callback when done
if (cb !== undefined && typeof cb === 'function') {
if (cb !== undefined && typeof cb === "function") {
cb();
}
@ -36,7 +45,7 @@ function wrapOutput(context: TestContext) {
export function setupTests(test: TestInterface<any>) {
const typedTest = test as TestInterface<TestContext>;
typedTest.beforeEach(t => {
typedTest.beforeEach((t) => {
// Set an empty CodeQL object so that all method calls will fail
// unless the test explicitly sets one up.
CodeQL.setCodeQL({});
@ -57,7 +66,7 @@ export function setupTests(test: TestInterface<any>) {
Object.assign(t.context.env, process.env);
});
typedTest.afterEach.always(t => {
typedTest.afterEach.always((t) => {
// Restore stdout and stderr
// The captured output is only replayed if the test failed
process.stdout.write = t.context.stdoutWrite;

287
src/toolrunner-error-catcher.test.ts
View file

@ -1,150 +1,209 @@
import * as exec from '@actions/exec';
import test from 'ava';
import * as exec from "@actions/exec";
import test from "ava";
import { ErrorMatcher } from './error-matcher';
import {setupTests} from './testing-utils';
import { toolrunnerErrorCatcher } from './toolrunner-error-catcher';
import { ErrorMatcher } from "./error-matcher";
import { setupTests } from "./testing-utils";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
setupTests(test);
test('matchers are never applied if non-error exit', async t => {
test("matchers are never applied if non-error exit", async (t) => {
const testArgs = buildDummyArgs(
"foo bar\\nblort qux",
"foo bar\\nblort qux",
"",
0
);
const testArgs = buildDummyArgs("foo bar\\nblort qux", "foo bar\\nblort qux", '', 0);
const matchers: ErrorMatcher[] = [
{ exitCode: 123, outputRegex: new RegExp("foo bar"), message: "error!!!" },
];
const matchers: ErrorMatcher[] = [{exitCode: 123, outputRegex: new RegExp("foo bar"), message: 'error!!!' }];
t.deepEqual(await exec.exec('node', testArgs), 0);
t.deepEqual(await toolrunnerErrorCatcher('node', testArgs, matchers), 0);
t.deepEqual(await exec.exec("node", testArgs), 0);
t.deepEqual(await toolrunnerErrorCatcher("node", testArgs, matchers), 0);
});
test('regex matchers are applied to stdout for non-zero exit code', async t => {
test("regex matchers are applied to stdout for non-zero exit code", async (t) => {
const testArgs = buildDummyArgs("foo bar\\nblort qux", "", "", 1);
const testArgs = buildDummyArgs("foo bar\\nblort qux", '', '', 1);
const matchers: ErrorMatcher[] = [
{ exitCode: 123, outputRegex: new RegExp("foo bar"), message: "🦄" },
];
const matchers: ErrorMatcher[] = [{exitCode: 123, outputRegex: new RegExp("foo bar"), message: '🦄' }];
await t.throwsAsync(exec.exec("node", testArgs), {
instanceOf: Error,
message: "The process 'node' failed with exit code 1",
});
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, matchers), {
instanceOf: Error,
message: "🦄",
});
});
test("regex matchers are applied to stderr for non-zero exit code", async (t) => {
const testArgs = buildDummyArgs(
"non matching string",
"foo bar\\nblort qux",
"",
1
);
const matchers: ErrorMatcher[] = [
{ exitCode: 123, outputRegex: new RegExp("foo bar"), message: "🦄" },
];
await t.throwsAsync(exec.exec("node", testArgs), {
instanceOf: Error,
message: "The process 'node' failed with exit code 1",
});
await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, matchers), {
instanceOf: Error,
message: "🦄",
});
});
test("matcher returns correct error message when multiple matchers defined", async (t) => {
const testArgs = buildDummyArgs(
"non matching string",
"foo bar\\nblort qux",
"",
1
);
const matchers: ErrorMatcher[] = [
{ exitCode: 456, outputRegex: new RegExp("lorem ipsum"), message: "😩" },
{ exitCode: 123, outputRegex: new RegExp("foo bar"), message: "🦄" },
{ exitCode: 789, outputRegex: new RegExp("blah blah"), message: "🤦‍♂️" },
];
await t.throwsAsync(exec.exec("node", testArgs), {
instanceOf: Error,
message: "The process 'node' failed with exit code 1",
});
await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, matchers), {
instanceOf: Error,
message: "🦄",
});
});
test("matcher returns first match to regex when multiple matches", async (t) => {
const testArgs = buildDummyArgs(
"non matching string",
"foo bar\\nblort qux",
"",
1
);
const matchers: ErrorMatcher[] = [
{ exitCode: 123, outputRegex: new RegExp("foo bar"), message: "🦄" },
{ exitCode: 789, outputRegex: new RegExp("blah blah"), message: "🤦‍♂️" },
{ exitCode: 987, outputRegex: new RegExp("foo bar"), message: "🚫" },
];
await t.throwsAsync(exec.exec("node", testArgs), {
instanceOf: Error,
message: "The process 'node' failed with exit code 1",
});
await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, matchers), {
instanceOf: Error,
message: "🦄",
});
});
test("exit code matchers are applied", async (t) => {
const testArgs = buildDummyArgs(
"non matching string",
"foo bar\\nblort qux",
"",
123
);
const matchers: ErrorMatcher[] = [
{
exitCode: 123,
outputRegex: new RegExp("this will not match"),
message: "🦄",
},
];
await t.throwsAsync(exec.exec("node", testArgs), {
instanceOf: Error,
message: "The process 'node' failed with exit code 123",
});
await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, matchers), {
instanceOf: Error,
message: "🦄",
});
});
test("execErrorCatcher respects the ignoreReturnValue option", async (t) => {
const testArgs = buildDummyArgs("standard output", "error output", "", 199);
await t.throwsAsync(
toolrunnerErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
toolrunnerErrorCatcher("node", testArgs, [], { ignoreReturnCode: false }),
{ instanceOf: Error }
);
t.deepEqual(
await toolrunnerErrorCatcher("node", testArgs, [], {
ignoreReturnCode: true,
}),
199
);
});
test('regex matchers are applied to stderr for non-zero exit code', async t => {
test("execErrorCatcher preserves behavior of provided listeners", async (t) => {
const stdoutExpected = "standard output";
const stderrExpected = "error output";
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 1);
let stdoutActual = "";
let stderrActual = "";
const matchers: ErrorMatcher[] = [{exitCode: 123, outputRegex: new RegExp("foo bar"), message: '🦄' }];
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
toolrunnerErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
});
test('matcher returns correct error message when multiple matchers defined', async t => {
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 1);
const matchers: ErrorMatcher[] = [{exitCode: 456, outputRegex: new RegExp("lorem ipsum"), message: '😩'},
{exitCode: 123, outputRegex: new RegExp("foo bar"), message: '🦄'},
{exitCode: 789, outputRegex: new RegExp("blah blah"), message: '🤦‍♂️'}];
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
toolrunnerErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
});
test('matcher returns first match to regex when multiple matches', async t => {
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 1);
const matchers: ErrorMatcher[] = [{exitCode: 123, outputRegex: new RegExp("foo bar"), message: '🦄'},
{exitCode: 789, outputRegex: new RegExp("blah blah"), message: '🤦‍♂️'},
{exitCode: 987, outputRegex: new RegExp("foo bar"), message: '🚫'}];
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
toolrunnerErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
});
test('exit code matchers are applied', async t => {
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 123);
const matchers: ErrorMatcher[] = [{exitCode: 123, outputRegex: new RegExp("this will not match"), message: '🦄' }];
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 123'});
await t.throwsAsync(
toolrunnerErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
});
test('execErrorCatcher respects the ignoreReturnValue option', async t => {
const testArgs = buildDummyArgs("standard output", 'error output', '', 199);
await t.throwsAsync(toolrunnerErrorCatcher('node', testArgs, [], {ignoreReturnCode: false}), {instanceOf: Error});
t.deepEqual(await toolrunnerErrorCatcher('node', testArgs, [], {ignoreReturnCode: true}), 199);
});
test('execErrorCatcher preserves behavior of provided listeners', async t => {
let stdoutExpected = 'standard output';
let stderrExpected = 'error output';
let stdoutActual = '';
let stderrActual = '';
let listeners = {
const listeners = {
stdout: (data: Buffer) => {
stdoutActual += data.toString();
},
stderr: (data: Buffer) => {
stderrActual += data.toString();
}
},
};
const testArgs = buildDummyArgs(stdoutExpected, stderrExpected, '', 0);
const testArgs = buildDummyArgs(stdoutExpected, stderrExpected, "", 0);
t.deepEqual(await toolrunnerErrorCatcher('node', testArgs, [], {listeners: listeners}), 0);
t.deepEqual(stdoutActual, stdoutExpected + "\n");
t.deepEqual(stderrActual, stderrExpected + "\n");
t.deepEqual(
await toolrunnerErrorCatcher("node", testArgs, [], {
listeners,
}),
0
);
t.deepEqual(stdoutActual, `${stdoutExpected}\n`);
t.deepEqual(stderrActual, `${stderrExpected}\n`);
});
function buildDummyArgs(stdoutContents: string, stderrContents: string,
desiredErrorMessage?: string, desiredExitCode?: number): string[] {
function buildDummyArgs(
stdoutContents: string,
stderrContents: string,
desiredErrorMessage?: string,
desiredExitCode?: number
): string[] {
let command = "";
let command = '';
if (stdoutContents) command += `console.log("${stdoutContents}");`;
if (stderrContents) command += `console.error("${stderrContents}");`;
if (stdoutContents) command += 'console.log("' + stdoutContents + '");';
if (stderrContents) command += 'console.error("' + stderrContents + '");';
if (command.length === 0)
throw new Error("Must provide contents for either stdout or stderr");
if (command.length === 0) throw new Error("Must provide contents for either stdout or stderr");
if (desiredErrorMessage) command += 'throw new Error("' + desiredErrorMessage + '");';
if (desiredExitCode) command += 'process.exitCode = ' + desiredExitCode + ';';
if (desiredErrorMessage)
command += `throw new Error("${desiredErrorMessage}");`;
if (desiredExitCode) command += `process.exitCode = ${desiredExitCode};`;
return ["-e", command];
}

49
src/toolrunner-error-catcher.ts
View file

@ -1,7 +1,7 @@
import * as im from '@actions/exec/lib/interfaces';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as im from "@actions/exec/lib/interfaces";
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import {ErrorMatcher} from './error-matcher';
import { ErrorMatcher } from "./error-matcher";
/**
* Wrapper for toolrunner.Toolrunner which checks for specific return code and/or regex matches in console output.
@ -14,14 +14,16 @@ import {ErrorMatcher} from './error-matcher';
* @param options optional exec options. See ExecOptions
* @returns Promise<number> exit code
*/
export async function toolrunnerErrorCatcher(commandLine: string, args?: string[],
matchers?: ErrorMatcher[],
options?: im.ExecOptions): Promise<number> {
export async function toolrunnerErrorCatcher(
commandLine: string,
args?: string[],
matchers?: ErrorMatcher[],
options?: im.ExecOptions
): Promise<number> {
let stdout = "";
let stderr = "";
let stdout = '';
let stderr = '';
let listeners = {
const listeners = {
stdout: (data: Buffer) => {
stdout += data.toString();
if (options?.listeners?.stdout !== undefined) {
@ -30,7 +32,6 @@ export async function toolrunnerErrorCatcher(commandLine: string, args?: string[
// if no stdout listener was originally defined then we match default behavior of Toolrunner
process.stdout.write(data);
}
},
stderr: (data: Buffer) => {
stderr += data.toString();
@ -40,21 +41,17 @@ export async function toolrunnerErrorCatcher(commandLine: string, args?: string[
// if no stderr listener was originally defined then we match default behavior of Toolrunner
process.stderr.write(data);
}
}
},
};
// we capture the original return code or error so that if no match is found we can duplicate the behavior
let returnState: Error|number;
let returnState: Error | number;
try {
returnState = await new toolrunnner.ToolRunner(
commandLine,
args,
{
...options, // we want to override the original options, so include them first
listeners: listeners,
ignoreReturnCode: true, // so we can check for specific codes using the matchers
}
).exec();
returnState = await new toolrunnner.ToolRunner(commandLine, args, {
...options, // we want to override the original options, so include them first
listeners,
ignoreReturnCode: true, // so we can check for specific codes using the matchers
}).exec();
} catch (e) {
returnState = e;
}
@ -68,18 +65,20 @@ export async function toolrunnerErrorCatcher(commandLine: string, args?: string[
matcher.exitCode === returnState ||
matcher.outputRegex?.test(stderr) ||
matcher.outputRegex?.test(stdout)
) {
) {
throw new Error(matcher.message);
}
}
}
if (typeof returnState === 'number') {
if (typeof returnState === "number") {
// only if we were instructed to ignore the return code do we ever return it non-zero
if (options?.ignoreReturnCode) {
return returnState;
} else {
throw new Error(`The process \'${commandLine}\' failed with exit code ${returnState}`);
throw new Error(
`The process \'${commandLine}\' failed with exit code ${returnState}`
);
}
} else {
throw returnState;

326
src/tracer-config.test.ts
View file

@ -1,13 +1,17 @@
import test from 'ava';
import * as fs from 'fs';
import * as path from 'path';
import test from "ava";
import * as fs from "fs";
import * as path from "path";
import { setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from './languages';
import { setupTests } from './testing-utils';
import { concatTracerConfigs, getCombinedTracerConfig, getTracerConfigForLanguage } from './tracer-config';
import * as util from './util';
import { setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import { setupTests } from "./testing-utils";
import {
concatTracerConfigs,
getCombinedTracerConfig,
getTracerConfigForLanguage,
} from "./tracer-config";
import * as util from "./util";
setupTests(test);
@ -20,236 +24,260 @@ function getTestConfig(tmpDir: string): configUtils.Config {
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: '',
codeQLCmd: "",
};
}
// A very minimal setup
test('getTracerConfigForLanguage - minimal setup', async t => {
await util.withTmpDir(async tmpDir => {
test("getTracerConfigForLanguage - minimal setup", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const codeQL = setCodeQL({
getTracerEnv: async function() {
async getTracerEnv() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
ODASA_TRACER_CONFIGURATION: "abc",
foo: "bar",
};
},
});
const result = await getTracerConfigForLanguage(codeQL, config, Language.javascript);
t.deepEqual(result, { spec: 'abc', env: {'foo': 'bar'} });
const result = await getTracerConfigForLanguage(
codeQL,
config,
Language.javascript
);
t.deepEqual(result, { spec: "abc", env: { foo: "bar" } });
});
});
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
test('getTracerConfigForLanguage - existing / critical vars', async t => {
await util.withTmpDir(async tmpDir => {
test("getTracerConfigForLanguage - existing / critical vars", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
// Set up some variables in the environment
process.env['foo'] = 'abc';
process.env['SEMMLE_PRELOAD_libtrace'] = 'abc';
process.env['SEMMLE_RUNNER'] = 'abc';
process.env['SEMMLE_COPY_EXECUTABLES_ROOT'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['SEMMLE_JAVA_TOOL_OPTIONS'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['CODEQL_VAR'] = 'abc';
process.env["foo"] = "abc";
process.env["SEMMLE_PRELOAD_libtrace"] = "abc";
process.env["SEMMLE_RUNNER"] = "abc";
process.env["SEMMLE_COPY_EXECUTABLES_ROOT"] = "abc";
process.env["SEMMLE_DEPTRACE_SOCKET"] = "abc";
process.env["SEMMLE_JAVA_TOOL_OPTIONS"] = "abc";
process.env["SEMMLE_DEPTRACE_SOCKET"] = "abc";
process.env["CODEQL_VAR"] = "abc";
// Now CodeQL returns all these variables, and one more, with different values
const codeQL = setCodeQL({
getTracerEnv: async function() {
async getTracerEnv() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar',
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
ODASA_TRACER_CONFIGURATION: "abc",
foo: "bar",
baz: "qux",
SEMMLE_PRELOAD_libtrace: "SEMMLE_PRELOAD_libtrace",
SEMMLE_RUNNER: "SEMMLE_RUNNER",
SEMMLE_COPY_EXECUTABLES_ROOT: "SEMMLE_COPY_EXECUTABLES_ROOT",
SEMMLE_DEPTRACE_SOCKET: "SEMMLE_DEPTRACE_SOCKET",
SEMMLE_JAVA_TOOL_OPTIONS: "SEMMLE_JAVA_TOOL_OPTIONS",
CODEQL_VAR: "CODEQL_VAR",
};
},
});
const result = await getTracerConfigForLanguage(codeQL, config, Language.javascript);
const result = await getTracerConfigForLanguage(
codeQL,
config,
Language.javascript
);
t.deepEqual(result, {
spec: 'abc',
spec: "abc",
env: {
// Should contain all variables except 'foo', because that already existed in the
// environment with a different value, and is not deemed a "critical" variable.
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
}
baz: "qux",
SEMMLE_PRELOAD_libtrace: "SEMMLE_PRELOAD_libtrace",
SEMMLE_RUNNER: "SEMMLE_RUNNER",
SEMMLE_COPY_EXECUTABLES_ROOT: "SEMMLE_COPY_EXECUTABLES_ROOT",
SEMMLE_DEPTRACE_SOCKET: "SEMMLE_DEPTRACE_SOCKET",
SEMMLE_JAVA_TOOL_OPTIONS: "SEMMLE_JAVA_TOOL_OPTIONS",
CODEQL_VAR: "CODEQL_VAR",
},
});
});
});
test('concatTracerConfigs - minimal configs correctly combined', async t => {
await util.withTmpDir(async tmpDir => {
test("concatTracerConfigs - minimal configs correctly combined", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const spec1 = path.join(tmpDir, "spec1");
fs.writeFileSync(spec1, "foo.log\n2\nabc\ndef");
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
a: "a",
b: "b",
},
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const spec2 = path.join(tmpDir, "spec2");
fs.writeFileSync(spec2, "foo.log\n1\nghi");
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
c: "c",
},
};
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
const result = concatTracerConfigs(
{ javascript: tc1, python: tc2 },
config
);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
spec: path.join(tmpDir, "compound-spec"),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
a: "a",
b: "b",
c: "c",
},
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, 'utf8'),
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nabc\ndef\nghi');
fs.readFileSync(result.spec, "utf8"),
`${path.join(tmpDir, "compound-build-tracer.log")}\n3\nabc\ndef\nghi`
);
});
});
test('concatTracerConfigs - conflicting env vars', async t => {
await util.withTmpDir(async tmpDir => {
test("concatTracerConfigs - conflicting env vars", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
const spec = path.join(tmpDir, "spec");
fs.writeFileSync(spec, "foo.log\n0");
// Ok if env vars have the same name and the same value
t.deepEqual(
concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'b': 'b', 'c': 'c'}},
javascript: { spec, env: { a: "a", b: "b" } },
python: { spec, env: { b: "b", c: "c" } },
},
config).env,
config
).env,
{
'a': 'a',
'b': 'b',
'c': 'c',
});
a: "a",
b: "b",
c: "c",
}
);
// Throws if env vars have same name but different values
const e = t.throws(() =>
concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'b': 'c'}},
javascript: { spec, env: { a: "a", b: "b" } },
python: { spec, env: { b: "c" } },
},
config));
t.deepEqual(e.message, 'Incompatible values in environment parameter b: b and c');
config
)
);
t.deepEqual(
e.message,
"Incompatible values in environment parameter b: b and c"
);
});
});
test('concatTracerConfigs - cpp spec lines come last if present', async t => {
await util.withTmpDir(async tmpDir => {
test("concatTracerConfigs - cpp spec lines come last if present", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const spec1 = path.join(tmpDir, "spec1");
fs.writeFileSync(spec1, "foo.log\n2\nabc\ndef");
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
a: "a",
b: "b",
},
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const spec2 = path.join(tmpDir, "spec2");
fs.writeFileSync(spec2, "foo.log\n1\nghi");
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
c: "c",
},
};
const result = concatTracerConfigs({ 'cpp': tc1, 'python': tc2 }, config);
const result = concatTracerConfigs({ cpp: tc1, python: tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
spec: path.join(tmpDir, "compound-spec"),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
a: "a",
b: "b",
c: "c",
},
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, 'utf8'),
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nghi\nabc\ndef');
fs.readFileSync(result.spec, "utf8"),
`${path.join(tmpDir, "compound-build-tracer.log")}\n3\nghi\nabc\ndef`
);
});
});
test('concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT is updated to point to compound spec', async t => {
await util.withTmpDir(async tmpDir => {
test("concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT is updated to point to compound spec", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
const spec = path.join(tmpDir, "spec");
fs.writeFileSync(spec, "foo.log\n0");
const result = concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'SEMMLE_COPY_EXECUTABLES_ROOT': 'foo'}},
javascript: { spec, env: { a: "a", b: "b" } },
python: { spec, env: { SEMMLE_COPY_EXECUTABLES_ROOT: "foo" } },
},
config);
config
);
t.deepEqual(result.env, {
'a': 'a',
'b': 'b',
'SEMMLE_COPY_EXECUTABLES_ROOT': path.join(tmpDir, 'compound-temp')
a: "a",
b: "b",
SEMMLE_COPY_EXECUTABLES_ROOT: path.join(tmpDir, "compound-temp"),
});
});
});
test('concatTracerConfigs - compound environment file is created correctly', async t => {
await util.withTmpDir(async tmpDir => {
test("concatTracerConfigs - compound environment file is created correctly", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const spec1 = path.join(tmpDir, "spec1");
fs.writeFileSync(spec1, "foo.log\n2\nabc\ndef");
const tc1 = {
spec: spec1,
env: {
'a': 'a',
}
a: "a",
},
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const spec2 = path.join(tmpDir, "spec2");
fs.writeFileSync(spec2, "foo.log\n1\nghi");
const tc2 = {
spec: spec2,
env: {
'foo': 'bar_baz',
}
foo: "bar_baz",
},
};
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
const envPath = result.spec + '.environment';
const result = concatTracerConfigs(
{ javascript: tc1, python: tc2 },
config
);
const envPath = `${result.spec}.environment`;
t.true(fs.existsSync(envPath));
const buffer: Buffer = fs.readFileSync(envPath);
@ -257,23 +285,23 @@ test('concatTracerConfigs - compound environment file is created correctly', asy
t.deepEqual(buffer.length, 28);
t.deepEqual(buffer.readInt32LE(0), 2); // number of env vars
t.deepEqual(buffer.readInt32LE(4), 4); // length of env var definition
t.deepEqual(buffer.toString('utf8', 8, 12), 'a=a\0'); // [key]=[value]\0
t.deepEqual(buffer.readInt32LE(12), 12); // length of env var definition
t.deepEqual(buffer.toString('utf8', 16, 28), 'foo=bar_baz\0'); // [key]=[value]\0
t.deepEqual(buffer.toString("utf8", 8, 12), "a=a\0"); // [key]=[value]\0
t.deepEqual(buffer.readInt32LE(12), 12); // length of env var definition
t.deepEqual(buffer.toString("utf8", 16, 28), "foo=bar_baz\0"); // [key]=[value]\0
});
});
test('getCombinedTracerConfig - return undefined when no languages are traced languages', async t => {
await util.withTmpDir(async tmpDir => {
test("getCombinedTracerConfig - return undefined when no languages are traced languages", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
// No traced languages
config.languages = [Language.javascript, Language.python];
const codeQL = setCodeQL({
getTracerEnv: async function() {
async getTracerEnv() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
ODASA_TRACER_CONFIGURATION: "abc",
foo: "bar",
};
},
});
@ -282,18 +310,18 @@ test('getCombinedTracerConfig - return undefined when no languages are traced la
});
});
test('getCombinedTracerConfig - valid spec file', async t => {
await util.withTmpDir(async tmpDir => {
test("getCombinedTracerConfig - valid spec file", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n2\nabc\ndef');
const spec = path.join(tmpDir, "spec");
fs.writeFileSync(spec, "foo.log\n2\nabc\ndef");
const codeQL = setCodeQL({
getTracerEnv: async function() {
async getTracerEnv() {
return {
'ODASA_TRACER_CONFIGURATION': spec,
'foo': 'bar',
ODASA_TRACER_CONFIGURATION: spec,
foo: "bar",
};
},
});
@ -301,17 +329,27 @@ test('getCombinedTracerConfig - valid spec file', async t => {
const result = await getCombinedTracerConfig(config, codeQL);
const expectedEnv = {
'foo': 'bar',
'ODASA_TRACER_CONFIGURATION': result!.spec,
foo: "bar",
ODASA_TRACER_CONFIGURATION: result!.spec,
};
if (process.platform === 'darwin') {
expectedEnv['DYLD_INSERT_LIBRARIES'] = path.join(path.dirname(codeQL.getPath()), 'tools', 'osx64', 'libtrace.dylib');
} else if (process.platform !== 'win32') {
expectedEnv['LD_PRELOAD'] = path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so');
if (process.platform === "darwin") {
expectedEnv["DYLD_INSERT_LIBRARIES"] = path.join(
path.dirname(codeQL.getPath()),
"tools",
"osx64",
"libtrace.dylib"
);
} else if (process.platform !== "win32") {
expectedEnv["LD_PRELOAD"] = path.join(
path.dirname(codeQL.getPath()),
"tools",
"linux64",
"${LIB}trace.so"
);
}
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
spec: path.join(tmpDir, "compound-spec"),
env: expectedEnv,
});
});

136
src/tracer-config.ts
View file

@ -1,49 +1,59 @@
import * as fs from 'fs';
import * as path from 'path';
import * as fs from "fs";
import * as path from "path";
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isTracedLanguage, Language } from './languages';
import * as util from './util';
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Language, isTracedLanguage } from "./languages";
import * as util from "./util";
export type TracerConfig = {
spec: string;
env: { [key: string]: string };
};
const CRITICAL_TRACER_VARS = new Set(
['SEMMLE_PRELOAD_libtrace',
, 'SEMMLE_RUNNER',
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
, 'SEMMLE_DEPTRACE_SOCKET',
, 'SEMMLE_JAVA_TOOL_OPTIONS'
]);
const CRITICAL_TRACER_VARS = new Set([
"SEMMLE_PRELOAD_libtrace",
,
"SEMMLE_RUNNER",
,
"SEMMLE_COPY_EXECUTABLES_ROOT",
,
"SEMMLE_DEPTRACE_SOCKET",
,
"SEMMLE_JAVA_TOOL_OPTIONS",
]);
export async function getTracerConfigForLanguage(
codeql: CodeQL,
config: configUtils.Config,
language: Language): Promise<TracerConfig> {
language: Language
): Promise<TracerConfig> {
const env = await codeql.getTracerEnv(
util.getCodeQLDatabasePath(config.tempDir, language)
);
const env = await codeql.getTracerEnv(util.getCodeQLDatabasePath(config.tempDir, language));
const spec = env['ODASA_TRACER_CONFIGURATION'];
const spec = env["ODASA_TRACER_CONFIGURATION"];
const info: TracerConfig = { spec, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
for (const entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
if (key === "ODASA_TRACER_CONFIGURATION") {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
if (typeof value === "undefined") {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
if (
typeof process.env[key] === "undefined" ||
CRITICAL_TRACER_VARS.has(key) ||
key.startsWith("CODEQL_")
) {
info.env[key] = value;
}
}
@ -52,26 +62,27 @@ export async function getTracerConfigForLanguage(
export function concatTracerConfigs(
tracerConfigs: { [lang: string]: TracerConfig },
config: configUtils.Config): TracerConfig {
config: configUtils.Config
): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env: { [key: string]: string; } = {};
const env: { [key: string]: string } = {};
let copyExecutables = false;
let envSize = 0;
for (const v of Object.values(tracerConfigs)) {
for (let e of Object.entries(v.env)) {
for (const e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
if (name === "SEMMLE_COPY_EXECUTABLES_ROOT") {
copyExecutables = true;
} else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
throw Error(
`Incompatible values in environment parameter ${name}: ${env[name]} and ${value}`
);
}
} else {
env[name] = value;
@ -81,49 +92,58 @@ export function concatTracerConfigs(
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
const languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf("cpp");
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
const lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines: string[] = [];
const totalLines: string[] = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
for (const lang of languages) {
const lines = fs
.readFileSync(tracerConfigs[lang].spec, "utf8")
.split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
const newLogFilePath = path.resolve(
config.tempDir,
"compound-build-tracer.log"
);
const spec = path.resolve(config.tempDir, "compound-spec");
const compoundTempFolder = path.resolve(config.tempDir, "compound-temp");
const newSpecContent = [
newLogFilePath,
totalCount.toString(10),
...totalLines,
];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
env["SEMMLE_COPY_EXECUTABLES_ROOT"] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
fs.writeFileSync(spec, newSpecContent.join("\n"));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
for (const e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const lineBuffer = new Buffer(`${key}=${value}\0`, "utf8");
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
const envPath = `${spec}.environment`;
fs.writeFileSync(envPath, buffer);
return { env, spec };
@ -131,8 +151,8 @@ export function concatTracerConfigs(
export async function getCombinedTracerConfig(
config: configUtils.Config,
codeql: CodeQL): Promise<TracerConfig | undefined> {
codeql: CodeQL
): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter(isTracedLanguage);
if (tracedLanguages.length === 0) {
@ -142,17 +162,31 @@ export async function getCombinedTracerConfig(
// Get all the tracer configs and combine them together
const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
for (const language of tracedLanguages) {
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(codeql, config, language);
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(
codeql,
config,
language
);
}
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
// Add a couple more variables
mainTracerConfig.env['ODASA_TRACER_CONFIGURATION'] = mainTracerConfig.spec;
mainTracerConfig.env["ODASA_TRACER_CONFIGURATION"] = mainTracerConfig.spec;
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
mainTracerConfig.env['DYLD_INSERT_LIBRARIES'] = path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib');
} else if (process.platform !== 'win32') {
mainTracerConfig.env['LD_PRELOAD'] = path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so');
if (process.platform === "darwin") {
mainTracerConfig.env["DYLD_INSERT_LIBRARIES"] = path.join(
codeQLDir,
"tools",
"osx64",
"libtrace.dylib"
);
} else if (process.platform !== "win32") {
mainTracerConfig.env["LD_PRELOAD"] = path.join(
codeQLDir,
"tools",
"linux64",
"${LIB}trace.so"
);
}
return mainTracerConfig;

24
src/upload-lib.test.ts
View file

@ -1,17 +1,21 @@
import test from 'ava';
import test from "ava";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as uploadLib from './upload-lib';
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as uploadLib from "./upload-lib";
setupTests(test);
test('validateSarifFileSchema - valid', t => {
const inputFile = __dirname + '/../src/testdata/valid-sarif.sarif';
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)));
test("validateSarifFileSchema - valid", (t) => {
const inputFile = `${__dirname}/../src/testdata/valid-sarif.sarif`;
t.notThrows(() =>
uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true))
);
});
test('validateSarifFileSchema - invalid', t => {
const inputFile = __dirname + '/../src/testdata/invalid-sarif.sarif';
t.throws(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)));
test("validateSarifFileSchema - invalid", (t) => {
const inputFile = `${__dirname}/../src/testdata/invalid-sarif.sarif`;
t.throws(() =>
uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true))
);
});

180
src/upload-lib.ts
View file

@ -1,32 +1,32 @@
import * as core from '@actions/core';
import fileUrl from 'file-url';
import * as fs from 'fs';
import * as jsonschema from 'jsonschema';
import * as path from 'path';
import zlib from 'zlib';
import * as core from "@actions/core";
import fileUrl from "file-url";
import * as fs from "fs";
import * as jsonschema from "jsonschema";
import * as path from "path";
import zlib from "zlib";
import * as api from './api-client';
import * as fingerprints from './fingerprints';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as util from './util';
import * as api from "./api-client";
import * as fingerprints from "./fingerprints";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import * as sharedEnv from "./shared-environment";
import * as util from "./util";
// Takes a list of paths to sarif files and combines them together,
// returning the contents of the combined sarif file.
export function combineSarifFiles(sarifFiles: string[]): string {
let combinedSarif = {
const combinedSarif = {
version: null,
runs: [] as any[]
runs: [] as any[],
};
for (let sarifFile of sarifFiles) {
let sarifObject = JSON.parse(fs.readFileSync(sarifFile, 'utf8'));
for (const sarifFile of sarifFiles) {
const sarifObject = JSON.parse(fs.readFileSync(sarifFile, "utf8"));
// Check SARIF version
if (combinedSarif.version === null) {
combinedSarif.version = sarifObject.version;
} else if (combinedSarif.version !== sarifObject.version) {
throw "Different SARIF versions encountered: " + combinedSarif.version + " and " + sarifObject.version;
throw `Different SARIF versions encountered: ${combinedSarif.version} and ${sarifObject.version}`;
}
combinedSarif.runs.push(...sarifObject.runs);
@ -43,12 +43,12 @@ async function uploadPayload(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger) {
logger.info('Uploading results');
logger: Logger
) {
logger.info("Uploading results");
// If in test mode we don't want to upload the results
const testMode = process.env['TEST_MODE'] === 'true' || false;
const testMode = process.env["TEST_MODE"] === "true" || false;
if (testMode) {
return;
}
@ -62,16 +62,17 @@ async function uploadPayload(
const client = api.getApiClient(githubAuth, githubUrl);
for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
const reqURL = mode === 'actions'
? 'PUT /repos/:owner/:repo/code-scanning/analysis'
: 'POST /repos/:owner/:repo/code-scanning/sarifs';
const response = await client.request(reqURL, ({
const reqURL =
mode === "actions"
? "PUT /repos/:owner/:repo/code-scanning/analysis"
: "POST /repos/:owner/:repo/code-scanning/sarifs";
const response = await client.request(reqURL, {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
data: payload,
}));
});
logger.debug('response status: ' + response.status);
logger.debug(`response status: ${response.status}`);
const statusCode = response.status;
if (statusCode === 202) {
@ -83,30 +84,41 @@ async function uploadPayload(
// On any other status code that's not 5xx mark the upload as failed
if (!statusCode || statusCode < 500 || statusCode >= 600) {
throw new Error('Upload failed (' + requestID + '): (' + statusCode + ') ' + JSON.stringify(response.data));
throw new Error(
`Upload failed (${requestID}): (${statusCode}) ${JSON.stringify(
response.data
)}`
);
}
// On a 5xx status code we may retry the request
if (attempt < backoffPeriods.length) {
// Log the failure as a warning but don't mark the action as failed yet
logger.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
' seconds: (' + statusCode + ') ' + JSON.stringify(response.data));
logger.warning(
`Upload attempt (${attempt + 1} of ${
backoffPeriods.length + 1
}) failed (${requestID}). Retrying in ${
backoffPeriods[attempt]
} seconds: (${statusCode}) ${JSON.stringify(response.data)}`
);
// Sleep for the backoff period
await new Promise(r => setTimeout(r, backoffPeriods[attempt] * 1000));
await new Promise((r) => setTimeout(r, backoffPeriods[attempt] * 1000));
continue;
} else {
// If the upload fails with 5xx then we assume it is a temporary problem
// and not an error that the user has caused or can fix.
// We avoid marking the job as failed to avoid breaking CI workflows.
throw new Error('Upload failed (' + requestID + '): (' + statusCode + ') ' + JSON.stringify(response.data));
throw new Error(
`Upload failed (${requestID}): (${statusCode}) ${JSON.stringify(
response.data
)}`
);
}
}
// This case shouldn't ever happen as the final iteration of the loop
// will always throw an error instead of exiting to here.
throw new Error('Upload failed');
throw new Error("Upload failed");
}
export interface UploadStatusReport {
@ -134,19 +146,19 @@ export async function upload(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
logger: Logger
): Promise<UploadStatusReport> {
const sarifFiles: string[] = [];
if (!fs.existsSync(sarifPath)) {
throw new Error(`Path does not exist: ${sarifPath}`);
}
if (fs.lstatSync(sarifPath).isDirectory()) {
fs.readdirSync(sarifPath)
.filter(f => f.endsWith(".sarif"))
.map(f => path.resolve(sarifPath, f))
.forEach(f => sarifFiles.push(f));
.filter((f) => f.endsWith(".sarif"))
.map((f) => path.resolve(sarifPath, f))
.forEach((f) => sarifFiles.push(f));
if (sarifFiles.length === 0) {
throw new Error("No SARIF files found to upload in \"" + sarifPath + "\".");
throw new Error(`No SARIF files found to upload in "${sarifPath}".`);
}
} else {
sarifFiles.push(sarifPath);
@ -165,7 +177,8 @@ export async function upload(
githubAuth,
githubUrl,
mode,
logger);
logger
);
}
// Counts the number of results in the given SARIF file
@ -180,22 +193,26 @@ export function countResultsInSarif(sarif: string): number {
// Validates that the given file path refers to a valid SARIF file.
// Throws an error if the file is invalid.
export function validateSarifFileSchema(sarifFilePath: string, logger: Logger) {
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, 'utf8'));
const schema = require('../src/sarif_v2.1.0_schema.json');
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, "utf8"));
const schema = require("../src/sarif_v2.1.0_schema.json");
const result = new jsonschema.Validator().validate(sarif, schema);
if (!result.valid) {
// Output the more verbose error messages in groups as these may be very large.
for (const error of result.errors) {
logger.startGroup("Error details: " + error.stack);
logger.startGroup(`Error details: ${error.stack}`);
logger.info(JSON.stringify(error, null, 2));
logger.endGroup();
}
// Set the main error message to the stacks of all the errors.
// This should be of a manageable size and may even give enough to fix the error.
const sarifErrors = result.errors.map(e => "- " + e.stack);
throw new Error("Unable to upload \"" + sarifFilePath + "\" as it is not valid SARIF:\n" + sarifErrors.join("\n"));
const sarifErrors = result.errors.map((e) => `- ${e.stack}`);
throw new Error(
`Unable to upload "${sarifFilePath}" as it is not valid SARIF:\n${sarifErrors.join(
"\n"
)}`
);
}
}
@ -214,15 +231,17 @@ async function uploadFiles(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
logger: Logger
): Promise<UploadStatusReport> {
logger.info(`Uploading sarif files: ${JSON.stringify(sarifFiles)}`);
logger.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
if (mode === 'actions') {
if (mode === "actions") {
// This check only works on actions as env vars don't persist between calls to the runner
const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
if (process.env[sentinelEnvVar]) {
throw new Error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
throw new Error(
"Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job"
);
}
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
}
@ -233,47 +252,58 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
sarifPayload = fingerprints.addFingerprints(
sarifPayload,
checkoutPath,
logger
);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutURI = fileUrl(checkoutPath);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString("base64");
const checkoutURI = fileUrl(checkoutPath);
const toolNames = util.getToolNames(sarifPayload);
let payload: string;
if (mode === 'actions') {
if (mode === "actions") {
payload = JSON.stringify({
"commit_oid": commitOid,
"ref": ref,
"analysis_key": analysisKey,
"analysis_name": analysisName,
"sarif": zipped_sarif,
"workflow_run_id": workflowRunID,
"checkout_uri": checkoutURI,
"environment": environment,
"started_at": process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
"tool_names": toolNames,
commit_oid: commitOid,
ref,
analysis_key: analysisKey,
analysis_name: analysisName,
sarif: zipped_sarif,
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
});
} else {
payload = JSON.stringify({
"commit_sha": commitOid,
"ref": ref,
"sarif": zipped_sarif,
"checkout_uri": checkoutURI,
"tool_name": toolNames[0],
commit_sha: commitOid,
ref,
sarif: zipped_sarif,
checkout_uri: checkoutURI,
tool_name: toolNames[0],
});
}
// Log some useful debug info about the info
const rawUploadSizeBytes = sarifPayload.length;
logger.debug("Raw upload size: " + rawUploadSizeBytes + " bytes");
logger.debug(`Raw upload size: ${rawUploadSizeBytes} bytes`);
const zippedUploadSizeBytes = zipped_sarif.length;
logger.debug("Base64 zipped upload size: " + zippedUploadSizeBytes + " bytes");
logger.debug(`Base64 zipped upload size: ${zippedUploadSizeBytes} bytes`);
const numResultInSarif = countResultsInSarif(sarifPayload);
logger.debug("Number of results in upload: " + numResultInSarif);
logger.debug(`Number of results in upload: ${numResultInSarif}`);
// Make the upload
await uploadPayload(payload, repositoryNwo, githubAuth, githubUrl, mode, logger);
await uploadPayload(
payload,
repositoryNwo,
githubAuth,
githubUrl,
mode,
logger
);
return {
raw_upload_size_bytes: rawUploadSizeBytes,

73
src/upload-sarif-action.ts
View file

@ -1,58 +1,75 @@
import * as core from '@actions/core';
import * as core from "@actions/core";
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import * as util from './util';
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
import * as util from "./util";
interface UploadSarifStatusReport extends util.StatusReportBase, upload_lib.UploadStatusReport {}
interface UploadSarifStatusReport
extends util.StatusReportBase,
upload_lib.UploadStatusReport {}
async function sendSuccessStatusReport(startedAt: Date, uploadStats: upload_lib.UploadStatusReport) {
const statusReportBase = await util.createStatusReportBase('upload-sarif', 'success', startedAt);
async function sendSuccessStatusReport(
startedAt: Date,
uploadStats: upload_lib.UploadStatusReport
) {
const statusReportBase = await util.createStatusReportBase(
"upload-sarif",
"success",
startedAt
);
const statusReport: UploadSarifStatusReport = {
...statusReportBase,
... uploadStats,
...uploadStats,
};
await util.sendStatusReport(statusReport);
}
async function run() {
const startedAt = new Date();
if (!await util.sendStatusReport(await util.createStatusReportBase('upload-sarif', 'starting', startedAt), true)) {
if (
!(await util.sendStatusReport(
await util.createStatusReportBase("upload-sarif", "starting", startedAt),
true
))
) {
return;
}
try {
const uploadStats = await upload_lib.upload(
core.getInput('sarif_file'),
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
core.getInput("sarif_file"),
parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY")),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getRequiredEnvParam("GITHUB_WORKFLOW"),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
'actions',
getActionsLogger());
core.getInput("checkout_path"),
core.getInput("matrix"),
core.getInput("token"),
util.getRequiredEnvParam("GITHUB_SERVER_URL"),
"actions",
getActionsLogger()
);
await sendSuccessStatusReport(startedAt, uploadStats);
} catch (error) {
core.setFailed(error.message);
console.log(error);
await util.sendStatusReport(await util.createStatusReportBase(
'upload-sarif',
'failure',
startedAt,
error.message,
error.stack));
await util.sendStatusReport(
await util.createStatusReportBase(
"upload-sarif",
"failure",
startedAt,
error.message,
error.stack
)
);
return;
}
}
run().catch(e => {
core.setFailed("codeql/upload-sarif action failed: " + e);
run().catch((e) => {
core.setFailed(`codeql/upload-sarif action failed: ${e}`);
console.log(e);
});

75
src/util.test.ts
View file

@ -1,21 +1,23 @@
import test from 'ava';
import * as fs from 'fs';
import test from "ava";
import * as fs from "fs";
import * as os from "os";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test('getToolNames', t => {
const input = fs.readFileSync(__dirname + '/../src/testdata/tool-names.sarif', 'utf8');
test("getToolNames", (t) => {
const input = fs.readFileSync(
`${__dirname}/../src/testdata/tool-names.sarif`,
"utf8"
);
const toolNames = util.getToolNames(input);
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
test('getMemoryFlag() should return the correct --ram flag', t => {
test("getMemoryFlag() should return the correct --ram flag", (t) => {
const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
const tests = {
@ -29,14 +31,13 @@ test('getMemoryFlag() should return the correct --ram flag', t => {
}
});
test('getMemoryFlag() throws if the ram input is < 0 or NaN', t => {
test("getMemoryFlag() throws if the ram input is < 0 or NaN", (t) => {
for (const input of ["-1", "hello!"]) {
t.throws(() => util.getMemoryFlag(input));
}
});
test('getAddSnippetsFlag() should return the correct flag', t => {
test("getAddSnippetsFlag() should return the correct flag", (t) => {
t.deepEqual(util.getAddSnippetsFlag(true), "--sarif-add-snippets");
t.deepEqual(util.getAddSnippetsFlag("true"), "--sarif-add-snippets");
@ -44,18 +45,16 @@ test('getAddSnippetsFlag() should return the correct flag', t => {
t.deepEqual(util.getAddSnippetsFlag(undefined), "--no-sarif-add-snippets");
t.deepEqual(util.getAddSnippetsFlag("false"), "--no-sarif-add-snippets");
t.deepEqual(util.getAddSnippetsFlag("foo bar"), "--no-sarif-add-snippets");
});
test('getThreadsFlag() should return the correct --threads flag', t => {
test("getThreadsFlag() should return the correct --threads flag", (t) => {
const numCpus = os.cpus().length;
const tests = {
"0": "--threads=0",
"1": "--threads=1",
[`${numCpus + 1}`]: `--threads=${numCpus}`,
[`${-numCpus - 1}`]: `--threads=${-numCpus}`
[`${-numCpus - 1}`]: `--threads=${-numCpus}`,
};
for (const [input, expectedFlag] of Object.entries(tests)) {
@ -64,68 +63,68 @@ test('getThreadsFlag() should return the correct --threads flag', t => {
}
});
test('getThreadsFlag() throws if the threads input is not an integer', t => {
test("getThreadsFlag() throws if the threads input is not an integer", (t) => {
t.throws(() => util.getThreadsFlag("hello!", getRunnerLogger(true)));
});
test('getRef() throws on the empty string', t => {
test("getRef() throws on the empty string", (t) => {
process.env["GITHUB_REF"] = "";
t.throws(util.getRef);
});
test('isLocalRun() runs correctly', t => {
test("isLocalRun() runs correctly", (t) => {
const origLocalRun = process.env.CODEQL_LOCAL_RUN;
process.env.CODEQL_LOCAL_RUN = '';
process.env.CODEQL_LOCAL_RUN = "";
t.assert(!util.isLocalRun());
process.env.CODEQL_LOCAL_RUN = 'false';
process.env.CODEQL_LOCAL_RUN = "false";
t.assert(!util.isLocalRun());
process.env.CODEQL_LOCAL_RUN = '0';
process.env.CODEQL_LOCAL_RUN = "0";
t.assert(!util.isLocalRun());
process.env.CODEQL_LOCAL_RUN = 'true';
process.env.CODEQL_LOCAL_RUN = "true";
t.assert(util.isLocalRun());
process.env.CODEQL_LOCAL_RUN = 'hucairz';
process.env.CODEQL_LOCAL_RUN = "hucairz";
t.assert(util.isLocalRun());
process.env.CODEQL_LOCAL_RUN = origLocalRun;
});
test('prepareEnvironment() when a local run', t => {
test("prepareEnvironment() when a local run", (t) => {
const origLocalRun = process.env.CODEQL_LOCAL_RUN;
process.env.CODEQL_LOCAL_RUN = 'false';
process.env.GITHUB_JOB = 'YYY';
process.env.CODEQL_LOCAL_RUN = "false";
process.env.GITHUB_JOB = "YYY";
util.prepareLocalRunEnvironment();
// unchanged
t.deepEqual(process.env.GITHUB_JOB, 'YYY');
t.deepEqual(process.env.GITHUB_JOB, "YYY");
process.env.CODEQL_LOCAL_RUN = 'true';
process.env.CODEQL_LOCAL_RUN = "true";
util.prepareLocalRunEnvironment();
// unchanged
t.deepEqual(process.env.GITHUB_JOB, 'YYY');
t.deepEqual(process.env.GITHUB_JOB, "YYY");
process.env.GITHUB_JOB = '';
process.env.GITHUB_JOB = "";
util.prepareLocalRunEnvironment();
// updated
t.deepEqual(process.env.GITHUB_JOB, 'UNKNOWN-JOB');
t.deepEqual(process.env.GITHUB_JOB, "UNKNOWN-JOB");
process.env.CODEQL_LOCAL_RUN = origLocalRun;
});
test('getExtraOptionsEnvParam() succeeds on valid JSON with invalid options (for now)', t => {
test("getExtraOptionsEnvParam() succeeds on valid JSON with invalid options (for now)", (t) => {
const origExtraOptions = process.env.CODEQL_ACTION_EXTRA_OPTIONS;
const options = {foo: 42};
const options = { foo: 42 };
process.env.CODEQL_ACTION_EXTRA_OPTIONS = JSON.stringify(options);
@ -134,20 +133,18 @@ test('getExtraOptionsEnvParam() succeeds on valid JSON with invalid options (for
process.env.CODEQL_ACTION_EXTRA_OPTIONS = origExtraOptions;
});
test('getExtraOptionsEnvParam() succeeds on valid options', t => {
test("getExtraOptionsEnvParam() succeeds on valid options", (t) => {
const origExtraOptions = process.env.CODEQL_ACTION_EXTRA_OPTIONS;
const options = { database: { init: ["--debug"] } };
process.env.CODEQL_ACTION_EXTRA_OPTIONS =
JSON.stringify(options);
process.env.CODEQL_ACTION_EXTRA_OPTIONS = JSON.stringify(options);
t.deepEqual(util.getExtraOptionsEnvParam(), options);
process.env.CODEQL_ACTION_EXTRA_OPTIONS = origExtraOptions;
});
test('getExtraOptionsEnvParam() fails on invalid JSON', t => {
test("getExtraOptionsEnvParam() fails on invalid JSON", (t) => {
const origExtraOptions = process.env.CODEQL_ACTION_EXTRA_OPTIONS;
process.env.CODEQL_ACTION_EXTRA_OPTIONS = "{{invalid-json}}";

222
src/util.ts
View file

@ -1,34 +1,33 @@
import * as core from '@actions/core';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as core from "@actions/core";
import * as toolrunnner from "@actions/exec/lib/toolrunner";
import * as fs from "fs";
import * as os from 'os';
import * as path from 'path';
import * as os from "os";
import * as path from "path";
import * as api from './api-client';
import { Language } from './languages';
import { Logger } from './logging';
import * as sharedEnv from './shared-environment';
import * as api from "./api-client";
import { Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
/**
* Are we running on actions, or not.
*/
export type Mode = 'actions' | 'runner';
export type Mode = "actions" | "runner";
/**
* The URL for github.com.
*/
export const GITHUB_DOTCOM_URL = "https://github.com";
/**
* Get an environment parameter, but throw an error if it is not set.
*/
export function getRequiredEnvParam(paramName: string): string {
const value = process.env[paramName];
if (value === undefined || value.length === 0) {
throw new Error(paramName + ' environment variable must be set');
throw new Error(`${paramName} environment variable must be set`);
}
core.debug(paramName + '=' + value);
core.debug(`${paramName}=${value}`);
return value;
}
@ -36,7 +35,7 @@ export function getRequiredEnvParam(paramName: string): string {
* Get the extra options for the codeql commands.
*/
export function getExtraOptionsEnvParam(): object {
const varName = 'CODEQL_ACTION_EXTRA_OPTIONS';
const varName = "CODEQL_ACTION_EXTRA_OPTIONS";
const raw = process.env[varName];
if (raw === undefined || raw.length === 0) {
return {};
@ -45,17 +44,17 @@ export function getExtraOptionsEnvParam(): object {
return JSON.parse(raw);
} catch (e) {
throw new Error(
varName +
' environment variable is set, but does not contain valid JSON: ' +
e.message
`${varName} environment variable is set, but does not contain valid JSON: ${e.message}`
);
}
}
export function isLocalRun(): boolean {
return !!process.env.CODEQL_LOCAL_RUN
&& process.env.CODEQL_LOCAL_RUN !== 'false'
&& process.env.CODEQL_LOCAL_RUN !== '0';
return (
!!process.env.CODEQL_LOCAL_RUN &&
process.env.CODEQL_LOCAL_RUN !== "false" &&
process.env.CODEQL_LOCAL_RUN !== "0"
);
}
/**
@ -66,9 +65,9 @@ export function prepareLocalRunEnvironment() {
return;
}
core.debug('Action is running locally.');
core.debug("Action is running locally.");
if (!process.env.GITHUB_JOB) {
core.exportVariable('GITHUB_JOB', 'UNKNOWN-JOB');
core.exportVariable("GITHUB_JOB", "UNKNOWN-JOB");
}
}
@ -84,18 +83,24 @@ export async function getCommitOid(): Promise<string> {
// Even if this does go wrong, it's not a huge problem for the alerts to
// reported on the merge commit.
try {
let commitOid = '';
await new toolrunnner.ToolRunner('git', ['rev-parse', 'HEAD'], {
let commitOid = "";
await new toolrunnner.ToolRunner("git", ["rev-parse", "HEAD"], {
silent: true,
listeners: {
stdout: (data) => { commitOid += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
stdout: (data) => {
commitOid += data.toString();
},
stderr: (data) => {
process.stderr.write(data);
},
},
}).exec();
return commitOid.trim();
} catch (e) {
core.info("Failed to call git to get current commit. Continuing with data from environment: " + e);
return getRequiredEnvParam('GITHUB_SHA');
core.info(
`Failed to call git to get current commit. Continuing with data from environment: ${e}`
);
return getRequiredEnvParam("GITHUB_SHA");
}
}
@ -103,20 +108,23 @@ export async function getCommitOid(): Promise<string> {
* Get the path of the currently executing workflow.
*/
async function getWorkflowPath(): Promise<string> {
const repo_nwo = getRequiredEnvParam('GITHUB_REPOSITORY').split("/");
const repo_nwo = getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
const owner = repo_nwo[0];
const repo = repo_nwo[1];
const run_id = Number(getRequiredEnvParam('GITHUB_RUN_ID'));
const run_id = Number(getRequiredEnvParam("GITHUB_RUN_ID"));
const apiClient = api.getActionsApiClient();
const runsResponse = await apiClient.request('GET /repos/:owner/:repo/actions/runs/:run_id', {
owner,
repo,
run_id
});
const runsResponse = await apiClient.request(
"GET /repos/:owner/:repo/actions/runs/:run_id",
{
owner,
repo,
run_id,
}
);
const workflowUrl = runsResponse.data.workflow_url;
const workflowResponse = await apiClient.request('GET ' + workflowUrl);
const workflowResponse = await apiClient.request(`GET ${workflowUrl}`);
return workflowResponse.data.path;
}
@ -125,9 +133,9 @@ async function getWorkflowPath(): Promise<string> {
* Get the workflow run ID.
*/
export function getWorkflowRunID(): number {
const workflowRunID = parseInt(getRequiredEnvParam('GITHUB_RUN_ID'), 10);
const workflowRunID = parseInt(getRequiredEnvParam("GITHUB_RUN_ID"), 10);
if (Number.isNaN(workflowRunID)) {
throw new Error('GITHUB_RUN_ID must define a non NaN workflow run ID');
throw new Error("GITHUB_RUN_ID must define a non NaN workflow run ID");
}
return workflowRunID;
}
@ -140,7 +148,7 @@ export function getWorkflowRunID(): number {
* the github API, but after that the result will be cached.
*/
export async function getAnalysisKey(): Promise<string> {
const analysisKeyEnvVar = 'CODEQL_ACTION_ANALYSIS_KEY';
const analysisKeyEnvVar = "CODEQL_ACTION_ANALYSIS_KEY";
let analysisKey = process.env[analysisKeyEnvVar];
if (analysisKey !== undefined) {
@ -148,9 +156,9 @@ export async function getAnalysisKey(): Promise<string> {
}
const workflowPath = await getWorkflowPath();
const jobName = getRequiredEnvParam('GITHUB_JOB');
const jobName = getRequiredEnvParam("GITHUB_JOB");
analysisKey = workflowPath + ':' + jobName;
analysisKey = `${workflowPath}:${jobName}`;
core.exportVariable(analysisKeyEnvVar, analysisKey);
return analysisKey;
}
@ -161,7 +169,7 @@ export async function getAnalysisKey(): Promise<string> {
export function getRef(): string {
// Will be in the form "refs/heads/master" on a push event
// or in the form "refs/pull/N/merge" on a pull_request event
const ref = getRequiredEnvParam('GITHUB_REF');
const ref = getRequiredEnvParam("GITHUB_REF");
// For pull request refs we want to convert from the 'merge' ref
// to the 'head' ref, as that is what we want to analyse.
@ -169,46 +177,46 @@ export function getRef(): string {
// the checkout, but we have no way of verifying that here.
const pull_ref_regex = /refs\/pull\/(\d+)\/merge/;
if (pull_ref_regex.test(ref)) {
return ref.replace(pull_ref_regex, 'refs/pull/$1/head');
return ref.replace(pull_ref_regex, "refs/pull/$1/head");
} else {
return ref;
}
}
type ActionName = 'init' | 'autobuild' | 'finish' | 'upload-sarif';
type ActionStatus = 'starting' | 'aborted' | 'success' | 'failure';
type ActionName = "init" | "autobuild" | "finish" | "upload-sarif";
type ActionStatus = "starting" | "aborted" | "success" | "failure";
export interface StatusReportBase {
// ID of the workflow run containing the action run
"workflow_run_id": number;
workflow_run_id: number;
// Workflow name. Converted to analysis_name further down the pipeline.
"workflow_name": string;
workflow_name: string;
// Job name from the workflow
"job_name": string;
job_name: string;
// Analysis key, normally composed from the workflow path and job name
"analysis_key": string;
analysis_key: string;
// Value of the matrix for this instantiation of the job
"matrix_vars"?: string;
matrix_vars?: string;
// Commit oid that the workflow was triggered on
"commit_oid": string;
commit_oid: string;
// Ref that the workflow was triggered on
"ref": string;
ref: string;
// Name of the action being executed
"action_name": ActionName;
action_name: ActionName;
// Version if the action being executed, as a commit oid
"action_oid": string;
action_oid: string;
// Time the first action started. Normally the init action
"started_at": string;
started_at: string;
// Time this action started
"action_started_at": string;
action_started_at: string;
// Time this action completed, or undefined if not yet completed
"completed_at"?: string;
completed_at?: string;
// State this action is currently in
"status": ActionStatus;
status: ActionStatus;
// Cause of the failure (or undefined if status is not failure)
"cause"?: string;
cause?: string;
// Stack trace of the failure (or undefined if status is not failure)
"exception"?: string;
exception?: string;
}
/**
@ -225,37 +233,39 @@ export async function createStatusReportBase(
status: ActionStatus,
actionStartedAt: Date,
cause?: string,
exception?: string):
Promise<StatusReportBase> {
const commitOid = process.env['GITHUB_SHA'] || '';
exception?: string
): Promise<StatusReportBase> {
const commitOid = process.env["GITHUB_SHA"] || "";
const ref = getRef();
const workflowRunIDStr = process.env['GITHUB_RUN_ID'];
const workflowRunIDStr = process.env["GITHUB_RUN_ID"];
let workflowRunID = -1;
if (workflowRunIDStr) {
workflowRunID = parseInt(workflowRunIDStr, 10);
}
const workflowName = process.env['GITHUB_WORKFLOW'] || '';
const jobName = process.env['GITHUB_JOB'] || '';
const workflowName = process.env["GITHUB_WORKFLOW"] || "";
const jobName = process.env["GITHUB_JOB"] || "";
const analysis_key = await getAnalysisKey();
let workflowStartedAt = process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT];
if (workflowStartedAt === undefined) {
workflowStartedAt = actionStartedAt.toISOString();
core.exportVariable(sharedEnv.CODEQL_WORKFLOW_STARTED_AT, workflowStartedAt);
core.exportVariable(
sharedEnv.CODEQL_WORKFLOW_STARTED_AT,
workflowStartedAt
);
}
let statusReport: StatusReportBase = {
const statusReport: StatusReportBase = {
workflow_run_id: workflowRunID,
workflow_name: workflowName,
job_name: jobName,
analysis_key: analysis_key,
analysis_key,
commit_oid: commitOid,
ref: ref,
ref,
action_name: actionName,
action_oid: "unknown", // TODO decide if it's possible to fill this in
started_at: workflowStartedAt,
action_started_at: actionStartedAt.toISOString(),
status: status
status,
};
// Add optional parameters
@ -265,10 +275,10 @@ export async function createStatusReportBase(
if (exception) {
statusReport.exception = exception;
}
if (status === 'success' || status === 'failure' || status === 'aborted') {
if (status === "success" || status === "failure" || status === "aborted") {
statusReport.completed_at = new Date().toISOString();
}
let matrix: string | undefined = core.getInput('matrix');
const matrix: string | undefined = core.getInput("matrix");
if (matrix) {
statusReport.matrix_vars = matrix;
}
@ -287,8 +297,8 @@ export async function createStatusReportBase(
*/
export async function sendStatusReport<S extends StatusReportBase>(
statusReport: S,
ignoreFailures?: boolean): Promise<boolean> {
ignoreFailures?: boolean
): Promise<boolean> {
if (getRequiredEnvParam("GITHUB_SERVER_URL") !== GITHUB_DOTCOM_URL) {
core.debug("Not sending status report to GitHub Enterprise");
return true;
@ -300,16 +310,19 @@ export async function sendStatusReport<S extends StatusReportBase>(
}
const statusReportJSON = JSON.stringify(statusReport);
core.debug('Sending status report: ' + statusReportJSON);
core.debug(`Sending status report: ${statusReportJSON}`);
const nwo = getRequiredEnvParam("GITHUB_REPOSITORY");
const [owner, repo] = nwo.split("/");
const client = api.getActionsApiClient();
const statusResponse = await client.request('PUT /repos/:owner/:repo/code-scanning/analysis/status', {
owner: owner,
repo: repo,
data: statusReportJSON,
});
const statusResponse = await client.request(
"PUT /repos/:owner/:repo/code-scanning/analysis/status",
{
owner,
repo,
data: statusReportJSON,
}
);
if (!ignoreFailures) {
// If the status report request fails with a 403 or a 404, then this is a deliberate
@ -319,11 +332,15 @@ export async function sendStatusReport<S extends StatusReportBase>(
// Other failure responses (or lack thereof) could be transitory and should not
// cause the action to fail.
if (statusResponse.status === 403) {
core.setFailed('The repo on which this action is running is not opted-in to CodeQL code scanning.');
core.setFailed(
"The repo on which this action is running is not opted-in to CodeQL code scanning."
);
return false;
}
if (statusResponse.status === 404) {
core.setFailed('Not authorized to used the CodeQL code scanning feature on this repo.');
core.setFailed(
"Not authorized to used the CodeQL code scanning feature on this repo."
);
return false;
}
}
@ -353,12 +370,14 @@ export function getToolNames(sarifContents: string): string[] {
// Creates a random temporary directory, runs the given body, and then deletes the directory.
// Mostly intended for use within tests.
export async function withTmpDir<T>(body: (tmpDir: string) => Promise<T>): Promise<T> {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codeql-action-'));
const realSubdir = path.join(tmpDir, 'real');
export async function withTmpDir<T>(
body: (tmpDir: string) => Promise<T>
): Promise<T> {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "codeql-action-"));
const realSubdir = path.join(tmpDir, "real");
fs.mkdirSync(realSubdir);
const symlinkSubdir = path.join(tmpDir, 'symlink');
fs.symlinkSync(realSubdir, symlinkSubdir, 'dir');
const symlinkSubdir = path.join(tmpDir, "symlink");
fs.symlinkSync(realSubdir, symlinkSubdir, "dir");
const result = await body(symlinkSubdir);
fs.rmdirSync(tmpDir, { recursive: true });
return result;
@ -375,7 +394,7 @@ export function getMemoryFlag(userInput: string | undefined): string {
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + userInput + "\", specified.");
throw new Error(`Invalid RAM setting "${userInput}", specified.`);
}
} else {
const totalMemoryBytes = os.totalmem();
@ -383,7 +402,7 @@ export function getMemoryFlag(userInput: string | undefined): string {
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
}
return "--ram=" + Math.floor(memoryToUseMegaBytes);
return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
}
/**
@ -391,12 +410,14 @@ export function getMemoryFlag(userInput: string | undefined): string {
*
* @returns string
*/
export function getAddSnippetsFlag(userInput: string | boolean | undefined): string {
export function getAddSnippetsFlag(
userInput: string | boolean | undefined
): string {
if (typeof userInput === "string") {
// have to process specifically because any non-empty string is truthy
userInput = userInput.toLowerCase() === "true";
}
return userInput ? "--sarif-add-snippets" : "--no-sarif-add-snippets";
return userInput ? "--sarif-add-snippets" : "--no-sarif-add-snippets";
}
/**
@ -407,7 +428,10 @@ export function getAddSnippetsFlag(userInput: string | boolean | undefined): str
*
* @returns string
*/
export function getThreadsFlag(userInput: string | undefined, logger: Logger): string {
export function getThreadsFlag(
userInput: string | undefined,
logger: Logger
): string {
let numThreads: number;
const maxThreads = os.cpus().length;
if (userInput) {
@ -416,12 +440,16 @@ export function getThreadsFlag(userInput: string | undefined, logger: Logger): s
throw new Error(`Invalid threads setting "${userInput}", specified.`);
}
if (numThreads > maxThreads) {
logger.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
logger.info(
`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`
);
numThreads = maxThreads;
}
const minThreads = -maxThreads;
if (numThreads < minThreads) {
logger.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
logger.info(
`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`
);
numThreads = minThreads;
}
} else {
@ -435,7 +463,7 @@ export function getThreadsFlag(userInput: string | undefined, logger: Logger): s
* Get the directory where CodeQL databases should be placed.
*/
export function getCodeQLDatabasesDir(tempDir: string) {
return path.resolve(tempDir, 'codeql_databases');
return path.resolve(tempDir, "codeql_databases");
}
/**