robojerk/codeql-action

Merge branch 'main' into nickfyson/error_wrapper

Browse source 
# Conflicts:
#	lib/codeql.js
#	lib/codeql.js.map
#	src/codeql.ts
This commit is contained in:
Nick Fyson 2020-09-07 23:55:32 +01:00
commit 3cd41279f2
3466 changed files with 8685 additions and 446173 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/analysis-paths.ts
View file

@ -1,6 +1,5 @@
import * as core from '@actions/core';
import * as configUtils from './config-utils';
import { Logger } from './logging';
function isInterpretedLanguage(language): boolean {
return language === 'javascript' || language === 'python';
@ -22,6 +21,17 @@ function buildIncludeExcludeEnvVar(paths: string[]): string {
return paths.join('\n');
}
export function printPathFiltersWarning(config: configUtils.Config, logger: Logger) {
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
logger.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
}
}
export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// The 'LGTM_INDEX_INCLUDE' and 'LGTM_INDEX_EXCLUDE' environment variables
// control which files/directories are traversed when scanning.
@ -31,10 +41,10 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// traverse the entire file tree to determine which files are matched.
// Any paths containing "*" are not included in these.
if (config.paths.length !== 0) {
core.exportVariable('LGTM_INDEX_INCLUDE', buildIncludeExcludeEnvVar(config.paths));
process.env['LGTM_INDEX_INCLUDE'] = buildIncludeExcludeEnvVar(config.paths);
}
if (config.pathsIgnore.length !== 0) {
core.exportVariable('LGTM_INDEX_EXCLUDE', buildIncludeExcludeEnvVar(config.pathsIgnore));
process.env['LGTM_INDEX_EXCLUDE'] = buildIncludeExcludeEnvVar(config.pathsIgnore);
}
// The 'LGTM_INDEX_FILTERS' environment variable controls which files are
@ -44,15 +54,6 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
filters.push(...config.paths.map(p => 'include:' + p));
filters.push(...config.pathsIgnore.map(p => 'exclude:' + p));
if (filters.length !== 0) {
core.exportVariable('LGTM_INDEX_FILTERS', filters.join('\n'));
}
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0 ||
filters.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
core.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
process.env['LGTM_INDEX_FILTERS'] = filters.join('\n');
}
}

70
src/analyze-action.ts Normal file
View file

@ -0,0 +1,70 @@
import * as core from '@actions/core';
import { AnalysisStatusReport, runAnalyze } from './analyze';
import { getConfig } from './config-utils';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as util from './util';
interface FinishStatusReport extends util.StatusReportBase, AnalysisStatusReport {}
async function sendStatusReport(
startedAt: Date,
stats: AnalysisStatusReport | undefined,
error?: Error) {
const status = stats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase('finish', status, startedAt, error?.message, error?.stack);
const statusReport: FinishStatusReport = {
...statusReportBase,
...(stats || {}),
};
await util.sendStatusReport(statusReport);
}
async function run() {
const startedAt = new Date();
let stats: AnalysisStatusReport | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
return;
}
const logger = getActionsLogger();
const config = await getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
}
stats = await runAnalyze(
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
core.getInput('upload') === 'true',
'actions',
core.getInput('output'),
util.getMemoryFlag(core.getInput('ram')),
util.getThreadsFlag(core.getInput('threads'), logger),
config,
logger);
} catch (error) {
core.setFailed(error.message);
console.log(error);
await sendStatusReport(startedAt, stats, error);
return;
}
await sendStatusReport(startedAt, stats);
}
run().catch(e => {
core.setFailed("analyze action failed: " + e);
console.log(e);
});

172
src/analyze.ts Normal file
View file

@ -0,0 +1,172 @@
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
export interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no langauge failed)
analyze_failure_language?: string;
}
export interface AnalysisStatusReport extends upload_lib.UploadStatusReport, QueriesStatusReport {}
async function createdDBForScannedLanguages(
config: configUtils.Config,
logger: Logger) {
// Insert the LGTM_INDEX_X env vars at this point so they are set when
// we extract any scanned languages.
analysisPaths.includeAndExcludeAnalysisPaths(config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
logger.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
logger.endGroup();
}
}
}
async function finalizeDatabaseCreation(
config: configUtils.Config,
logger: Logger) {
await createdDBForScannedLanguages(config, logger);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
logger.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
logger.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
sarifFolder: string,
memoryFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of config.languages) {
logger.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
logger.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, threadsFlag);
logger.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
logger.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
analyze_failure_language: language,
};
}
}
return {};
}
export async function runAnalyze(
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string | undefined,
analysisName: string | undefined,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubUrl: string,
doUpload: boolean,
mode: util.Mode,
outputDir: string,
memoryFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<AnalysisStatusReport> {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
logger.info('Finalizing database creation');
await finalizeDatabaseCreation(config, logger);
logger.info('Analyzing database');
const queriesStats = await runQueries(outputDir, memoryFlag, threadsFlag, config, logger);
if (!doUpload) {
logger.info('Not uploading results');
return { ...queriesStats };
}
const uploadStats = await upload_lib.upload(
outputDir,
repositoryNwo,
commitOid,
ref,
analysisKey,
analysisName,
workflowRunID,
checkoutPath,
environment,
githubAuth,
githubUrl,
mode,
logger);
return { ...queriesStats, ...uploadStats };
}

28
src/api-client.ts
View file

@ -1,33 +1,35 @@
import * as core from "@actions/core";
import * as github from "@actions/github";
import consoleLogLevel from "console-log-level";
import * as path from 'path';
import { getRequiredEnvParam, isLocalRun } from "./util";
export const getApiClient = function(githubAuth: string, githubApiUrl: string, allowLocalRun = false) {
export const getApiClient = function(githubAuth: string, githubUrl: string, allowLocalRun = false) {
if (isLocalRun() && !allowLocalRun) {
throw new Error('Invalid API call in local run');
}
return new github.GitHub(
{
auth: parseAuth(githubAuth),
baseUrl: githubApiUrl,
auth: githubAuth,
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
log: consoleLogLevel({ level: "debug" })
});
};
// Parses the user input as either a single token,
// or a username and password / PAT.
function parseAuth(auth: string): string {
// Check if it's a username:password pair
const c = auth.indexOf(':');
if (c !== -1) {
return 'basic ' + Buffer.from(auth).toString('base64');
function getApiUrl(githubUrl: string): string {
const url = new URL(githubUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
}
// Otherwise use the token as it is
return auth;
// Add the /api/v3 API prefix
url.pathname = path.join(url.pathname, 'api', 'v3');
return url.toString();
}
// Temporary function to aid in the transition to running on and off of github actions.
@ -36,6 +38,6 @@ function parseAuth(auth: string): string {
export function getActionsApiClient(allowLocalRun = false) {
return getApiClient(
core.getInput('token'),
getRequiredEnvParam('GITHUB_API_URL'),
getRequiredEnvParam('GITHUB_SERVER_URL'),
allowLocalRun);
}

69
src/autobuild-action.ts Normal file
View file

@ -0,0 +1,69 @@
import * as core from '@actions/core';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import * as config_utils from './config-utils';
import { Language } from './languages';
import { getActionsLogger } from './logging';
import * as util from './util';
interface AutobuildStatusReport extends util.StatusReportBase {
// Comma-separated set of languages being autobuilt
autobuild_languages: string;
// Language that failed autobuilding (or undefined if all languages succeeded).
autobuild_failure?: string;
}
async function sendCompletedStatusReport(
startedAt: Date,
allLanguages: string[],
failingLanguage?: string,
cause?: Error) {
const status = failingLanguage !== undefined || cause !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase(
'autobuild',
status,
startedAt,
cause?.message,
cause?.stack);
const statusReport: AutobuildStatusReport = {
...statusReportBase,
autobuild_languages: allLanguages.join(','),
autobuild_failure: failingLanguage,
};
await util.sendStatusReport(statusReport);
}
async function run() {
const logger = getActionsLogger();
const startedAt = new Date();
let language: Language | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('autobuild', 'starting', startedAt), true)) {
return;
}
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
}
language = determineAutobuildLanguage(config, logger);
if (language !== undefined) {
await runAutobuild(language, config, logger);
}
} catch (error) {
core.setFailed("We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. " + error.message);
console.log(error);
await sendCompletedStatusReport(startedAt, language ? [language] : [], language, error);
return;
}
await sendCompletedStatusReport(startedAt, language ? [language] : []);
}
run().catch(e => {
core.setFailed("autobuild action failed. " + e);
console.log(e);
});

107
src/autobuild.ts
View file

@ -1,84 +1,41 @@
import * as core from '@actions/core';
import { getCodeQL } from './codeql';
import * as config_utils from './config-utils';
import { isTracedLanguage } from './languages';
import * as util from './util';
import { isTracedLanguage, Language } from './languages';
import { Logger } from './logging';
interface AutobuildStatusReport extends util.StatusReportBase {
// Comma-separated set of languages being autobuilt
autobuild_languages: string;
// Language that failed autobuilding (or undefined if all languages succeeded).
autobuild_failure?: string;
}
export function determineAutobuildLanguage(
config: config_utils.Config,
logger: Logger
): Language | undefined {
async function sendCompletedStatusReport(
startedAt: Date,
allLanguages: string[],
failingLanguage?: string,
cause?: Error) {
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = config.languages.filter(isTracedLanguage);
const language = autobuildLanguages[0];
const status = failingLanguage !== undefined || cause !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase(
'autobuild',
status,
startedAt,
cause?.message,
cause?.stack);
const statusReport: AutobuildStatusReport = {
...statusReportBase,
autobuild_languages: allLanguages.join(','),
autobuild_failure: failingLanguage,
};
await util.sendStatusReport(statusReport);
}
async function run() {
const startedAt = new Date();
let language;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('autobuild', 'starting', startedAt), true)) {
return;
}
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = config.languages.filter(isTracedLanguage);
language = autobuildLanguages[0];
if (!language) {
core.info("None of the languages in this project require extra build steps");
return;
}
core.debug(`Detected dominant traced language: ${language}`);
if (autobuildLanguages.length > 1) {
core.warning(`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages.slice(1).join(' and ')}, you must replace this block with custom build steps.`);
}
core.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = getCodeQL(config.codeQLCmd);
await codeQL.runAutobuild(language);
core.endGroup();
} catch (error) {
core.setFailed("We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. " + error.message);
console.log(error);
await sendCompletedStatusReport(startedAt, [language], language, error);
return;
if (!language) {
logger.info("None of the languages in this project require extra build steps");
return undefined;
}
await sendCompletedStatusReport(startedAt, [language]);
logger.debug(`Detected dominant traced language: ${language}`);
if (autobuildLanguages.length > 1) {
logger.warning(`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages.slice(1).join(' and ')}, you must replace this call with custom build steps.`);
}
return language;
}
run().catch(e => {
core.setFailed("autobuild action failed. " + e);
console.log(e);
});
export async function runAutobuild(
language: Language,
config: config_utils.Config,
logger: Logger) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = getCodeQL(config.codeQLCmd);
await codeQL.runAutobuild(language);
logger.endGroup();
}

78
src/cli.ts
View file

@ -1,78 +0,0 @@
import { Command } from 'commander';
import * as path from 'path';
import { getCLILogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
const program = new Command();
program.version('0.0.1');
interface UploadArgs {
sarifFile: string;
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
}
function parseGithubApiUrl(inputUrl: string): string {
try {
const url = new URL(inputUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
}
// Add the API path if it's not already present.
if (url.pathname.indexOf('/api/v3') === -1) {
url.pathname = path.join(url.pathname, 'api', 'v3');
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
const logger = getCLILogger();
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload; can also be a directory for uploading multiple')
.requiredOption('--repository <repository>', 'Repository name')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed')
.requiredOption('--github-url <url>', 'URL of GitHub instance')
.requiredOption('--github-auth <auth>', 'GitHub Apps token, or of the form "username:token" if using a personal access token')
.option('--checkout-path <path>', 'Checkout path (default: current working directory)')
.action(async (cmd: UploadArgs) => {
try {
await upload_lib.upload(
cmd.sarifFile,
parseRepositoryNwo(cmd.repository),
cmd.commit,
cmd.ref,
undefined,
undefined,
undefined,
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubApiUrl(cmd.githubUrl),
'cli',
logger);
} catch (e) {
logger.error('Upload failed');
logger.error(e);
process.exitCode = 1;
}
});
program.parse(process.argv);

21
src/codeql.test.ts
View file

@ -4,6 +4,7 @@ import nock from 'nock';
import * as path from 'path';
import * as codeql from './codeql';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
@ -12,12 +13,6 @@ setupTests(test);
test('download codeql bundle cache', async t => {
await util.withTmpDir(async tmpDir => {
process.env['GITHUB_WORKSPACE'] = tmpDir;
process.env['RUNNER_TEMP'] = path.join(tmpDir, 'temp');
process.env['RUNNER_TOOL_CACHE'] = path.join(tmpDir, 'cache');
const versions = ['20200601', '20200610'];
for (let i = 0; i < versions.length; i++) {
@ -27,10 +22,14 @@ test('download codeql bundle cache', async t => {
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(200, path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`));
process.env['INPUT_TOOLS'] = `https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
await codeql.setupCodeQL();
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
'token',
'https://github.example.com',
tmpDir,
tmpDir,
'runner',
getRunnerLogger(true));
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
}
@ -56,7 +55,7 @@ test('parse codeql bundle url version', t => {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = codeql.getCodeQLURLVersion(url);
const parsedVersion = codeql.getCodeQLURLVersion(url, getRunnerLogger(true));
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e.message);

163
src/codeql.ts
View file

@ -1,5 +1,4 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as http from '@actions/http-client';
import { IHeaders } from '@actions/http-client/interfaces';
import * as toolcache from '@actions/tool-cache';
@ -15,6 +14,7 @@ import * as defaults from './defaults.json'; // Referenced from codeql-action-sy
import { errorMatchers} from './error-matcher';
import { execErrorCatcher } from './exec-wrapper';
import { Language } from './languages';
import { Logger } from './logging';
import * as util from './util';
type Options = (string|number|boolean)[];
@ -51,7 +51,7 @@ export interface CodeQL {
* Run 'codeql database trace-command' on 'tracer-env.js' and parse
* the result to get environment variables set by CodeQL.
*/
getTracerEnv(databasePath: string, compilerSpec: string | undefined): Promise<{ [key: string]: string }>;
getTracerEnv(databasePath: string): Promise<{ [key: string]: string }>;
/**
* Run 'codeql database init'.
*/
@ -76,7 +76,12 @@ export interface CodeQL {
/**
* Run 'codeql database analyze'.
*/
databaseAnalyze(databasePath: string, sarifFile: string, querySuite: string): Promise<void>;
databaseAnalyze(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
threadsFlag: string): Promise<void>;
}
export interface ResolveQueriesOutput {
@ -103,7 +108,11 @@ const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
const CODEQL_BUNDLE_NAME = "codeql-bundle.tar.gz";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
function getCodeQLActionRepository(): string {
function getCodeQLActionRepository(mode: util.Mode): string {
if (mode !== 'actions') {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
// Actions do not know their own repository name,
// so we currently use this hack to find the name based on where our files are.
// This can be removed once the change to the runner in https://github.com/actions/runner/pull/585 is deployed.
@ -119,15 +128,20 @@ function getCodeQLActionRepository(): string {
return relativeScriptPathParts[0] + "/" + relativeScriptPathParts[1];
}
async function getCodeQLBundleDownloadURL(): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository();
async function getCodeQLBundleDownloadURL(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(mode);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[util.getInstanceAPIURL(), codeQLActionRepository],
[githubUrl, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
[util.getInstanceAPIURL(), CODEQL_DEFAULT_ACTION_REPOSITORY],
[githubUrl, CODEQL_DEFAULT_ACTION_REPOSITORY],
// GitHub.com, and the canonical Action.
[util.GITHUB_DOTCOM_API_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
[util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
@ -135,24 +149,24 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
for (let downloadSource of uniqueDownloadSources) {
let [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (apiURL === util.GITHUB_DOTCOM_API_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
if (apiURL === util.GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
break;
}
let [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getActionsApiClient().repos.getReleaseByTag({
const release = await api.getApiClient(githubAuth, githubUrl).repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION
});
for (let asset of release.data.assets) {
if (asset.name === CODEQL_BUNDLE_NAME) {
core.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
logger.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
return asset.url;
}
}
} catch (e) {
core.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
logger.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${CODEQL_BUNDLE_NAME}`;
@ -160,16 +174,18 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
// We have to download CodeQL manually because the toolcache doesn't support Accept headers.
// This can be removed once https://github.com/actions/toolkit/pull/530 is merged and released.
async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<string> {
async function toolcacheDownloadTool(
url: string,
headers: IHeaders | undefined,
tempDir: string,
logger: Logger): Promise<string> {
const client = new http.HttpClient('CodeQL Action');
const dest = path.join(util.getRequiredEnvParam('RUNNER_TEMP'), uuidV4());
const dest = path.join(tempDir, uuidV4());
const response: http.HttpClientResponse = await client.get(url, headers);
if (response.message.statusCode !== 200) {
const err = new toolcache.HTTPError(response.message.statusCode);
core.info(
`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`
);
throw err;
logger.info(`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`);
throw new Error(`Unexpected HTTP response: ${response.message.statusCode}`);
}
const pipeline = globalutil.promisify(stream.pipeline);
fs.mkdirSync(path.dirname(dest), { recursive: true });
@ -177,32 +193,45 @@ async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<s
return dest;
}
export async function setupCodeQL(): Promise<CodeQL> {
export async function setupCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
// Setting these two env vars makes the toolcache code safe to use outside,
// of actions but this is obviously not a great thing we're doing and it would
// be better to write our own implementation to use outside of actions.
process.env['RUNNER_TEMP'] = tempDir;
process.env['RUNNER_TOOL_CACHE'] = toolsDir;
try {
let codeqlURL = core.getInput('tools');
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`, logger);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL();
codeqlURL = await getCodeQLBundleDownloadURL(githubAuth, githubUrl, mode, logger);
}
const headers: IHeaders = {accept: 'application/octet-stream'};
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
if (codeqlURL.startsWith(util.getInstanceAPIURL() + "/")) {
core.debug('Downloading CodeQL bundle with token.');
let token = core.getInput('token', { required: true });
headers.authorization = `token ${token}`;
if (codeqlURL.startsWith(githubUrl + "/")) {
logger.debug('Downloading CodeQL bundle with token.');
headers.authorization = `token ${githubAuth}`;
} else {
core.debug('Downloading CodeQL bundle without token.');
logger.debug('Downloading CodeQL bundle without token.');
}
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers);
core.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers, tempDir, logger);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
@ -219,12 +248,12 @@ export async function setupCodeQL(): Promise<CodeQL> {
return cachedCodeQL;
} catch (e) {
core.error(e);
logger.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string): string {
export function getCodeQLURLVersion(url: string, logger: Logger): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
@ -234,7 +263,7 @@ export function getCodeQLURLVersion(url: string): string {
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}
@ -313,35 +342,49 @@ function getCodeQLForCmd(cmd: string): CodeQL {
return cmd;
},
printVersion: async function() {
await exec.exec(cmd, [
await new toolrunnner.ToolRunner(cmd, [
'version',
'--format=json'
]);
]).exec();
},
getTracerEnv: async function(databasePath: string, compilerSpec: string | undefined) {
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
await exec.exec(cmd, [
getTracerEnv: async function(databasePath: string) {
// Write tracer-env.js to a temp location.
const tracerEnvJs = path.resolve(databasePath, 'working', 'tracer-env.js');
fs.mkdirSync(path.dirname(tracerEnvJs), {recursive: true});
fs.writeFileSync(tracerEnvJs, `
const fs = require('fs');
const env = {};
for (let entry of Object.entries(process.env)) {
const key = entry[0];
const value = entry[1];
if (typeof value !== 'undefined' && key !== '_' && !key.startsWith('JAVA_MAIN_CLASS_')) {
env[key] = value;
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');`);
const envFile = path.resolve(databasePath, 'working', 'env.tmp');
await new toolrunnner.ToolRunner(cmd, [
'database',
'trace-command',
databasePath,
...compilerSpecArg,
...getExtraOptionsFromEnv(['database', 'trace-command']),
process.execPath,
path.resolve(__dirname, 'tracer-env.js'),
tracerEnvJs,
envFile
]);
]).exec();
return JSON.parse(fs.readFileSync(envFile, 'utf-8'));
},
databaseInit: async function(databasePath: string, language: Language, sourceRoot: string) {
await exec.exec(cmd, [
await new toolrunnner.ToolRunner(cmd, [
'database',
'init',
databasePath,
'--language=' + language,
'--source-root=' + sourceRoot,
...getExtraOptionsFromEnv(['database', 'init']),
]);
]).exec();
},
runAutobuild: async function(language: Language) {
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
@ -355,12 +398,12 @@ function getCodeQLForCmd(cmd: string): CodeQL {
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
await exec.exec(autobuildCmd);
await new toolrunnner.ToolRunner(autobuildCmd).exec();
},
extractScannedLanguage: async function(databasePath: string, language: Language) {
// Get extractor location
let extractorPath = '';
await exec.exec(
await new toolrunnner.ToolRunner(
cmd,
[
'resolve',
@ -375,7 +418,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
}).exec();
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
@ -416,29 +459,35 @@ function getCodeQLForCmd(cmd: string): CodeQL {
codeqlArgs.push('--search-path', extraSearchPath);
}
let output = '';
await exec.exec(cmd, codeqlArgs, {
await new toolrunnner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
}
}
});
}).exec();
return JSON.parse(output);
},
databaseAnalyze: async function(databasePath: string, sarifFile: string, querySuite: string) {
await exec.exec(cmd, [
databaseAnalyze: async function(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
threadsFlag: string) {
await new toolrunnner.ToolRunner(cmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
memoryFlag,
threadsFlag,
databasePath,
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
...getExtraOptionsFromEnv(['database', 'analyze']),
querySuite
]);
]).exec();
}
};
}

512
src/config-utils.test.ts
View file

@ -8,22 +8,12 @@ import * as api from './api-client';
import { getCachedCodeQL, setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import {setupTests} from './testing-utils';
import * as util from './util';
setupTests(test);
function setInput(name: string, value: string | undefined) {
// Transformation copied from
// https://github.com/actions/toolkit/blob/05e39f551d33e1688f61b209ab5cdd335198f1b8/packages/core/src/core.ts#L69
const envVar = `INPUT_${name.replace(/ /g, '_').toUpperCase()}`;
if (value !== undefined) {
process.env[envVar] = value;
} else {
delete process.env[envVar];
}
}
type GetContentsResponse = { content?: string; } | {}[];
function mockGetContents(content: GetContentsResponse): sinon.SinonStub<any, any> {
@ -52,11 +42,8 @@ function mockListLanguages(languages: string[]) {
test("load empty config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', undefined);
setInput('languages', 'javascript,python');
const logger = getRunnerLogger(true);
const languages = 'javascript,python';
const codeQL = setCodeQL({
resolveQueries: async function() {
@ -68,19 +55,36 @@ test("load empty config", async t => {
},
});
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger);
t.deepEqual(config, await configUtils.getDefaultConfig(tmpDir, tmpDir, codeQL));
t.deepEqual(config, await configUtils.getDefaultConfig(
languages,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger));
});
});
test("loading config saves config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', undefined);
setInput('languages', 'javascript,python');
const logger = getRunnerLogger(true);
const codeQL = setCodeQL({
resolveQueries: async function() {
@ -96,29 +100,46 @@ test("loading config saves config", async t => {
// Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
// Sanity check that getConfig throws before we have called initConfig
await t.throwsAsync(() => configUtils.getConfig(tmpDir));
// Sanity check that getConfig returns undefined before we have called initConfig
t.deepEqual(await configUtils.getConfig(tmpDir, logger), undefined);
const config1 = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config1 = await configUtils.initConfig(
'javascript,python',
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger);
// The saved config file should now exist
t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
// And that same newly-initialised config should now be returned by getConfig
const config2 = await configUtils.getConfig(tmpDir);
const config2 = await configUtils.getConfig(tmpDir, logger);
t.deepEqual(config1, config2);
});
});
test("load input outside of workspace", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', '../input');
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
'../input',
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileOutsideWorkspaceErrorMessage(path.join(tmpDir, '../input'))));
@ -128,14 +149,22 @@ test("load input outside of workspace", async t => {
test("load non-local input with invalid repo syntax", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
// no filename given, just a repo
setInput('config-file', 'octo-org/codeql-config@main');
const configFile = 'octo-org/codeql-config@main';
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileRepoFormatInvalidMessage('octo-org/codeql-config@main')));
@ -145,15 +174,23 @@ test("load non-local input with invalid repo syntax", async t => {
test("load non-existent input", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
t.false(fs.existsSync(path.join(tmpDir, 'input')));
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
t.false(fs.existsSync(path.join(tmpDir, configFile)));
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileDoesNotExistErrorMessage(path.join(tmpDir, 'input'))));
@ -163,9 +200,6 @@ test("load non-existent input", async t => {
test("load non-empty input", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@ -213,11 +247,22 @@ test("load non-empty input", async t => {
codeQLCmd: codeQL.getPath(),
};
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
const actualConfig = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const actualConfig = await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Should exactly equal the object we constructed earlier
t.deepEqual(actualConfig, expectedConfig);
@ -226,9 +271,6 @@ test("load non-empty input", async t => {
test("default queries are used", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
// Check that the default behaviour is to add the default queries.
// In this case if a config file is specified but does not include
// the disable-default-queries field.
@ -260,11 +302,22 @@ test("default queries are used", async t => {
fs.mkdirSync(path.join(tmpDir, 'foo'));
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolve queries was called correctly
t.deepEqual(resolveQueriesArgs.length, 1);
@ -273,11 +326,229 @@ test("default queries are used", async t => {
});
});
test("Queries can be specified in config file", async t => {
return await util.withTmpDir(async tmpDir => {
const inputFileContents = `
name: my config
queries:
- uses: ./foo`;
const configFile = path.join(tmpDir, 'input');
fs.writeFileSync(configFile, inputFileContents, 'utf8');
fs.mkdirSync(path.join(tmpDir, 'foo'));
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
const codeQL = setCodeQL({
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
resolveQueriesArgs.push({queries, extraSearchPath});
// Return what we're given, just in the right format for a resolved query
// This way we can test by seeing which returned items are in the final
// configuration.
const dummyResolvedQueries = {};
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
return {
byLanguage: {
'javascript': dummyResolvedQueries,
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const languages = 'javascript';
const config = await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly
// It'll be called once for the default queries
// and once for `./foo` from the config file.
t.deepEqual(resolveQueriesArgs.length, 2);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/foo$/);
// Now check that the end result contains the default queries and the query from config
t.deepEqual(config.queries['javascript'].length, 2);
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
t.regex(config.queries['javascript'][1], /.*\/foo$/);
});
});
test("Queries from config file can be overridden in workflow file", async t => {
return await util.withTmpDir(async tmpDir => {
const inputFileContents = `
name: my config
queries:
- uses: ./foo`;
const configFile = path.join(tmpDir, 'input');
fs.writeFileSync(configFile, inputFileContents, 'utf8');
// This config item should take precedence over the config file but shouldn't affect the default queries.
const queries = './override';
fs.mkdirSync(path.join(tmpDir, 'foo'));
fs.mkdirSync(path.join(tmpDir, 'override'));
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
const codeQL = setCodeQL({
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
resolveQueriesArgs.push({queries, extraSearchPath});
// Return what we're given, just in the right format for a resolved query
// This way we can test overriding by seeing which returned items are in
// the final configuration.
const dummyResolvedQueries = {};
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
return {
byLanguage: {
'javascript': dummyResolvedQueries,
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const languages = 'javascript';
const config = await configUtils.initConfig(
languages,
queries,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly
// It'll be called once for the default queries and once for `./override`,
// but won't be called for './foo' from the config file.
t.deepEqual(resolveQueriesArgs.length, 2);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override$/);
// Now check that the end result contains only the default queries and the override query
t.deepEqual(config.queries['javascript'].length, 2);
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
t.regex(config.queries['javascript'][1], /.*\/override$/);
});
});
test("Multiple queries can be specified in workflow file, no config file required", async t => {
return await util.withTmpDir(async tmpDir => {
fs.mkdirSync(path.join(tmpDir, 'override1'));
fs.mkdirSync(path.join(tmpDir, 'override2'));
const queries = './override1,./override2';
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
const codeQL = setCodeQL({
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
resolveQueriesArgs.push({queries, extraSearchPath});
// Return what we're given, just in the right format for a resolved query
// This way we can test overriding by seeing which returned items are in
// the final configuration.
const dummyResolvedQueries = {};
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
return {
byLanguage: {
'javascript': dummyResolvedQueries,
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const languages = 'javascript';
const config = await configUtils.initConfig(
languages,
queries,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly:
// It'll be called once for the default queries,
// and then once for each of the two queries from the workflow
t.deepEqual(resolveQueriesArgs.length, 3);
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
t.deepEqual(resolveQueriesArgs[2].queries.length, 1);
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override1$/);
t.regex(resolveQueriesArgs[2].queries[0], /.*\/override2$/);
// Now check that the end result contains both the queries from the workflow, as well as the defaults
t.deepEqual(config.queries['javascript'].length, 3);
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
t.regex(config.queries['javascript'][1], /.*\/override1$/);
t.regex(config.queries['javascript'][2], /.*\/override2$/);
});
});
test("Invalid queries in workflow file handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
const queries = 'foo/bar@v1@v3';
const languages = 'javascript';
// This function just needs to be type-correct; it doesn't need to do anything,
// since we're deliberately passing in invalid data
const codeQL = setCodeQL({
resolveQueries: async function(_queries: string[], _extraSearchPath: string | undefined) {
return {
byLanguage: {
'javascript': {},
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
try {
await configUtils.initConfig(
languages,
queries,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
t.fail('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3")));
}
});
});
test("API client used when reading remote config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@ -310,28 +581,46 @@ test("API client used when reading remote config", async t => {
const spyGetContents = mockGetContents(dummyResponse);
// Create checkout directory for remote queries repository
fs.mkdirSync(path.join(tmpDir, 'foo/bar'), { recursive: true });
fs.mkdirSync(path.join(tmpDir, 'foo/bar/dev'), { recursive: true });
setInput('config-file', 'octo-org/codeql-config/config.yaml@main');
setInput('languages', 'javascript');
const configFile = 'octo-org/codeql-config/config.yaml@main';
const languages = 'javascript';
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
t.assert(spyGetContents.called);
});
});
test("Remote config handles the case where a directory is provided", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const dummyResponse = []; // directories are returned as arrays
mockGetContents(dummyResponse);
const repoReference = 'octo-org/codeql-config/config.yaml@main';
setInput('config-file', repoReference);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
repoReference,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileDirectoryGivenMessage(repoReference)));
@ -341,18 +630,25 @@ test("Remote config handles the case where a directory is provided", async t =>
test("Invalid format of remote config handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const dummyResponse = {
// note no "content" property here
};
mockGetContents(dummyResponse);
const repoReference = 'octo-org/codeql-config/config.yaml@main';
setInput('config-file', repoReference);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
repoReference,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileFormatInvalidMessage(repoReference)));
@ -362,13 +658,21 @@ test("Invalid format of remote config handled correctly", async t => {
test("No detected languages", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
mockListLanguages([]);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getNoLanguagesError()));
@ -378,13 +682,21 @@ test("No detected languages", async t => {
test("Unknown languages", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('languages', 'ruby,english');
const languages = 'ruby,english';
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
languages,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(['ruby', 'english'])));
@ -399,9 +711,6 @@ function doInvalidInputTest(
test("load invalid input - " + testName, async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@ -412,13 +721,24 @@ function doInvalidInputTest(
},
});
const inputFile = path.join(tmpDir, 'input');
const languages = 'javascript';
const configFile = 'input';
const inputFile = path.join(tmpDir, configFile);
fs.writeFileSync(inputFile, inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
try {
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(expectedErrorMessageGenerator(inputFile)));
@ -526,10 +846,10 @@ test('path validations', t => {
const configFile = './.github/codeql/config.yml';
for (const path of validPaths) {
t.truthy(configUtils.validateAndSanitisePath(path, propertyName, configFile));
t.truthy(configUtils.validateAndSanitisePath(path, propertyName, configFile, getRunnerLogger(true)));
}
for (const path of invalidPaths) {
t.throws(() => configUtils.validateAndSanitisePath(path, propertyName, configFile));
t.throws(() => configUtils.validateAndSanitisePath(path, propertyName, configFile, getRunnerLogger(true)));
}
});
@ -540,11 +860,11 @@ test('path sanitisation', t => {
// Valid paths are not modified
t.deepEqual(
configUtils.validateAndSanitisePath('foo/bar', propertyName, configFile),
configUtils.validateAndSanitisePath('foo/bar', propertyName, configFile, getRunnerLogger(true)),
'foo/bar');
// Trailing stars are stripped
t.deepEqual(
configUtils.validateAndSanitisePath('foo/**', propertyName, configFile),
configUtils.validateAndSanitisePath('foo/**', propertyName, configFile, getRunnerLogger(true)),
'foo/');
});

341
src/config-utils.ts
View file

@ -1,4 +1,3 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import * as yaml from 'js-yaml';
import * as path from 'path';
@ -7,7 +6,8 @@ import * as api from './api-client';
import { CodeQL, ResolveQueriesOutput } from './codeql';
import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import * as util from './util';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
// Property names from the user-supplied config file.
const NAME_PROPERTY = 'name';
@ -160,11 +160,11 @@ const builtinSuites = ['security-extended', 'security-and-quality'] as const;
* Throws an error if suiteName is not a valid builtin suite.
*/
async function addBuiltinSuiteQueries(
configFile: string,
languages: string[],
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
suiteName: string) {
suiteName: string,
configFile?: string) {
const suite = builtinSuites.find((suite) => suite === suiteName);
if (!suite) {
@ -179,15 +179,15 @@ async function addBuiltinSuiteQueries(
* Retrieve the set of queries at localQueryPath and add them to resultMap.
*/
async function addLocalQueries(
configFile: string,
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
localQueryPath: string) {
localQueryPath: string,
checkoutPath: string,
configFile?: string) {
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE'));
let absoluteQueryPath = path.join(workspacePath, localQueryPath);
let absoluteQueryPath = path.join(checkoutPath, localQueryPath);
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
@ -198,25 +198,24 @@ async function addLocalQueries(
absoluteQueryPath = fs.realpathSync(absoluteQueryPath);
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (!(absoluteQueryPath + path.sep).startsWith(workspacePath + path.sep)) {
if (!(absoluteQueryPath + path.sep).startsWith(fs.realpathSync(checkoutPath) + path.sep)) {
throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
}
// Get the root of the current repo to use when resolving query dependencies
const rootOfRepo = util.getRequiredEnvParam('GITHUB_WORKSPACE');
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], rootOfRepo, true);
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath, true);
}
/**
* Retrieve the set of queries at the referenced remote repo and add them to resultMap.
*/
async function addRemoteQueries(
configFile: string,
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
queryUses: string,
tempDir: string) {
tempDir: string,
githubUrl: string,
logger: Logger,
configFile?: string) {
let tok = queryUses.split('@');
if (tok.length !== 2) {
@ -239,13 +238,18 @@ async function addRemoteQueries(
const nwo = tok[0] + '/' + tok[1];
// Checkout the external repository
const rootOfRepo = await externalQueries.checkoutExternalRepository(nwo, ref, tempDir);
const checkoutPath = await externalQueries.checkoutExternalRepository(
nwo,
ref,
githubUrl,
tempDir,
logger);
const queryPath = tok.length > 2
? path.join(rootOfRepo, tok.slice(2).join('/'))
: rootOfRepo;
? path.join(checkoutPath, tok.slice(2).join('/'))
: checkoutPath;
await runResolveQueries(codeQL, resultMap, [queryPath], rootOfRepo, true);
await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath, true);
}
/**
@ -257,12 +261,15 @@ async function addRemoteQueries(
* a finite set of hardcoded terms for builtin suites.
*/
async function parseQueryUses(
configFile: string,
languages: string[],
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
queryUses: string,
tempDir: string) {
tempDir: string,
checkoutPath: string,
githubUrl: string,
logger: Logger,
configFile?: string) {
queryUses = queryUses.trim();
if (queryUses === "") {
@ -271,18 +278,18 @@ async function parseQueryUses(
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
await addLocalQueries(configFile, codeQL, resultMap, queryUses.slice(2));
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), checkoutPath, configFile);
return;
}
// Check for one of the builtin suites
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
await addBuiltinSuiteQueries(configFile, languages, codeQL, resultMap, queryUses);
await addBuiltinSuiteQueries(languages, codeQL, resultMap, queryUses, configFile);
return;
}
// Otherwise, must be a reference to another repo
await addRemoteQueries(configFile, codeQL, resultMap, queryUses, tempDir);
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, githubUrl, logger, configFile);
}
// Regex validating stars in paths or paths-ignore entries.
@ -299,7 +306,8 @@ const filterPatternCharactersRegex = /.*[\?\+\[\]!].*/;
export function validateAndSanitisePath(
originalPath: string,
propertyName: string,
configFile: string): string {
configFile: string,
logger: Logger): string {
// Take a copy so we don't modify the original path, so we can still construct error messages
let path = originalPath;
@ -335,7 +343,7 @@ export function validateAndSanitisePath(
// Check for other regex characters that we don't support.
// Output a warning so the user knows, but otherwise continue normally.
if (path.match(filterPatternCharactersRegex)) {
core.warning(getConfigFilePropertyError(
logger.warning(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" contains an unsupported character. ' +
@ -356,6 +364,9 @@ export function validateAndSanitisePath(
return path;
}
// An undefined configFile in some of these functions indicates that
// the property was in a workflow file, not a config file
export function getNameInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, NAME_PROPERTY, 'must be a non-empty string');
}
@ -368,7 +379,7 @@ export function getQueriesInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY, 'must be an array');
}
export function getQueryUsesInvalid(configFile: string, queryUses?: string): string {
export function getQueryUsesInvalid(configFile: string | undefined, queryUses?: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
@ -385,14 +396,14 @@ export function getPathsInvalid(configFile: string): string {
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, 'must be an array of non-empty strings');
}
export function getLocalPathOutsideOfRepository(configFile: string, localPath: string): string {
export function getLocalPathOutsideOfRepository(configFile: string | undefined, localPath: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
'is invalid as the local path "' + localPath + '" is outside of the repository');
}
export function getLocalPathDoesNotExist(configFile: string, localPath: string): string {
export function getLocalPathDoesNotExist(configFile: string | undefined, localPath: string): string {
return getConfigFilePropertyError(
configFile,
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
@ -422,8 +433,12 @@ export function getConfigFileDirectoryGivenMessage(configFile: string): string {
return 'The configuration file "' + configFile + '" looks like a directory, not a file';
}
function getConfigFilePropertyError(configFile: string, property: string, error: string): string {
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
function getConfigFilePropertyError(configFile: string | undefined, property: string, error: string): string {
if (configFile === undefined) {
return 'The workflow property "' + property + '" is invalid: ' + error;
} else {
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
}
}
export function getNoLanguagesError(): string {
@ -438,35 +453,32 @@ export function getUnknownLanguagesError(languages: string[]): string {
/**
* Gets the set of languages in the current repository
*/
async function getLanguagesInRepo(): Promise<Language[]> {
let repo_nwo = process.env['GITHUB_REPOSITORY']?.split("/");
if (repo_nwo) {
let owner = repo_nwo[0];
let repo = repo_nwo[1];
async function getLanguagesInRepo(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
core.debug(`GitHub repo ${owner} ${repo}`);
const response = await api.getActionsApiClient(true).repos.listLanguages({
owner,
repo
});
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.listLanguages({
owner: repository.owner,
repo: repository.repo
});
core.debug("Languages API response: " + JSON.stringify(response));
logger.debug("Languages API response: " + JSON.stringify(response));
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
let languages: Set<Language> = new Set();
for (let lang of Object.keys(response.data)) {
let parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
let languages: Set<Language> = new Set();
for (let lang of Object.keys(response.data)) {
let parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
return [...languages];
} else {
return [];
}
return [...languages];
}
/**
@ -479,19 +491,28 @@ async function getLanguagesInRepo(): Promise<Language[]> {
* If no languages could be detected from either the workflow or the repository
* then throw an error.
*/
async function getLanguages(): Promise<Language[]> {
async function getLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
// Obtain from action input 'languages' if set
let languages = core.getInput('languages', { required: false })
let languages = (languagesInput || "")
.split(',')
.map(x => x.trim())
.filter(x => x.length > 0);
core.info("Languages from configuration: " + JSON.stringify(languages));
logger.info("Languages from configuration: " + JSON.stringify(languages));
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
languages = await getLanguagesInRepo();
core.info("Automatically detected languages: " + JSON.stringify(languages));
languages = await getLanguagesInRepo(
repository,
githubAuth,
githubUrl,
logger);
logger.info("Automatically detected languages: " + JSON.stringify(languages));
}
// If the languages parameter was not given and no languages were
@ -518,13 +539,68 @@ async function getLanguages(): Promise<Language[]> {
return parsedLanguages;
}
/**
* Returns true if queries were provided in the workflow file
* (and thus added), otherwise false
*/
async function addQueriesFromWorkflow(
codeQL: CodeQL,
queriesInput: string,
languages: string[],
resultMap: { [language: string]: string[] },
tempDir: string,
checkoutPath: string,
githubUrl: string,
logger: Logger) {
for (const query of queriesInput.split(',')) {
await parseQueryUses(
languages,
codeQL,
resultMap,
query,
tempDir,
checkoutPath,
githubUrl,
logger);
}
}
/**
* Get the default config for when the user has not supplied one.
*/
export async function getDefaultConfig(tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
const languages = await getLanguages();
export async function getDefaultConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
const queries = {};
await addDefaultQueries(codeQL, languages, queries);
if (queriesInput) {
await addQueriesFromWorkflow(
codeQL,
queriesInput,
languages,
queries,
tempDir,
checkoutPath,
githubUrl,
logger);
}
return {
languages: languages,
queries: queries,
@ -540,17 +616,30 @@ export async function getDefaultConfig(tempDir: string, toolCacheDir: string, co
/**
* Load the config from the given file.
*/
async function loadConfig(configFile: string, tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
async function loadConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
let parsedYAML: UserConfig;
if (isLocal(configFile)) {
// Treat the config file as relative to the workspace
const workspacePath = util.getRequiredEnvParam('GITHUB_WORKSPACE');
configFile = path.resolve(workspacePath, configFile);
parsedYAML = getLocalConfig(configFile, workspacePath);
configFile = path.resolve(checkoutPath, configFile);
parsedYAML = getLocalConfig(configFile, checkoutPath);
} else {
parsedYAML = await getRemoteConfig(configFile);
parsedYAML = await getRemoteConfig(
configFile,
githubAuth,
githubUrl);
}
// Validate that the 'name' property is syntactically correct,
@ -564,7 +653,12 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
}
}
const languages = await getLanguages();
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
const queries = {};
const pathsIgnore: string[] = [];
@ -581,7 +675,19 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
await addDefaultQueries(codeQL, languages, queries);
}
if (QUERIES_PROPERTY in parsedYAML) {
// If queries were provided using `with` in the action configuration,
// they should take precedence over the queries in the config file
if (queriesInput) {
await addQueriesFromWorkflow(
codeQL,
queriesInput,
languages,
queries,
tempDir,
checkoutPath,
githubUrl,
logger);
} else if (QUERIES_PROPERTY in parsedYAML) {
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
throw new Error(getQueriesInvalid(configFile));
}
@ -589,7 +695,16 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
throw new Error(getQueryUsesInvalid(configFile));
}
await parseQueryUses(configFile, languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir);
await parseQueryUses(
languages,
codeQL,
queries,
query[QUERIES_USES_PROPERTY],
tempDir,
checkoutPath,
githubUrl,
logger,
configFile);
}
}
@ -601,7 +716,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (typeof path !== "string" || path === '') {
throw new Error(getPathsIgnoreInvalid(configFile));
}
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile));
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile, logger));
});
}
@ -613,7 +728,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (typeof path !== "string" || path === '') {
throw new Error(getPathsInvalid(configFile));
}
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile));
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile, logger));
});
}
@ -644,20 +759,52 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
* This will parse the config from the user input if present, or generate
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
const configFile = core.getInput('config-file');
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
let config: Config;
// If no config file was provided create an empty one
if (configFile === '') {
core.debug('No configuration file was provided');
config = await getDefaultConfig(tempDir, toolCacheDir, codeQL);
if (!configFile) {
logger.debug('No configuration file was provided');
config = await getDefaultConfig(
languagesInput,
queriesInput,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
} else {
config = await loadConfig(configFile, tempDir, toolCacheDir, codeQL);
config = await loadConfig(
languagesInput,
queriesInput,
configFile,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config);
await saveConfig(config, logger);
return config;
}
@ -670,9 +817,9 @@ function isLocal(configPath: string): boolean {
return (configPath.indexOf("@") === -1);
}
function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
// Error if the config file is now outside of the workspace
if (!(configFile + path.sep).startsWith(workspacePath + path.sep)) {
if (!(configFile + path.sep).startsWith(checkoutPath + path.sep)) {
throw new Error(getConfigFileOutsideWorkspaceErrorMessage(configFile));
}
@ -684,7 +831,11 @@ function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
return yaml.safeLoad(fs.readFileSync(configFile, 'utf8'));
}
async function getRemoteConfig(configFile: string): Promise<UserConfig> {
async function getRemoteConfig(
configFile: string,
githubAuth: string,
githubUrl: string): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp('(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)');
const pieces = format.exec(configFile);
@ -693,7 +844,7 @@ async function getRemoteConfig(configFile: string): Promise<UserConfig> {
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
}
const response = await api.getActionsApiClient(true).repos.getContents({
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.getContents({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
@ -722,30 +873,26 @@ export function getPathToParsedConfigFile(tempDir: string): string {
/**
* Store the given config to the path returned from getPathToParsedConfigFile.
*/
async function saveConfig(config: Config) {
async function saveConfig(config: Config, logger: Logger) {
const configString = JSON.stringify(config);
const configFile = getPathToParsedConfigFile(config.tempDir);
fs.mkdirSync(path.dirname(configFile), { recursive: true });
fs.writeFileSync(configFile, configString, 'utf8');
core.debug('Saved config:');
core.debug(configString);
logger.debug('Saved config:');
logger.debug(configString);
}
/**
* Get the config.
*
* If this is the first time in a workflow that this is being called then
* this will parse the config from the user input. The parsed config is then
* stored to a known location. On the second and further calls, this will
* return the contents of the parsed config from the known location.
* Get the config that has been saved to the given temp dir.
* If the config could not be found then returns undefined.
*/
export async function getConfig(tempDir: string): Promise<Config> {
export async function getConfig(tempDir: string, logger: Logger): Promise<Config | undefined> {
const configFile = getPathToParsedConfigFile(tempDir);
if (!fs.existsSync(configFile)) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
return undefined;
}
const configString = fs.readFileSync(configFile, 'utf8');
core.debug('Loaded config:');
core.debug(configString);
logger.debug('Loaded config:');
logger.debug(configString);
return JSON.parse(configString);
}

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20200630"
"bundleVersion": "codeql-bundle-20200826"
}

53
src/exec-wrapper.test.ts
View file

@ -1,4 +1,5 @@
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import test from 'ava';
import { ErrorMatcher } from './error-matcher';
@ -9,26 +10,26 @@ setupTests(test);
test('matchers are never applied if non-error exit', async t => {
const testCommand = buildDummyCommand("foo bar\\nblort qux", "foo bar\\nblort qux", '', 0);
const testArgs = buildDummyArgs("foo bar\\nblort qux", "foo bar\\nblort qux", '', 0);
const matchers: ErrorMatcher[] = [[123, new RegExp("foo bar"), 'error!!!']];
t.deepEqual(await exec.exec(testCommand), 0);
t.deepEqual(await exec.exec('node', testArgs), 0);
t.deepEqual(await execErrorCatcher(testCommand, [], matchers), 0);
t.deepEqual(await execErrorCatcher('node', testArgs, matchers), 0);
});
test('regex matchers are applied to stdout for non-zero exit code', async t => {
const testCommand = buildDummyCommand("foo bar\\nblort qux", '', '', 1);
const testArgs = buildDummyArgs("foo bar\\nblort qux", '', '', 1);
const matchers: ErrorMatcher[] = [[123, new RegExp("foo bar"), '🦄']];
await t.throwsAsync(exec.exec(testCommand), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
execErrorCatcher(testCommand, [], matchers),
execErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
@ -36,14 +37,14 @@ test('regex matchers are applied to stdout for non-zero exit code', async t => {
test('regex matchers are applied to stderr for non-zero exit code', async t => {
const testCommand = buildDummyCommand("non matching string", 'foo bar\\nblort qux', '', 1);
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 1);
const matchers: ErrorMatcher[] = [[123, new RegExp("foo bar"), '🦄']];
await t.throwsAsync(exec.exec(testCommand), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
execErrorCatcher(testCommand, [], matchers),
execErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
@ -51,16 +52,16 @@ test('regex matchers are applied to stderr for non-zero exit code', async t => {
test('matcher returns correct error message when multiple matchers defined', async t => {
const testCommand = buildDummyCommand("non matching string", 'foo bar\\nblort qux', '', 1);
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 1);
const matchers: ErrorMatcher[] = [[456, new RegExp("lorem ipsum"), '😩'],
[123, new RegExp("foo bar"), '🦄'],
[789, new RegExp("blah blah"), '🤦‍♂️']];
await t.throwsAsync(exec.exec(testCommand), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
execErrorCatcher(testCommand, [], matchers),
execErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
@ -68,16 +69,16 @@ test('matcher returns correct error message when multiple matchers defined', asy
test('matcher returns first match to regex when multiple matches', async t => {
const testCommand = buildDummyCommand("non matching string", 'foo bar\\nblort qux', '', 1);
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 1);
const matchers: ErrorMatcher[] = [[123, new RegExp("foo bar"), '🦄'],
[789, new RegExp("blah blah"), '🤦‍♂️'],
[987, new RegExp("foo bar"), '🚫']];
await t.throwsAsync(exec.exec(testCommand), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 1'});
await t.throwsAsync(
execErrorCatcher(testCommand, [], matchers),
execErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
@ -85,25 +86,25 @@ test('matcher returns first match to regex when multiple matches', async t => {
test('exit code matchers are applied', async t => {
const testCommand = buildDummyCommand("non matching string", 'foo bar\\nblort qux', '', 123);
const testArgs = buildDummyArgs("non matching string", 'foo bar\\nblort qux', '', 123);
const matchers: ErrorMatcher[] = [[123, new RegExp("this will not match"), '🦄']];
await t.throwsAsync(exec.exec(testCommand), {instanceOf: Error, message: 'The process \'node\' failed with exit code 123'});
await t.throwsAsync(exec.exec('node', testArgs), {instanceOf: Error, message: 'The process \'node\' failed with exit code 123'});
await t.throwsAsync(
execErrorCatcher(testCommand, [], matchers),
execErrorCatcher('node', testArgs, matchers),
{instanceOf: Error, message: '🦄'}
);
});
test('execErrorCatcher respects the ignoreReturnValue option', async t => {
const testCommand = buildDummyCommand("standard output", 'error output', '', 199);
const testArgs = buildDummyArgs("standard output", 'error output', '', 199);
await t.throwsAsync(execErrorCatcher(testCommand, [], [], {ignoreReturnCode: false}), {instanceOf: Error});
await t.throwsAsync(execErrorCatcher('node', testArgs, [], {ignoreReturnCode: false}), {instanceOf: Error});
t.deepEqual(await execErrorCatcher(testCommand, [], [], {ignoreReturnCode: true}), 199);
t.deepEqual(await execErrorCatcher('node', testArgs, [], {ignoreReturnCode: true}), 199);
});
@ -124,17 +125,17 @@ test('execErrorCatcher preserves behavior of provided listeners', async t => {
}
};
const testCommand = buildDummyCommand(stdoutExpected, stderrExpected, '', 0);
const testArgs = buildDummyArgs(stdoutExpected, stderrExpected, '', 0);
t.deepEqual(await execErrorCatcher(testCommand, [], [], {listeners: listeners}), 0);
t.deepEqual(await execErrorCatcher('node', testArgs, [], {listeners: listeners}), 0);
t.deepEqual(stdoutActual, stdoutExpected + "\n");
t.deepEqual(stderrActual, stderrExpected + "\n");
});
function buildDummyCommand(stdoutContents: string, stderrContents: string,
desiredErrorMessage?: string, desiredExitCode?: number): string {
function buildDummyArgs(stdoutContents: string, stderrContents: string,
desiredErrorMessage?: string, desiredExitCode?: number): string[] {
let command = '';
@ -146,5 +147,5 @@ function buildDummyCommand(stdoutContents: string, stderrContents: string,
if (desiredErrorMessage) command += 'throw new Error(\\"' + desiredErrorMessage + '\\");';
if (desiredExitCode) command += 'process.exitCode = ' + desiredExitCode + ';';
return 'node -e "' + command + '"';
return toolrunnner.argStringToArray('-e "' + command + '"');
}

7
src/exec-wrapper.ts
View file

@ -1,5 +1,5 @@
import * as exec from '@actions/exec';
import * as im from '@actions/exec/lib/interfaces';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import {ErrorMatcher} from './error-matcher';
@ -46,14 +46,15 @@ export async function execErrorCatcher(commandLine: string, args?: string[],
// we capture the original return code or error so that if no match is found we can duplicate the behavior
let returnState: Error|number;
try {
returnState = await exec.exec(
returnState = await new toolrunnner.ToolRunner(
commandLine,
args,
{
...options, // pass original options first in order to override below
listeners: listeners,
ignoreReturnCode: true, // so we can check for specific codes using the matchers
});
}
).exec();
} catch (e) {
returnState = e;
}

10
src/external-queries.test.ts
View file

@ -3,6 +3,7 @@ import * as fs from "fs";
import * as path from "path";
import * as externalQueries from "./external-queries";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from "./util";
@ -10,12 +11,15 @@ setupTests(test);
test("checkoutExternalQueries", async t => {
await util.withTmpDir(async tmpDir => {
const ref = "df4c6869212341b601005567381944ed90906b6b";
await externalQueries.checkoutExternalRepository(
"github/codeql-go",
"df4c6869212341b601005567381944ed90906b6b",
tmpDir);
ref,
'https://github.com',
tmpDir,
getRunnerLogger(true));
// COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in the default branch
t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", "COPYRIGHT")));
t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", ref, "COPYRIGHT")));
});
});

31
src/external-queries.ts
View file

@ -1,23 +1,36 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from 'fs';
import * as path from 'path';
import { Logger } from './logging';
/**
* Check out repository at the given ref, and return the directory of the checkout.
*/
export async function checkoutExternalRepository(repository: string, ref: string, tempDir: string): Promise<string> {
core.info('Checking out ' + repository);
export async function checkoutExternalRepository(
repository: string,
ref: string,
githubUrl: string,
tempDir: string,
logger: Logger): Promise<string> {
logger.info('Checking out ' + repository);
const checkoutLocation = path.join(tempDir, repository, ref);
if (!checkoutLocation.startsWith(tempDir)) {
// this still permits locations that mess with sibling repositories in `tempDir`, but that is acceptable
throw new Error(`'${repository}@${ref}' is not a valid repository and reference.`);
}
const checkoutLocation = path.join(tempDir, repository);
if (!fs.existsSync(checkoutLocation)) {
const repoURL = 'https://github.com/' + repository + '.git';
await exec.exec('git', ['clone', repoURL, checkoutLocation]);
await exec.exec('git', [
const repoURL = githubUrl + '/' + repository + '.git';
await new toolrunnner.ToolRunner('git', ['clone', repoURL, checkoutLocation]).exec();
await new toolrunnner.ToolRunner('git', [
'--work-tree=' + checkoutLocation,
'--git-dir=' + checkoutLocation + '/.git',
'checkout', ref,
]);
]).exec();
}
return checkoutLocation;

179
src/finalize-db.ts
View file

@ -1,179 +0,0 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import * as path from 'path';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no langauge failed)
analyze_failure_language?: string;
}
interface FinishStatusReport extends util.StatusReportBase, upload_lib.UploadStatusReport, QueriesStatusReport {}
async function sendStatusReport(
startedAt: Date,
queriesStats: QueriesStatusReport | undefined,
uploadStats: upload_lib.UploadStatusReport | undefined,
error?: Error) {
const status = queriesStats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase('finish', status, startedAt, error?.message, error?.stack);
const statusReport: FinishStatusReport = {
...statusReportBase,
...(queriesStats || {}),
...(uploadStats || {}),
};
await util.sendStatusReport(statusReport);
}
async function createdDBForScannedLanguages(databaseFolder: string, config: configUtils.Config) {
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
core.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(databaseFolder: string, config: configUtils.Config) {
await createdDBForScannedLanguages(databaseFolder, config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
core.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(path.join(databaseFolder, language));
core.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
databaseFolder: string,
sarifFolder: string,
config: configUtils.Config): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of fs.readdirSync(databaseFolder)) {
core.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = path.join(databaseFolder, language + '-queries.qls');
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(path.join(databaseFolder, language), sarifFile, querySuite);
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
core.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
analyze_failure_language: language,
};
}
}
return {};
}
async function run() {
const startedAt = new Date();
let queriesStats: QueriesStatusReport | undefined = undefined;
let uploadStats: upload_lib.UploadStatusReport | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
return;
}
const config = await configUtils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
const sarifFolder = core.getInput('output');
fs.mkdirSync(sarifFolder, { recursive: true });
core.info('Finalizing database creation');
await finalizeDatabaseCreation(databaseFolder, config);
core.info('Analyzing database');
queriesStats = await runQueries(databaseFolder, sarifFolder, config);
if ('true' === core.getInput('upload')) {
uploadStats = await upload_lib.upload(
sarifFolder,
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
'actions',
getActionsLogger());
}
} catch (error) {
core.setFailed(error.message);
console.log(error);
await sendStatusReport(startedAt, queriesStats, uploadStats, error);
return;
}
await sendStatusReport(startedAt, queriesStats, uploadStats);
}
run().catch(e => {
core.setFailed("analyze action failed: " + e);
console.log(e);
});

14
src/fingerprints.test.ts
View file

@ -4,7 +4,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as fingerprints from './fingerprints';
import { getCLILogger } from './logging';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
setupTests(test);
@ -116,7 +116,7 @@ test('hash', (t: ava.Assertions) => {
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
const location = { "uri": uri, "index": index };
const artifacts = artifactsURIs.map(uri => ({ "location": { "uri": uri } }));
return fingerprints.resolveUriToFile(location, artifacts, getCLILogger());
return fingerprints.resolveUriToFile(location, artifacts, process.cwd(), getRunnerLogger(true));
}
test('resolveUriToFile', t => {
@ -129,8 +129,6 @@ test('resolveUriToFile', t => {
t.true(filepath.startsWith(cwd + '/'));
const relativeFilepaht = filepath.substring(cwd.length + 1);
process.env['GITHUB_WORKSPACE'] = cwd;
// Absolute paths are unmodified
t.is(testResolveUriToFile(filepath, undefined, []), filepath);
t.is(testResolveUriToFile('file://' + filepath, undefined, []), filepath);
@ -173,9 +171,9 @@ test('addFingerprints', t => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input, getCLILogger()), expected);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)), expected);
});
test('missingRegions', t => {
@ -188,7 +186,7 @@ test('missingRegions', t => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input, getCLILogger()), expected);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)), expected);
});

17
src/fingerprints.ts
View file

@ -161,7 +161,12 @@ function locationUpdateCallback(result: any, location: any, logger: Logger): has
// the source file so we can hash it.
// If possible returns a absolute file path for the source file,
// or if not possible then returns undefined.
export function resolveUriToFile(location: any, artifacts: any[], logger: Logger): string | undefined {
export function resolveUriToFile(
location: any,
artifacts: any[],
checkoutPath: string,
logger: Logger): string | undefined {
// This may be referencing an artifact
if (!location.uri && location.index !== undefined) {
if (typeof location.index !== 'number' ||
@ -192,7 +197,7 @@ export function resolveUriToFile(location: any, artifacts: any[], logger: Logger
}
// Discard any absolute paths that aren't in the src root
const srcRootPrefix = process.env['GITHUB_WORKSPACE'] + '/';
const srcRootPrefix = checkoutPath + '/';
if (uri.startsWith('/') && !uri.startsWith(srcRootPrefix)) {
logger.debug(`Ignoring location URI "${uri}" as it is outside of the src root`);
return undefined;
@ -216,7 +221,7 @@ export function resolveUriToFile(location: any, artifacts: any[], logger: Logger
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(sarifContents: string, logger: Logger): string {
export function addFingerprints(sarifContents: string, checkoutPath: string, logger: Logger): string {
let sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
@ -234,7 +239,11 @@ export function addFingerprints(sarifContents: string, logger: Logger): string {
continue;
}
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, logger);
const filepath = resolveUriToFile(
primaryLocation.physicalLocation.artifactLocation,
artifacts,
checkoutPath,
logger);
if (!filepath) {
continue;
}

124
src/init-action.ts Normal file
View file

@ -0,0 +1,124 @@
import * as core from '@actions/core';
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { initCodeQL, initConfig, runInit } from './init';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as util from './util';
interface InitSuccessStatusReport extends util.StatusReportBase {
// Comma-separated list of languages that analysis was run for
// This may be from the workflow file or may be calculated from repository contents
languages: string;
// Comma-separated list of languages specified explicitly in the workflow file
workflow_languages: string;
// Comma-separated list of paths, from the 'paths' config field
paths: string;
// Comma-separated list of paths, from the 'paths-ignore' config field
paths_ignore: string;
// Commas-separated list of languages where the default queries are disabled
disable_default_queries: string;
// Comma-separated list of queries sources, from the 'queries' config field
queries: string;
}
async function sendSuccessStatusReport(startedAt: Date, config: configUtils.Config) {
const statusReportBase = await util.createStatusReportBase('init', 'success', startedAt);
const languages = config.languages.join(',');
const workflowLanguages = core.getInput('languages', { required: false });
const paths = (config.originalUserInput.paths || []).join(',');
const pathsIgnore = (config.originalUserInput['paths-ignore'] || []).join(',');
const disableDefaultQueries = config.originalUserInput['disable-default-queries'] ? languages : '';
const queries = (config.originalUserInput.queries || []).map(q => q.uses).join(',');
const statusReport: InitSuccessStatusReport = {
...statusReportBase,
languages: languages,
workflow_languages: workflowLanguages,
paths: paths,
paths_ignore: pathsIgnore,
disable_default_queries: disableDefaultQueries,
queries: queries,
};
await util.sendStatusReport(statusReport);
}
async function run() {
const startedAt = new Date();
const logger = getActionsLogger();
let config: configUtils.Config;
let codeql: CodeQL;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('init', 'starting', startedAt), true)) {
return;
}
codeql = await initCodeQL(
core.getInput('tools'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
'actions',
logger);
config = await initConfig(
core.getInput('languages'),
core.getInput('queries'),
core.getInput('config-file'),
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql,
util.getRequiredEnvParam('GITHUB_WORKSPACE'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
logger);
} catch (e) {
core.setFailed(e.message);
console.log(e);
await util.sendStatusReport(await util.createStatusReportBase('init', 'aborted', startedAt, e.message));
return;
}
try {
// Forward Go flags
const goFlags = process.env['GOFLAGS'];
if (goFlags) {
core.exportVariable('GOFLAGS', goFlags);
core.warning("Passing the GOFLAGS env parameter to the init action is deprecated. Please move this to the analyze action.");
}
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig !== undefined) {
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
}
} catch (error) {
core.setFailed(error.message);
console.log(error);
await util.sendStatusReport(await util.createStatusReportBase(
'init',
'failure',
startedAt,
error.message,
error.stack));
return;
}
await sendSuccessStatusReport(startedAt, config);
}
run().catch(e => {
core.setFailed("init action failed: " + e);
console.log(e);
});

104
src/init.ts Normal file
View file

@ -0,0 +1,104 @@
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import { getCombinedTracerConfig, TracerConfig } from './tracer-config';
import * as util from './util';
export async function initCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
logger.startGroup('Setup CodeQL tools');
const codeql = await setupCodeQL(
codeqlURL,
githubAuth,
githubUrl,
tempDir,
toolsDir,
mode,
logger);
await codeql.printVersion();
logger.endGroup();
return codeql;
}
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<configUtils.Config> {
logger.startGroup('Load language configuration');
const config = await configUtils.initConfig(
languagesInput,
queriesInput,
configFile,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
analysisPaths.printPathFiltersWarning(config, logger);
logger.endGroup();
return config;
}
export async function runInit(
codeql: CodeQL,
config: configUtils.Config): Promise<TracerConfig | undefined> {
const sourceRoot = path.resolve();
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
// Init language database
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
const tracerConfig = await getCombinedTracerConfig(config, codeql);
if (tracerConfig !== undefined && process.platform === 'win32') {
const injectTracerPath = path.join(config.tempDir, 'inject-tracer.ps1');
fs.writeFileSync(injectTracerPath, `
Param(
[Parameter(Position=0)]
[String]
$tracer
)
Get-Process -Name Runner.Worker
$process=Get-Process -Name Runner.Worker
$id=$process.Id
Invoke-Expression "&$tracer --inject=$id"`);
await new toolrunnner.ToolRunner(
'powershell',
[
injectTracerPath,
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } }).exec();
}
return tracerConfig;
}

9
src/inject-tracer.ps1
View file

@ -1,9 +0,0 @@
Param(
[Parameter(Position=0)]
[String]
$tracer
)
Get-Process -Name Runner.Worker
$process=Get-Process -Name Runner.Worker
$id=$process.Id
Invoke-Expression "&$tracer --inject=$id"

4
src/logging.ts
View file

@ -14,9 +14,9 @@ export function getActionsLogger(): Logger {
return core;
}
export function getCLILogger(): Logger {
export function getRunnerLogger(debugMode: boolean): Logger {
return {
debug: console.debug,
debug: debugMode ? console.debug : () => undefined,
info: console.info,
warning: console.warn,
error: console.error,

346
src/runner.ts Normal file
View file

@ -0,0 +1,346 @@
import { Command } from 'commander';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { runAnalyze } from './analyze';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import { CodeQL, getCodeQL } from './codeql';
import { Config, getConfig } from './config-utils';
import { initCodeQL, initConfig, runInit } from './init';
import { Language, parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import { getMemoryFlag, getThreadsFlag } from './util';
const program = new Command();
program.version('0.0.1');
function parseGithubUrl(inputUrl: string): string {
try {
const url = new URL(inputUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://github.com';
}
// Remove the API prefix if it's present
if (url.pathname.indexOf('/api/v3') !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf('/api/v3'));
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
function getTempDir(userInput: string | undefined): string {
const tempDir = path.join(userInput || process.cwd(), 'codeql-runner');
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
return tempDir;
}
function getToolsDir(userInput: string | undefined): string {
const toolsDir = userInput || path.join(os.homedir(), 'codeql-runner-tools');
if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true });
}
return toolsDir;
}
const codeqlEnvJsonFilename = 'codeql-env.json';
// Imports the environment from codeqlEnvJsonFilename if not already present
function importTracerEnvironment(config: Config) {
if (!('ODASA_TRACER_CONFIGURATION' in process.env)) {
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString('utf-8'));
Object.keys(env).forEach(key => process.env[key] = env[key]);
}
}
// Allow the user to specify refs in full refs/heads/branch format
// or just the short branch name and prepend "refs/heads/" to it.
function parseRef(userInput: string): string {
if (userInput.startsWith('refs/')) {
return userInput;
} else {
return 'refs/heads/' + userInput;
}
}
interface InitArgs {
languages: string | undefined;
queries: string | undefined;
configFile: string | undefined;
codeqlPath: string | undefined;
tempDir: string | undefined;
toolsDir: string | undefined;
checkoutPath: string | undefined;
repository: string;
githubUrl: string;
githubAuth: string;
debug: boolean;
}
program
.command('init')
.description('Initializes CodeQL')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--languages <languages>', 'Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo.')
.option('--queries <queries>', 'Comma-separated list of additional queries to run. This overrides the same setting in a configuration file.')
.option('--config-file <file>', 'Path to config file.')
.option('--codeql-path <path>', 'Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--tools-dir <dir>', 'Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory.')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: InitArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const tempDir = getTempDir(cmd.tempDir);
const toolsDir = getToolsDir(cmd.toolsDir);
// Wipe the temp dir
logger.info(`Cleaning temp directory ${tempDir}`);
fs.rmdirSync(tempDir, { recursive: true });
fs.mkdirSync(tempDir, { recursive: true });
let codeql: CodeQL;
if (cmd.codeqlPath !== undefined) {
codeql = getCodeQL(cmd.codeqlPath);
} else {
codeql = await initCodeQL(
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
tempDir,
toolsDir,
'runner',
logger);
}
const config = await initConfig(
cmd.languages,
cmd.queries,
cmd.configFile,
parseRepositoryNwo(cmd.repository),
tempDir,
toolsDir,
codeql,
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
logger);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig === undefined) {
return;
}
// Always output a json file of the env that can be consumed programatically
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
fs.writeFileSync(jsonEnvFile, JSON.stringify(tracerConfig.env));
if (process.platform === 'win32') {
const batEnvFile = path.join(config.tempDir, 'codeql-env.bat');
const batEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `Set ${key}=${value}`)
.join('\n');
fs.writeFileSync(batEnvFile, batEnvFileContents);
const powershellEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const powershellEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `$env:${key}="${value}"`)
.join('\n');
fs.writeFileSync(powershellEnvFile, powershellEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`);
} else {
// Assume that anything that's not windows is using a unix-style shell
const shEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const shEnvFileContents = Object.entries(tracerConfig.env)
// Some vars contain ${LIB} that we do not want to be expanded when executing this script
.map(([key, value]) => `export ${key}="${value.replace('$', '\\$')}"`)
.join('\n');
fs.writeFileSync(shEnvFile, shEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`for example by running ". ${shEnvFile}".`);
}
} catch (e) {
logger.error('Init failed');
logger.error(e);
process.exitCode = 1;
}
});
interface AutobuildArgs {
language: string;
tempDir: string | undefined;
debug: boolean;
}
program
.command('autobuild')
.description('Attempts to automatically build code')
.option('--language <language>', 'The language to build. Otherwise will detect the dominant compiled language.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: AutobuildArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
}
importTracerEnvironment(config);
let language: Language | undefined = undefined;
if (cmd.language !== undefined) {
language = parseLanguage(cmd.language);
if (language === undefined || !config.languages.includes(language)) {
throw new Error(`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(', ')}.`);
}
} else {
language = determineAutobuildLanguage(config, logger);
}
if (language !== undefined) {
await runAutobuild(language, config, logger);
}
} catch (e) {
logger.error('Autobuild failed');
logger.error(e);
process.exitCode = 1;
}
});
interface AnalyzeArgs {
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
upload: boolean;
outputDir: string | undefined;
ram: string | undefined;
threads: string | undefined;
tempDir: string | undefined;
debug: boolean;
}
program
.command('analyze')
.description('Finishes extracting code and runs CodeQL queries')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--no-upload', 'Do not upload results after analysis.', false)
.option('--output-dir <dir>', 'Directory to output SARIF files to. Default is in the temp directory.')
.option('--ram <ram>', 'Amount of memory to use when running queries. Default is to use all available memory.')
.option('--threads <threads>', 'Number of threads to use when running queries. ' +
'Default is to use all available cores.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: AnalyzeArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const tempDir = getTempDir(cmd.tempDir);
const outputDir = cmd.outputDir || path.join(tempDir, 'codeql-sarif');
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
}
await runAnalyze(
parseRepositoryNwo(cmd.repository),
cmd.commit,
parseRef(cmd.ref),
undefined,
undefined,
undefined,
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
cmd.upload,
'runner',
outputDir,
getMemoryFlag(cmd.ram),
getThreadsFlag(cmd.threads, logger),
config,
logger);
} catch (e) {
logger.error('Analyze failed');
logger.error(e);
process.exitCode = 1;
}
});
interface UploadArgs {
sarifFile: string;
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
debug: boolean;
}
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload, or a directory containing multiple SARIF files. (Required)')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: UploadArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
await upload_lib.upload(
cmd.sarifFile,
parseRepositoryNwo(cmd.repository),
cmd.commit,
parseRef(cmd.ref),
undefined,
undefined,
undefined,
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
'runner',
logger);
} catch (e) {
logger.error('Upload failed');
logger.error(e);
process.exitCode = 1;
}
});
program.parse(process.argv);

277
src/setup-tracer.ts
View file

@ -1,277 +0,0 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isTracedLanguage } from './languages';
import * as util from './util';
type TracerConfig = {
spec: string;
env: { [key: string]: string };
};
const CRITICAL_TRACER_VARS = new Set(
['SEMMLE_PRELOAD_libtrace',
, 'SEMMLE_RUNNER',
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
, 'SEMMLE_DEPTRACE_SOCKET',
, 'SEMMLE_JAVA_TOOL_OPTIONS'
]);
async function tracerConfig(
codeql: CodeQL,
database: string,
compilerSpec?: string): Promise<TracerConfig> {
const env = await codeql.getTracerEnv(database, compilerSpec);
const config = env['ODASA_TRACER_CONFIGURATION'];
const info: TracerConfig = { spec: config, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
info.env[key] = value;
}
}
return info;
}
function concatTracerConfigs(tracerConfigs: TracerConfig[], config: configUtils.Config): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env: { [key: string]: string; } = {};
let copyExecutables = false;
let envSize = 0;
for (const v of tracerConfigs) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
copyExecutables = true;
} else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
}
} else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines: string[] = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
fs.writeFileSync(envPath, buffer);
return { env, spec };
}
interface InitSuccessStatusReport extends util.StatusReportBase {
// Comma-separated list of languages that analysis was run for
// This may be from the workflow file or may be calculated from repository contents
languages: string;
// Comma-separated list of languages specified explicitly in the workflow file
workflow_languages: string;
// Comma-separated list of paths, from the 'paths' config field
paths: string;
// Comma-separated list of paths, from the 'paths-ignore' config field
paths_ignore: string;
// Commas-separated list of languages where the default queries are disabled
disable_default_queries: string;
// Comma-separated list of queries sources, from the 'queries' config field
queries: string;
}
async function sendSuccessStatusReport(startedAt: Date, config: configUtils.Config) {
const statusReportBase = await util.createStatusReportBase('init', 'success', startedAt);
const languages = config.languages.join(',');
const workflowLanguages = core.getInput('languages', { required: false });
const paths = (config.originalUserInput.paths || []).join(',');
const pathsIgnore = (config.originalUserInput['paths-ignore'] || []).join(',');
const disableDefaultQueries = config.originalUserInput['disable-default-queries'] ? languages : '';
const queries = (config.originalUserInput.queries || []).map(q => q.uses).join(',');
const statusReport: InitSuccessStatusReport = {
...statusReportBase,
languages: languages,
workflow_languages: workflowLanguages,
paths: paths,
paths_ignore: pathsIgnore,
disable_default_queries: disableDefaultQueries,
queries: queries,
};
await util.sendStatusReport(statusReport);
}
async function run() {
const startedAt = new Date();
let config: configUtils.Config;
let codeql: CodeQL;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('init', 'starting', startedAt), true)) {
return;
}
core.startGroup('Setup CodeQL tools');
codeql = await setupCodeQL();
await codeql.printVersion();
core.endGroup();
core.startGroup('Load language configuration');
config = await configUtils.initConfig(
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql);
analysisPaths.includeAndExcludeAnalysisPaths(config);
core.endGroup();
} catch (e) {
core.setFailed(e.message);
console.log(e);
await util.sendStatusReport(await util.createStatusReportBase('init', 'aborted', startedAt, e.message));
return;
}
try {
const sourceRoot = path.resolve();
// Forward Go flags
const goFlags = process.env['GOFLAGS'];
if (goFlags) {
core.exportVariable('GOFLAGS', goFlags);
core.warning("Passing the GOFLAGS env parameter to the init action is deprecated. Please move this to the analyze action.");
}
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
fs.mkdirSync(databaseFolder, { recursive: true });
let tracedLanguageConfigs: TracerConfig[] = [];
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
const languageDatabase = path.join(databaseFolder, language);
// Init language database
await codeql.databaseInit(languageDatabase, language, sourceRoot);
// TODO: add better detection of 'traced languages' instead of using a hard coded list
if (isTracedLanguage(language)) {
const config: TracerConfig = await tracerConfig(codeql, languageDatabase);
tracedLanguageConfigs.push(config);
}
}
if (tracedLanguageConfigs.length > 0) {
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
if (mainTracerConfig.spec) {
for (let entry of Object.entries(mainTracerConfig.env)) {
core.exportVariable(entry[0], entry[1]);
}
core.exportVariable('ODASA_TRACER_CONFIGURATION', mainTracerConfig.spec);
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
core.exportVariable(
'DYLD_INSERT_LIBRARIES',
path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib'));
} else if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(codeQLDir, 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
} else {
core.exportVariable('LD_PRELOAD', path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so'));
}
}
}
} catch (error) {
core.setFailed(error.message);
console.log(error);
await util.sendStatusReport(await util.createStatusReportBase(
'init',
'failure',
startedAt,
error.message,
error.stack));
return;
}
await sendSuccessStatusReport(startedAt, config);
}
run().catch(e => {
core.setFailed("init action failed: " + e);
console.log(e);
});

4
src/testing-utils.ts
View file

@ -55,10 +55,6 @@ export function setupTests(test: TestInterface<any>) {
// process.env only has strings fields, so a shallow copy is fine.
t.context.env = {};
Object.assign(t.context.env, process.env);
// Any test that runs code that expects to only be run on actions
// will depend on various environment variables.
process.env['GITHUB_API_URL'] = 'https://github.localhost/api/v3';
});
typedTest.afterEach.always(t => {

318
src/tracer-config.test.ts Normal file
View file

@ -0,0 +1,318 @@
import test from 'ava';
import * as fs from 'fs';
import * as path from 'path';
import { setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from './languages';
import { setupTests } from './testing-utils';
import { concatTracerConfigs, getCombinedTracerConfig, getTracerConfigForLanguage } from './tracer-config';
import * as util from './util';
setupTests(test);
function getTestConfig(tmpDir: string): configUtils.Config {
return {
languages: [Language.java],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: '',
};
}
// A very minimal setup
test('getTracerConfigForLanguage - minimal setup', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
};
},
});
const result = await getTracerConfigForLanguage(codeQL, config, Language.javascript);
t.deepEqual(result, { spec: 'abc', env: {'foo': 'bar'} });
});
});
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
test('getTracerConfigForLanguage - existing / critical vars', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
// Set up some variables in the environment
process.env['foo'] = 'abc';
process.env['SEMMLE_PRELOAD_libtrace'] = 'abc';
process.env['SEMMLE_RUNNER'] = 'abc';
process.env['SEMMLE_COPY_EXECUTABLES_ROOT'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['SEMMLE_JAVA_TOOL_OPTIONS'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['CODEQL_VAR'] = 'abc';
// Now CodeQL returns all these variables, and one more, with different values
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar',
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
};
},
});
const result = await getTracerConfigForLanguage(codeQL, config, Language.javascript);
t.deepEqual(result, {
spec: 'abc',
env: {
// Should contain all variables except 'foo', because that already existed in the
// environment with a different value, and is not deemed a "critical" variable.
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
}
});
});
});
test('concatTracerConfigs - minimal configs correctly combined', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
};
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, 'utf8'),
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nabc\ndef\nghi');
});
});
test('concatTracerConfigs - conflicting env vars', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
// Ok if env vars have the same name and the same value
t.deepEqual(
concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'b': 'b', 'c': 'c'}},
},
config).env,
{
'a': 'a',
'b': 'b',
'c': 'c',
});
// Throws if env vars have same name but different values
const e = t.throws(() =>
concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'b': 'c'}},
},
config));
t.deepEqual(e.message, 'Incompatible values in environment parameter b: b and c');
});
});
test('concatTracerConfigs - cpp spec lines come last if present', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
};
const result = concatTracerConfigs({ 'cpp': tc1, 'python': tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, 'utf8'),
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nghi\nabc\ndef');
});
});
test('concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT is updated to point to compound spec', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
const result = concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'SEMMLE_COPY_EXECUTABLES_ROOT': 'foo'}},
},
config);
t.deepEqual(result.env, {
'a': 'a',
'b': 'b',
'SEMMLE_COPY_EXECUTABLES_ROOT': path.join(tmpDir, 'compound-temp')
});
});
});
test('concatTracerConfigs - compound environment file is created correctly', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'foo': 'bar_baz',
}
};
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
const envPath = result.spec + '.environment';
t.true(fs.existsSync(envPath));
const buffer: Buffer = fs.readFileSync(envPath);
// Contents is binary data
t.deepEqual(buffer.length, 28);
t.deepEqual(buffer.readInt32LE(0), 2); // number of env vars
t.deepEqual(buffer.readInt32LE(4), 4); // length of env var definition
t.deepEqual(buffer.toString('utf8', 8, 12), 'a=a\0'); // [key]=[value]\0
t.deepEqual(buffer.readInt32LE(12), 12); // length of env var definition
t.deepEqual(buffer.toString('utf8', 16, 28), 'foo=bar_baz\0'); // [key]=[value]\0
});
});
test('getCombinedTracerConfig - return undefined when no languages are traced languages', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
// No traced languages
config.languages = [Language.javascript, Language.python];
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
};
},
});
t.deepEqual(await getCombinedTracerConfig(config, codeQL), undefined);
});
});
test('getCombinedTracerConfig - valid spec file', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n2\nabc\ndef');
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': spec,
'foo': 'bar',
};
},
});
const result = await getCombinedTracerConfig(config, codeQL);
const expectedEnv = {
'foo': 'bar',
'ODASA_TRACER_CONFIGURATION': result!.spec,
};
if (process.platform === 'darwin') {
expectedEnv['DYLD_INSERT_LIBRARIES'] = path.join(path.dirname(codeQL.getPath()), 'tools', 'osx64', 'libtrace.dylib');
} else if (process.platform !== 'win32') {
expectedEnv['LD_PRELOAD'] = path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so');
}
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: expectedEnv,
});
});
});

159
src/tracer-config.ts Normal file
View file

@ -0,0 +1,159 @@
import * as fs from 'fs';
import * as path from 'path';
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isTracedLanguage, Language } from './languages';
import * as util from './util';
export type TracerConfig = {
spec: string;
env: { [key: string]: string };
};
const CRITICAL_TRACER_VARS = new Set(
['SEMMLE_PRELOAD_libtrace',
, 'SEMMLE_RUNNER',
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
, 'SEMMLE_DEPTRACE_SOCKET',
, 'SEMMLE_JAVA_TOOL_OPTIONS'
]);
export async function getTracerConfigForLanguage(
codeql: CodeQL,
config: configUtils.Config,
language: Language): Promise<TracerConfig> {
const env = await codeql.getTracerEnv(util.getCodeQLDatabasePath(config.tempDir, language));
const spec = env['ODASA_TRACER_CONFIGURATION'];
const info: TracerConfig = { spec, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
info.env[key] = value;
}
}
return info;
}
export function concatTracerConfigs(
tracerConfigs: { [lang: string]: TracerConfig },
config: configUtils.Config): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env: { [key: string]: string; } = {};
let copyExecutables = false;
let envSize = 0;
for (const v of Object.values(tracerConfigs)) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
copyExecutables = true;
} else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
}
} else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines: string[] = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
fs.writeFileSync(envPath, buffer);
return { env, spec };
}
export async function getCombinedTracerConfig(
config: configUtils.Config,
codeql: CodeQL): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter(isTracedLanguage);
if (tracedLanguages.length === 0) {
return undefined;
}
// Get all the tracer configs and combine them together
const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
for (const language of tracedLanguages) {
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(codeql, config, language);
}
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
// Add a couple more variables
mainTracerConfig.env['ODASA_TRACER_CONFIGURATION'] = mainTracerConfig.spec;
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
mainTracerConfig.env['DYLD_INSERT_LIBRARIES'] = path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib');
} else if (process.platform !== 'win32') {
mainTracerConfig.env['LD_PRELOAD'] = path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so');
}
return mainTracerConfig;
}

12
src/tracer-env.ts
View file

@ -1,12 +0,0 @@
import * as fs from 'fs';
const env = {};
for (let entry of Object.entries(process.env)) {
const key = entry[0];
const value = entry[1];
if (typeof value !== 'undefined' && key !== '_' && !key.startsWith('JAVA_MAIN_CLASS_')) {
env[key] = value;
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');

6
src/upload-lib.test.ts
View file

@ -1,6 +1,6 @@
import test from 'ava';
import { getCLILogger } from './logging';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as uploadLib from './upload-lib';
@ -8,10 +8,10 @@ setupTests(test);
test('validateSarifFileSchema - valid', t => {
const inputFile = __dirname + '/../src/testdata/valid-sarif.sarif';
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile, getCLILogger()));
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)));
});
test('validateSarifFileSchema - invalid', t => {
const inputFile = __dirname + '/../src/testdata/invalid-sarif.sarif';
t.throws(() => uploadLib.validateSarifFileSchema(inputFile, getCLILogger()));
t.throws(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)));
});

24
src/upload-lib.ts
View file

@ -12,8 +12,6 @@ import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as util from './util';
type UploadMode = 'actions' | 'cli';
// Takes a list of paths to sarif files and combines them together,
// returning the contents of the combined sarif file.
export function combineSarifFiles(sarifFiles: string[]): string {
@ -43,8 +41,8 @@ async function uploadPayload(
payload: any,
repositoryNwo: RepositoryNwo,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger) {
logger.info('Uploading results');
@ -61,7 +59,7 @@ async function uploadPayload(
// minutes, but just waiting a little bit could maybe help.
const backoffPeriods = [1, 5, 15];
const client = api.getApiClient(githubAuth, githubApiUrl);
const client = api.getApiClient(githubAuth, githubUrl);
for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
const reqURL = mode === 'actions'
@ -134,8 +132,8 @@ export async function upload(
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
const sarifFiles: string[] = [];
@ -165,7 +163,7 @@ export async function upload(
checkoutPath,
environment,
githubAuth,
githubApiUrl,
githubUrl,
mode,
logger);
}
@ -214,14 +212,14 @@ async function uploadFiles(
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
logger.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
if (mode === 'actions') {
// This check only works on actions as env vars don't persist between calls to the CLI
// This check only works on actions as env vars don't persist between calls to the runner
const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
if (process.env[sentinelEnvVar]) {
throw new Error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
@ -235,7 +233,7 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload, logger);
sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutURI = fileUrl(checkoutPath);
@ -275,7 +273,7 @@ async function uploadFiles(
logger.debug("Number of results in upload: " + numResultInSarif);
// Make the upload
await uploadPayload(payload, repositoryNwo, githubAuth, githubApiUrl, mode, logger);
await uploadPayload(payload, repositoryNwo, githubAuth, githubUrl, mode, logger);
return {
raw_upload_size_bytes: rawUploadSizeBytes,

2
src/upload-sarif.ts → src/upload-sarif-action.ts
View file

@ -34,7 +34,7 @@ async function run() {
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
'actions',
getActionsLogger());
await sendSuccessStatusReport(startedAt, uploadStats);

17
src/util.test.ts
View file

@ -2,6 +2,7 @@ import test from 'ava';
import * as fs from 'fs';
import * as os from "os";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
@ -23,18 +24,14 @@ test('getMemoryFlag() should return the correct --ram flag', t => {
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_RAM'] = input;
const flag = util.getMemoryFlag();
const flag = util.getMemoryFlag(input);
t.deepEqual(flag, expectedFlag);
}
});
test('getMemoryFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["-1", "hello!"]) {
process.env['INPUT_RAM'] = input;
t.throws(util.getMemoryFlag);
t.throws(() => util.getMemoryFlag(input));
}
});
@ -50,17 +47,13 @@ test('getThreadsFlag() should return the correct --threads flag', t => {
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_THREADS'] = input;
const flag = util.getThreadsFlag();
const flag = util.getThreadsFlag(input, getRunnerLogger(true));
t.deepEqual(flag, expectedFlag);
}
});
test('getThreadsFlag() throws if the threads input is not an integer', t => {
process.env['INPUT_THREADS'] = "hello!";
t.throws(util.getThreadsFlag);
t.throws(() => util.getThreadsFlag("hello!", getRunnerLogger(true)));
});
test('getRef() throws on the empty string', t => {

56
src/util.ts
View file

@ -1,31 +1,24 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from "fs";
import * as os from 'os';
import * as path from 'path';
import * as api from './api-client';
import { Language } from './languages';
import { Logger } from './logging';
import * as sharedEnv from './shared-environment';
/**
* The API URL for github.com.
* Are we running on actions, or not.
*/
export const GITHUB_DOTCOM_API_URL = "https://api.github.com";
export type Mode = 'actions' | 'runner';
/**
* Get the API URL for the GitHub instance we are connected to.
* May be for github.com or for an enterprise instance.
* The URL for github.com.
*/
export function getInstanceAPIURL(): string {
return process.env["GITHUB_API_URL"] || GITHUB_DOTCOM_API_URL;
}
export const GITHUB_DOTCOM_URL = "https://github.com";
/**
* Are we running against a GitHub Enterpise instance, as opposed to github.com.
*/
export function isEnterprise(): boolean {
return getInstanceAPIURL() !== GITHUB_DOTCOM_API_URL;
}
/**
* Get an environment parameter, but throw an error if it is not set.
@ -92,13 +85,13 @@ export async function getCommitOid(): Promise<string> {
// reported on the merge commit.
try {
let commitOid = '';
await exec.exec('git', ['rev-parse', 'HEAD'], {
await new toolrunnner.ToolRunner('git', ['rev-parse', 'HEAD'], {
silent: true,
listeners: {
stdout: (data) => { commitOid += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
}).exec();
return commitOid.trim();
} catch (e) {
core.info("Failed to call git to get current commit. Continuing with data from environment: " + e);
@ -296,7 +289,7 @@ export async function sendStatusReport<S extends StatusReportBase>(
statusReport: S,
ignoreFailures?: boolean): Promise<boolean> {
if (isEnterprise()) {
if (getRequiredEnvParam("GITHUB_SERVER_URL") !== GITHUB_DOTCOM_URL) {
core.debug("Not sending status report to GitHub Enterprise");
return true;
}
@ -377,13 +370,12 @@ export async function withTmpDir<T>(body: (tmpDir: string) => Promise<T>): Promi
*
* @returns string
*/
export function getMemoryFlag(): string {
export function getMemoryFlag(userInput: string | undefined): string {
let memoryToUseMegaBytes: number;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
throw new Error("Invalid RAM setting \"" + userInput + "\", specified.");
}
} else {
const totalMemoryBytes = os.totalmem();
@ -402,22 +394,21 @@ export function getMemoryFlag(): string {
*
* @returns string
*/
export function getThreadsFlag(): string {
export function getThreadsFlag(userInput: string | undefined, logger: Logger): string {
let numThreads: number;
const numThreadsString = core.getInput("threads");
const maxThreads = os.cpus().length;
if (numThreadsString) {
numThreads = Number(numThreadsString);
if (userInput) {
numThreads = Number(userInput);
if (Number.isNaN(numThreads)) {
throw new Error(`Invalid threads setting "${numThreadsString}", specified.`);
throw new Error(`Invalid threads setting "${userInput}", specified.`);
}
if (numThreads > maxThreads) {
core.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
logger.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
numThreads = maxThreads;
}
const minThreads = -maxThreads;
if (numThreads < minThreads) {
core.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
logger.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
numThreads = minThreads;
}
} else {
@ -433,3 +424,10 @@ export function getThreadsFlag(): string {
export function getCodeQLDatabasesDir(tempDir: string) {
return path.resolve(tempDir, 'codeql_databases');
}
/**
* Get the path where the CodeQL database for the given language lives.
*/
export function getCodeQLDatabasePath(tempDir: string, language: Language) {
return path.resolve(getCodeQLDatabasesDir(tempDir), language);
}