robojerk/codeql-action

Merge branch 'main' into allow-additive-queries-in-workflow

Browse source
This commit is contained in:
Sam Partington 2020-09-08 10:00:16 +01:00
commit d677f16692
83 changed files with 2222 additions and 1242 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/analysis-paths.ts
View file

@ -1,6 +1,5 @@
import * as core from '@actions/core';
import * as configUtils from './config-utils';
import { Logger } from './logging';
function isInterpretedLanguage(language): boolean {
return language === 'javascript' || language === 'python';
@ -22,6 +21,17 @@ function buildIncludeExcludeEnvVar(paths: string[]): string {
return paths.join('\n');
}
export function printPathFiltersWarning(config: configUtils.Config, logger: Logger) {
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
logger.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
}
}
export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// The 'LGTM_INDEX_INCLUDE' and 'LGTM_INDEX_EXCLUDE' environment variables
// control which files/directories are traversed when scanning.
@ -31,10 +41,10 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// traverse the entire file tree to determine which files are matched.
// Any paths containing "*" are not included in these.
if (config.paths.length !== 0) {
core.exportVariable('LGTM_INDEX_INCLUDE', buildIncludeExcludeEnvVar(config.paths));
process.env['LGTM_INDEX_INCLUDE'] = buildIncludeExcludeEnvVar(config.paths);
}
if (config.pathsIgnore.length !== 0) {
core.exportVariable('LGTM_INDEX_EXCLUDE', buildIncludeExcludeEnvVar(config.pathsIgnore));
process.env['LGTM_INDEX_EXCLUDE'] = buildIncludeExcludeEnvVar(config.pathsIgnore);
}
// The 'LGTM_INDEX_FILTERS' environment variable controls which files are
@ -44,15 +54,6 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
filters.push(...config.paths.map(p => 'include:' + p));
filters.push(...config.pathsIgnore.map(p => 'exclude:' + p));
if (filters.length !== 0) {
core.exportVariable('LGTM_INDEX_FILTERS', filters.join('\n'));
}
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0 ||
filters.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
core.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
process.env['LGTM_INDEX_FILTERS'] = filters.join('\n');
}
}

169
src/analyze-action.ts
View file

@ -1,174 +1,67 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import * as path from 'path';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { AnalysisStatusReport, runAnalyze } from './analyze';
import { getConfig } from './config-utils';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no langauge failed)
analyze_failure_language?: string;
}
interface FinishStatusReport extends util.StatusReportBase, upload_lib.UploadStatusReport, QueriesStatusReport {}
interface FinishStatusReport extends util.StatusReportBase, AnalysisStatusReport {}
async function sendStatusReport(
startedAt: Date,
queriesStats: QueriesStatusReport | undefined,
uploadStats: upload_lib.UploadStatusReport | undefined,
stats: AnalysisStatusReport | undefined,
error?: Error) {
const status = queriesStats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const status = stats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase('finish', status, startedAt, error?.message, error?.stack);
const statusReport: FinishStatusReport = {
...statusReportBase,
...(queriesStats || {}),
...(uploadStats || {}),
...(stats || {}),
};
await util.sendStatusReport(statusReport);
}
async function createdDBForScannedLanguages(config: configUtils.Config) {
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
core.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(config: configUtils.Config) {
await createdDBForScannedLanguages(config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
core.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
core.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
sarifFolder: string,
config: configUtils.Config): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of config.languages) {
core.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
core.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
analyze_failure_language: language,
};
}
}
return {};
}
async function run() {
const startedAt = new Date();
let queriesStats: QueriesStatusReport | undefined = undefined;
let uploadStats: upload_lib.UploadStatusReport | undefined = undefined;
let stats: AnalysisStatusReport | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
return;
}
const config = await configUtils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const sarifFolder = core.getInput('output');
fs.mkdirSync(sarifFolder, { recursive: true });
core.info('Finalizing database creation');
await finalizeDatabaseCreation(config);
core.info('Analyzing database');
queriesStats = await runQueries(sarifFolder, config);
if ('true' === core.getInput('upload')) {
uploadStats = await upload_lib.upload(
sarifFolder,
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
'actions',
getActionsLogger());
const logger = getActionsLogger();
const config = await getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
}
stats = await runAnalyze(
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
core.getInput('upload') === 'true',
'actions',
core.getInput('output'),
util.getMemoryFlag(core.getInput('ram')),
util.getThreadsFlag(core.getInput('threads'), logger),
config,
logger);
} catch (error) {
core.setFailed(error.message);
console.log(error);
await sendStatusReport(startedAt, queriesStats, uploadStats, error);
await sendStatusReport(startedAt, stats, error);
return;
}
await sendStatusReport(startedAt, queriesStats, uploadStats);
await sendStatusReport(startedAt, stats);
}
run().catch(e => {

172
src/analyze.ts Normal file
View file

@ -0,0 +1,172 @@
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
export interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no langauge failed)
analyze_failure_language?: string;
}
export interface AnalysisStatusReport extends upload_lib.UploadStatusReport, QueriesStatusReport {}
async function createdDBForScannedLanguages(
config: configUtils.Config,
logger: Logger) {
// Insert the LGTM_INDEX_X env vars at this point so they are set when
// we extract any scanned languages.
analysisPaths.includeAndExcludeAnalysisPaths(config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
logger.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
logger.endGroup();
}
}
}
async function finalizeDatabaseCreation(
config: configUtils.Config,
logger: Logger) {
await createdDBForScannedLanguages(config, logger);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
logger.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
logger.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
sarifFolder: string,
memoryFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of config.languages) {
logger.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
logger.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, threadsFlag);
logger.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
logger.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
analyze_failure_language: language,
};
}
}
return {};
}
export async function runAnalyze(
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string | undefined,
analysisName: string | undefined,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubUrl: string,
doUpload: boolean,
mode: util.Mode,
outputDir: string,
memoryFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<AnalysisStatusReport> {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
logger.info('Finalizing database creation');
await finalizeDatabaseCreation(config, logger);
logger.info('Analyzing database');
const queriesStats = await runQueries(outputDir, memoryFlag, threadsFlag, config, logger);
if (!doUpload) {
logger.info('Not uploading results');
return { ...queriesStats };
}
const uploadStats = await upload_lib.upload(
outputDir,
repositoryNwo,
commitOid,
ref,
analysisKey,
analysisName,
workflowRunID,
checkoutPath,
environment,
githubAuth,
githubUrl,
mode,
logger);
return { ...queriesStats, ...uploadStats };
}

28
src/api-client.ts
View file

@ -1,33 +1,35 @@
import * as core from "@actions/core";
import * as github from "@actions/github";
import consoleLogLevel from "console-log-level";
import * as path from 'path';
import { getRequiredEnvParam, isLocalRun } from "./util";
export const getApiClient = function(githubAuth: string, githubApiUrl: string, allowLocalRun = false) {
export const getApiClient = function(githubAuth: string, githubUrl: string, allowLocalRun = false) {
if (isLocalRun() && !allowLocalRun) {
throw new Error('Invalid API call in local run');
}
return new github.GitHub(
{
auth: parseAuth(githubAuth),
baseUrl: githubApiUrl,
auth: githubAuth,
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
log: consoleLogLevel({ level: "debug" })
});
};
// Parses the user input as either a single token,
// or a username and password / PAT.
function parseAuth(auth: string): string {
// Check if it's a username:password pair
const c = auth.indexOf(':');
if (c !== -1) {
return 'basic ' + Buffer.from(auth).toString('base64');
function getApiUrl(githubUrl: string): string {
const url = new URL(githubUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
}
// Otherwise use the token as it is
return auth;
// Add the /api/v3 API prefix
url.pathname = path.join(url.pathname, 'api', 'v3');
return url.toString();
}
// Temporary function to aid in the transition to running on and off of github actions.
@ -36,6 +38,6 @@ function parseAuth(auth: string): string {
export function getActionsApiClient(allowLocalRun = false) {
return getApiClient(
core.getInput('token'),
getRequiredEnvParam('GITHUB_API_URL'),
getRequiredEnvParam('GITHUB_SERVER_URL'),
allowLocalRun);
}

41
src/autobuild-action.ts
View file

@ -1,8 +1,9 @@
import * as core from '@actions/core';
import { getCodeQL } from './codeql';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import * as config_utils from './config-utils';
import { isTracedLanguage } from './languages';
import { Language } from './languages';
import { getActionsLogger } from './logging';
import * as util from './util';
interface AutobuildStatusReport extends util.StatusReportBase {
@ -34,48 +35,32 @@ async function sendCompletedStatusReport(
}
async function run() {
const logger = getActionsLogger();
const startedAt = new Date();
let language;
let language: Language | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('autobuild', 'starting', startedAt), true)) {
return;
}
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = config.languages.filter(isTracedLanguage);
language = autobuildLanguages[0];
if (!language) {
core.info("None of the languages in this project require extra build steps");
return;
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
}
core.debug(`Detected dominant traced language: ${language}`);
if (autobuildLanguages.length > 1) {
core.warning(`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages.slice(1).join(' and ')}, you must replace this block with custom build steps.`);
language = determineAutobuildLanguage(config, logger);
if (language !== undefined) {
await runAutobuild(language, config, logger);
}
core.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = getCodeQL(config.codeQLCmd);
await codeQL.runAutobuild(language);
core.endGroup();
} catch (error) {
core.setFailed("We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. " + error.message);
console.log(error);
await sendCompletedStatusReport(startedAt, [language], language, error);
await sendCompletedStatusReport(startedAt, language ? [language] : [], language, error);
return;
}
await sendCompletedStatusReport(startedAt, [language]);
await sendCompletedStatusReport(startedAt, language ? [language] : []);
}
run().catch(e => {

41
src/autobuild.ts Normal file
View file

@ -0,0 +1,41 @@
import { getCodeQL } from './codeql';
import * as config_utils from './config-utils';
import { isTracedLanguage, Language } from './languages';
import { Logger } from './logging';
export function determineAutobuildLanguage(
config: config_utils.Config,
logger: Logger
): Language | undefined {
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = config.languages.filter(isTracedLanguage);
const language = autobuildLanguages[0];
if (!language) {
logger.info("None of the languages in this project require extra build steps");
return undefined;
}
logger.debug(`Detected dominant traced language: ${language}`);
if (autobuildLanguages.length > 1) {
logger.warning(`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages.slice(1).join(' and ')}, you must replace this call with custom build steps.`);
}
return language;
}
export async function runAutobuild(
language: Language,
config: config_utils.Config,
logger: Logger) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = getCodeQL(config.codeQLCmd);
await codeQL.runAutobuild(language);
logger.endGroup();
}

21
src/codeql.test.ts
View file

@ -4,6 +4,7 @@ import nock from 'nock';
import * as path from 'path';
import * as codeql from './codeql';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
@ -12,12 +13,6 @@ setupTests(test);
test('download codeql bundle cache', async t => {
await util.withTmpDir(async tmpDir => {
process.env['GITHUB_WORKSPACE'] = tmpDir;
process.env['RUNNER_TEMP'] = path.join(tmpDir, 'temp');
process.env['RUNNER_TOOL_CACHE'] = path.join(tmpDir, 'cache');
const versions = ['20200601', '20200610'];
for (let i = 0; i < versions.length; i++) {
@ -27,10 +22,14 @@ test('download codeql bundle cache', async t => {
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(200, path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`));
process.env['INPUT_TOOLS'] = `https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
await codeql.setupCodeQL();
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
'token',
'https://github.example.com',
tmpDir,
tmpDir,
'runner',
getRunnerLogger(true));
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
}
@ -56,7 +55,7 @@ test('parse codeql bundle url version', t => {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = codeql.getCodeQLURLVersion(url);
const parsedVersion = codeql.getCodeQLURLVersion(url, getRunnerLogger(true));
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e.message);

165
src/codeql.ts
View file

@ -1,5 +1,4 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as http from '@actions/http-client';
import { IHeaders } from '@actions/http-client/interfaces';
import * as toolcache from '@actions/tool-cache';
@ -13,6 +12,7 @@ import uuidV4 from 'uuid/v4';
import * as api from './api-client';
import * as defaults from './defaults.json'; // Referenced from codeql-action-sync-tool!
import { Language } from './languages';
import { Logger } from './logging';
import * as util from './util';
type Options = (string|number|boolean)[];
@ -74,7 +74,12 @@ export interface CodeQL {
/**
* Run 'codeql database analyze'.
*/
databaseAnalyze(databasePath: string, sarifFile: string, querySuite: string): Promise<void>;
databaseAnalyze(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
threadsFlag: string): Promise<void>;
}
export interface ResolveQueriesOutput {
@ -101,7 +106,11 @@ const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
const CODEQL_BUNDLE_NAME = "codeql-bundle.tar.gz";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
function getCodeQLActionRepository(): string {
function getCodeQLActionRepository(mode: util.Mode): string {
if (mode !== 'actions') {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
// Actions do not know their own repository name,
// so we currently use this hack to find the name based on where our files are.
// This can be removed once the change to the runner in https://github.com/actions/runner/pull/585 is deployed.
@ -117,15 +126,20 @@ function getCodeQLActionRepository(): string {
return relativeScriptPathParts[0] + "/" + relativeScriptPathParts[1];
}
async function getCodeQLBundleDownloadURL(): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository();
async function getCodeQLBundleDownloadURL(
githubAuth: string,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(mode);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[util.getInstanceAPIURL(), codeQLActionRepository],
[githubUrl, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
[util.getInstanceAPIURL(), CODEQL_DEFAULT_ACTION_REPOSITORY],
[githubUrl, CODEQL_DEFAULT_ACTION_REPOSITORY],
// GitHub.com, and the canonical Action.
[util.GITHUB_DOTCOM_API_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
[util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
@ -133,24 +147,24 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
for (let downloadSource of uniqueDownloadSources) {
let [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (apiURL === util.GITHUB_DOTCOM_API_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
if (apiURL === util.GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
break;
}
let [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getActionsApiClient().repos.getReleaseByTag({
const release = await api.getApiClient(githubAuth, githubUrl).repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION
});
for (let asset of release.data.assets) {
if (asset.name === CODEQL_BUNDLE_NAME) {
core.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
logger.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
return asset.url;
}
}
} catch (e) {
core.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
logger.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${CODEQL_BUNDLE_NAME}`;
@ -158,16 +172,18 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
// We have to download CodeQL manually because the toolcache doesn't support Accept headers.
// This can be removed once https://github.com/actions/toolkit/pull/530 is merged and released.
async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<string> {
async function toolcacheDownloadTool(
url: string,
headers: IHeaders | undefined,
tempDir: string,
logger: Logger): Promise<string> {
const client = new http.HttpClient('CodeQL Action');
const dest = path.join(util.getRequiredEnvParam('RUNNER_TEMP'), uuidV4());
const dest = path.join(tempDir, uuidV4());
const response: http.HttpClientResponse = await client.get(url, headers);
if (response.message.statusCode !== 200) {
const err = new toolcache.HTTPError(response.message.statusCode);
core.info(
`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`
);
throw err;
logger.info(`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`);
throw new Error(`Unexpected HTTP response: ${response.message.statusCode}`);
}
const pipeline = globalutil.promisify(stream.pipeline);
fs.mkdirSync(path.dirname(dest), { recursive: true });
@ -175,32 +191,45 @@ async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<s
return dest;
}
export async function setupCodeQL(): Promise<CodeQL> {
export async function setupCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
// Setting these two env vars makes the toolcache code safe to use outside,
// of actions but this is obviously not a great thing we're doing and it would
// be better to write our own implementation to use outside of actions.
process.env['RUNNER_TEMP'] = tempDir;
process.env['RUNNER_TOOL_CACHE'] = toolsDir;
try {
let codeqlURL = core.getInput('tools');
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`, logger);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL();
codeqlURL = await getCodeQLBundleDownloadURL(githubAuth, githubUrl, mode, logger);
}
const headers: IHeaders = {accept: 'application/octet-stream'};
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
if (codeqlURL.startsWith(util.getInstanceAPIURL() + "/")) {
core.debug('Downloading CodeQL bundle with token.');
let token = core.getInput('token', { required: true });
headers.authorization = `token ${token}`;
if (codeqlURL.startsWith(githubUrl + "/")) {
logger.debug('Downloading CodeQL bundle with token.');
headers.authorization = `token ${githubAuth}`;
} else {
core.debug('Downloading CodeQL bundle without token.');
logger.debug('Downloading CodeQL bundle without token.');
}
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers);
core.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers, tempDir, logger);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
@ -217,12 +246,12 @@ export async function setupCodeQL(): Promise<CodeQL> {
return cachedCodeQL;
} catch (e) {
core.error(e);
logger.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string): string {
export function getCodeQLURLVersion(url: string, logger: Logger): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
@ -232,7 +261,7 @@ export function getCodeQLURLVersion(url: string): string {
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}
@ -311,33 +340,49 @@ function getCodeQLForCmd(cmd: string): CodeQL {
return cmd;
},
printVersion: async function() {
await exec.exec(cmd, [
await new toolrunnner.ToolRunner(cmd, [
'version',
'--format=json'
]);
]).exec();
},
getTracerEnv: async function(databasePath: string) {
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
await exec.exec(cmd, [
// Write tracer-env.js to a temp location.
const tracerEnvJs = path.resolve(databasePath, 'working', 'tracer-env.js');
fs.mkdirSync(path.dirname(tracerEnvJs), {recursive: true});
fs.writeFileSync(tracerEnvJs, `
const fs = require('fs');
const env = {};
for (let entry of Object.entries(process.env)) {
const key = entry[0];
const value = entry[1];
if (typeof value !== 'undefined' && key !== '_' && !key.startsWith('JAVA_MAIN_CLASS_')) {
env[key] = value;
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');`);
const envFile = path.resolve(databasePath, 'working', 'env.tmp');
await new toolrunnner.ToolRunner(cmd, [
'database',
'trace-command',
databasePath,
...getExtraOptionsFromEnv(['database', 'trace-command']),
process.execPath,
path.resolve(__dirname, 'tracer-env.js'),
tracerEnvJs,
envFile
]);
]).exec();
return JSON.parse(fs.readFileSync(envFile, 'utf-8'));
},
databaseInit: async function(databasePath: string, language: Language, sourceRoot: string) {
await exec.exec(cmd, [
await new toolrunnner.ToolRunner(cmd, [
'database',
'init',
databasePath,
'--language=' + language,
'--source-root=' + sourceRoot,
...getExtraOptionsFromEnv(['database', 'init']),
]);
]).exec();
},
runAutobuild: async function(language: Language) {
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
@ -351,12 +396,12 @@ function getCodeQLForCmd(cmd: string): CodeQL {
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
await exec.exec(autobuildCmd);
await new toolrunnner.ToolRunner(autobuildCmd).exec();
},
extractScannedLanguage: async function(databasePath: string, language: Language) {
// Get extractor location
let extractorPath = '';
await exec.exec(
await new toolrunnner.ToolRunner(
cmd,
[
'resolve',
@ -371,29 +416,29 @@ function getCodeQLForCmd(cmd: string): CodeQL {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
}).exec();
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
const traceCommand = path.resolve(JSON.parse(extractorPath), 'tools', 'autobuild' + ext);
// Run trace command
await exec.exec(cmd, [
await new toolrunnner.ToolRunner(cmd, [
'database',
'trace-command',
...getExtraOptionsFromEnv(['database', 'trace-command']),
databasePath,
'--',
traceCommand
]);
]).exec();
},
finalizeDatabase: async function(databasePath: string) {
await exec.exec(cmd, [
await new toolrunnner.ToolRunner(cmd, [
'database',
'finalize',
...getExtraOptionsFromEnv(['database', 'finalize']),
databasePath
]);
]).exec();
},
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
const codeqlArgs = [
@ -407,29 +452,35 @@ function getCodeQLForCmd(cmd: string): CodeQL {
codeqlArgs.push('--search-path', extraSearchPath);
}
let output = '';
await exec.exec(cmd, codeqlArgs, {
await new toolrunnner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
}
}
});
}).exec();
return JSON.parse(output);
},
databaseAnalyze: async function(databasePath: string, sarifFile: string, querySuite: string) {
await exec.exec(cmd, [
databaseAnalyze: async function(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
threadsFlag: string) {
await new toolrunnner.ToolRunner(cmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
memoryFlag,
threadsFlag,
databasePath,
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
...getExtraOptionsFromEnv(['database', 'analyze']),
querySuite
]);
]).exec();
}
};
}

417
src/config-utils.test.ts
View file

@ -8,25 +8,17 @@ import * as api from './api-client';
import { getCachedCodeQL, setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import {setupTests} from './testing-utils';
import * as util from './util';
setupTests(test);
function setInput(name: string, value: string | undefined) {
// Transformation copied from
// https://github.com/actions/toolkit/blob/05e39f551d33e1688f61b209ab5cdd335198f1b8/packages/core/src/core.ts#L69
const envVar = `INPUT_${name.replace(/ /g, '_').toUpperCase()}`;
if (value !== undefined) {
process.env[envVar] = value;
} else {
delete process.env[envVar];
}
}
function setConfigFile(inputFileContents: string, tmpDir: string) {
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
// Returns the filepath of the newly-created file
function createConfigFile(inputFileContents: string, tmpDir: string): string {
const configFilePath = path.join(tmpDir, 'input');
fs.writeFileSync(configFilePath, inputFileContents, 'utf8');
return configFilePath;
}
type GetContentsResponse = { content?: string; } | {}[];
@ -57,11 +49,8 @@ function mockListLanguages(languages: string[]) {
test("load empty config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', undefined);
setInput('languages', 'javascript,python');
const logger = getRunnerLogger(true);
const languages = 'javascript,python';
const codeQL = setCodeQL({
resolveQueries: async function() {
@ -73,19 +62,36 @@ test("load empty config", async t => {
},
});
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger);
t.deepEqual(config, await configUtils.getDefaultConfig(tmpDir, tmpDir, codeQL));
t.deepEqual(config, await configUtils.getDefaultConfig(
languages,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger));
});
});
test("loading config saves config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', undefined);
setInput('languages', 'javascript,python');
const logger = getRunnerLogger(true);
const codeQL = setCodeQL({
resolveQueries: async function() {
@ -101,29 +107,46 @@ test("loading config saves config", async t => {
// Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
// Sanity check that getConfig throws before we have called initConfig
await t.throwsAsync(() => configUtils.getConfig(tmpDir));
// Sanity check that getConfig returns undefined before we have called initConfig
t.deepEqual(await configUtils.getConfig(tmpDir, logger), undefined);
const config1 = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config1 = await configUtils.initConfig(
'javascript,python',
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger);
// The saved config file should now exist
t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
// And that same newly-initialised config should now be returned by getConfig
const config2 = await configUtils.getConfig(tmpDir);
const config2 = await configUtils.getConfig(tmpDir, logger);
t.deepEqual(config1, config2);
});
});
test("load input outside of workspace", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', '../input');
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
'../input',
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileOutsideWorkspaceErrorMessage(path.join(tmpDir, '../input'))));
@ -133,14 +156,22 @@ test("load input outside of workspace", async t => {
test("load non-local input with invalid repo syntax", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
// no filename given, just a repo
setInput('config-file', 'octo-org/codeql-config@main');
const configFile = 'octo-org/codeql-config@main';
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileRepoFormatInvalidMessage('octo-org/codeql-config@main')));
@ -150,15 +181,23 @@ test("load non-local input with invalid repo syntax", async t => {
test("load non-existent input", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
t.false(fs.existsSync(path.join(tmpDir, 'input')));
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
t.false(fs.existsSync(path.join(tmpDir, configFile)));
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileDoesNotExistErrorMessage(path.join(tmpDir, 'input'))));
@ -168,9 +207,6 @@ test("load non-existent input", async t => {
test("load non-empty input", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@ -197,7 +233,6 @@ test("load non-empty input", async t => {
- b
paths:
- c/d`;
setConfigFile(inputFileContents, tmpDir);
fs.mkdirSync(path.join(tmpDir, 'foo'));
@ -219,20 +254,29 @@ test("load non-empty input", async t => {
codeQLCmd: codeQL.getPath(),
};
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = createConfigFile(inputFileContents, tmpDir);
const actualConfig = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const actualConfig = await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Should exactly equal the object we constructed earlier
t.deepEqual(actualConfig, expectedConfig);
});
});
test("default queries are used", async t => {
test("Default queries are used", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
// Check that the default behaviour is to add the default queries.
// In this case if a config file is specified but does not include
// the disable-default-queries field.
@ -261,13 +305,24 @@ test("default queries are used", async t => {
const inputFileContents = `
paths:
- foo`;
setConfigFile(inputFileContents, tmpDir);
fs.mkdirSync(path.join(tmpDir, 'foo'));
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = createConfigFile(inputFileContents, tmpDir);
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolve queries was called correctly
t.deepEqual(resolveQueriesArgs.length, 1);
@ -295,14 +350,12 @@ function queriesToResolvedQueryForm(queries: string[]) {
test("Queries can be specified in config file", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const inputFileContents = `
name: my config
queries:
- uses: ./foo`;
setConfigFile(inputFileContents, tmpDir);
const configFile = createConfigFile(inputFileContents, tmpDir);
fs.mkdirSync(path.join(tmpDir, 'foo'));
@ -314,9 +367,20 @@ test("Queries can be specified in config file", async t => {
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly
// It'll be called once for the default queries
@ -334,17 +398,15 @@ test("Queries can be specified in config file", async t => {
test("Queries from config file can be overridden in workflow file", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const inputFileContents = `
name: my config
queries:
- uses: ./foo`;
setConfigFile(inputFileContents, tmpDir);
const configFile = createConfigFile(inputFileContents, tmpDir);
// This config item should take precedence over the config file but shouldn't affect the default queries.
setInput('queries', './override');
const queries = './override';
fs.mkdirSync(path.join(tmpDir, 'foo'));
fs.mkdirSync(path.join(tmpDir, 'override'));
@ -357,9 +419,20 @@ test("Queries from config file can be overridden in workflow file", async t => {
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
queries,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly
// It'll be called once for the default queries and once for `./override`,
@ -383,9 +456,9 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
const inputFileContents = `
name: my config
disable-default-queries: true`;
setConfigFile(inputFileContents, tmpDir);
const configFile = createConfigFile(inputFileContents, tmpDir);
setInput('queries', './workflow-query');
const queries = './workflow-query';
fs.mkdirSync(path.join(tmpDir, 'workflow-query'));
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
@ -396,9 +469,20 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
queries,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly
// It'll be called once for `./workflow-query`,
@ -415,13 +499,10 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
test("Multiple queries can be specified in workflow file, no config file required", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
fs.mkdirSync(path.join(tmpDir, 'override1'));
fs.mkdirSync(path.join(tmpDir, 'override2'));
setInput('queries', './override1,./override2');
const queries = './override1,./override2';
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
const codeQL = setCodeQL({
@ -431,9 +512,20 @@ test("Multiple queries can be specified in workflow file, no config file require
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
queries,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly:
// It'll be called once for the default queries,
@ -461,10 +553,10 @@ test("Queries in workflow file can be added to the set of queries without overri
name: my config
queries:
- uses: ./foo`;
setConfigFile(inputFileContents, tmpDir);
const configFile = createConfigFile(inputFileContents, tmpDir);
// These queries shouldn't override anything, because the value is prefixed with "+"
setInput('queries', '+./additional1,./additional2');
const queries = '+./additional1,./additional2';
fs.mkdirSync(path.join(tmpDir, 'foo'));
fs.mkdirSync(path.join(tmpDir, 'additional1'));
@ -478,9 +570,20 @@ test("Queries in workflow file can be added to the set of queries without overri
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
queries,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
// Check resolveQueries was called correctly
// It'll be called once for the default queries,
@ -505,11 +608,8 @@ test("Queries in workflow file can be added to the set of queries without overri
test("Invalid queries in workflow file handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('queries', 'foo/bar@v1@v3');
setInput('languages', 'javascript');
const queries = 'foo/bar@v1@v3';
const languages = 'javascript';
// This function just needs to be type-correct; it doesn't need to do anything,
// since we're deliberately passing in invalid data
@ -526,7 +626,18 @@ test("Invalid queries in workflow file handled correctly", async t => {
});
try {
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
queries,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
t.fail('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3")));
@ -536,9 +647,6 @@ test("Invalid queries in workflow file handled correctly", async t => {
test("API client used when reading remote config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@ -573,26 +681,44 @@ test("API client used when reading remote config", async t => {
// Create checkout directory for remote queries repository
fs.mkdirSync(path.join(tmpDir, 'foo/bar/dev'), { recursive: true });
setInput('config-file', 'octo-org/codeql-config/config.yaml@main');
setInput('languages', 'javascript');
const configFile = 'octo-org/codeql-config/config.yaml@main';
const languages = 'javascript';
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
t.assert(spyGetContents.called);
});
});
test("Remote config handles the case where a directory is provided", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const dummyResponse = []; // directories are returned as arrays
mockGetContents(dummyResponse);
const repoReference = 'octo-org/codeql-config/config.yaml@main';
setInput('config-file', repoReference);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
repoReference,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileDirectoryGivenMessage(repoReference)));
@ -602,18 +728,25 @@ test("Remote config handles the case where a directory is provided", async t =>
test("Invalid format of remote config handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const dummyResponse = {
// note no "content" property here
};
mockGetContents(dummyResponse);
const repoReference = 'octo-org/codeql-config/config.yaml@main';
setInput('config-file', repoReference);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
repoReference,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileFormatInvalidMessage(repoReference)));
@ -623,13 +756,21 @@ test("Invalid format of remote config handled correctly", async t => {
test("No detected languages", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
mockListLanguages([]);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getNoLanguagesError()));
@ -639,13 +780,21 @@ test("No detected languages", async t => {
test("Unknown languages", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('languages', 'ruby,english');
const languages = 'ruby,english';
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
languages,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(['ruby', 'english'])));
@ -660,9 +809,6 @@ function doInvalidInputTest(
test("load invalid input - " + testName, async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@ -673,13 +819,24 @@ function doInvalidInputTest(
},
});
const inputFile = path.join(tmpDir, 'input');
const languages = 'javascript';
const configFile = 'input';
const inputFile = path.join(tmpDir, configFile);
fs.writeFileSync(inputFile, inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
try {
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(expectedErrorMessageGenerator(inputFile)));
@ -787,10 +944,10 @@ test('path validations', t => {
const configFile = './.github/codeql/config.yml';
for (const path of validPaths) {
t.truthy(configUtils.validateAndSanitisePath(path, propertyName, configFile));
t.truthy(configUtils.validateAndSanitisePath(path, propertyName, configFile, getRunnerLogger(true)));
}
for (const path of invalidPaths) {
t.throws(() => configUtils.validateAndSanitisePath(path, propertyName, configFile));
t.throws(() => configUtils.validateAndSanitisePath(path, propertyName, configFile, getRunnerLogger(true)));
}
});
@ -801,11 +958,11 @@ test('path sanitisation', t => {
// Valid paths are not modified
t.deepEqual(
configUtils.validateAndSanitisePath('foo/bar', propertyName, configFile),
configUtils.validateAndSanitisePath('foo/bar', propertyName, configFile, getRunnerLogger(true)),
'foo/bar');
// Trailing stars are stripped
t.deepEqual(
configUtils.validateAndSanitisePath('foo/**', propertyName, configFile),
configUtils.validateAndSanitisePath('foo/**', propertyName, configFile, getRunnerLogger(true)),
'foo/');
});

317
src/config-utils.ts
View file

@ -1,4 +1,3 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import * as yaml from 'js-yaml';
import * as path from 'path';
@ -7,7 +6,8 @@ import * as api from './api-client';
import { CodeQL, ResolveQueriesOutput } from './codeql';
import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import * as util from './util';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
// Property names from the user-supplied config file.
const NAME_PROPERTY = 'name';
@ -182,12 +182,12 @@ async function addLocalQueries(
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
localQueryPath: string,
checkoutPath: string,
configFile?: string) {
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE'));
let absoluteQueryPath = path.join(workspacePath, localQueryPath);
let absoluteQueryPath = path.join(checkoutPath, localQueryPath);
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
@ -198,14 +198,11 @@ async function addLocalQueries(
absoluteQueryPath = fs.realpathSync(absoluteQueryPath);
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (!(absoluteQueryPath + path.sep).startsWith(workspacePath + path.sep)) {
if (!(absoluteQueryPath + path.sep).startsWith(fs.realpathSync(checkoutPath) + path.sep)) {
throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
}
// Get the root of the current repo to use when resolving query dependencies
const rootOfRepo = util.getRequiredEnvParam('GITHUB_WORKSPACE');
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], rootOfRepo, true);
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath, true);
}
/**
@ -216,6 +213,8 @@ async function addRemoteQueries(
resultMap: { [language: string]: string[] },
queryUses: string,
tempDir: string,
githubUrl: string,
logger: Logger,
configFile?: string) {
let tok = queryUses.split('@');
@ -239,13 +238,18 @@ async function addRemoteQueries(
const nwo = tok[0] + '/' + tok[1];
// Checkout the external repository
const rootOfRepo = await externalQueries.checkoutExternalRepository(nwo, ref, tempDir);
const checkoutPath = await externalQueries.checkoutExternalRepository(
nwo,
ref,
githubUrl,
tempDir,
logger);
const queryPath = tok.length > 2
? path.join(rootOfRepo, tok.slice(2).join('/'))
: rootOfRepo;
? path.join(checkoutPath, tok.slice(2).join('/'))
: checkoutPath;
await runResolveQueries(codeQL, resultMap, [queryPath], rootOfRepo, true);
await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath, true);
}
/**
@ -262,6 +266,9 @@ async function parseQueryUses(
resultMap: { [language: string]: string[] },
queryUses: string,
tempDir: string,
checkoutPath: string,
githubUrl: string,
logger: Logger,
configFile?: string) {
queryUses = queryUses.trim();
@ -271,7 +278,7 @@ async function parseQueryUses(
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), configFile);
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), checkoutPath, configFile);
return;
}
@ -282,7 +289,7 @@ async function parseQueryUses(
}
// Otherwise, must be a reference to another repo
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile);
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, githubUrl, logger, configFile);
}
// Regex validating stars in paths or paths-ignore entries.
@ -299,7 +306,8 @@ const filterPatternCharactersRegex = /.*[\?\+\[\]!].*/;
export function validateAndSanitisePath(
originalPath: string,
propertyName: string,
configFile: string): string {
configFile: string,
logger: Logger): string {
// Take a copy so we don't modify the original path, so we can still construct error messages
let path = originalPath;
@ -335,7 +343,7 @@ export function validateAndSanitisePath(
// Check for other regex characters that we don't support.
// Output a warning so the user knows, but otherwise continue normally.
if (path.match(filterPatternCharactersRegex)) {
core.warning(getConfigFilePropertyError(
logger.warning(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" contains an unsupported character. ' +
@ -445,35 +453,32 @@ export function getUnknownLanguagesError(languages: string[]): string {
/**
* Gets the set of languages in the current repository
*/
async function getLanguagesInRepo(): Promise<Language[]> {
let repo_nwo = process.env['GITHUB_REPOSITORY']?.split("/");
if (repo_nwo) {
let owner = repo_nwo[0];
let repo = repo_nwo[1];
async function getLanguagesInRepo(
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
core.debug(`GitHub repo ${owner} ${repo}`);
const response = await api.getActionsApiClient(true).repos.listLanguages({
owner,
repo
});
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.listLanguages({
owner: repository.owner,
repo: repository.repo
});
core.debug("Languages API response: " + JSON.stringify(response));
logger.debug("Languages API response: " + JSON.stringify(response));
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
let languages: Set<Language> = new Set();
for (let lang of Object.keys(response.data)) {
let parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
let languages: Set<Language> = new Set();
for (let lang of Object.keys(response.data)) {
let parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
return [...languages];
} else {
return [];
}
return [...languages];
}
/**
@ -486,19 +491,28 @@ async function getLanguagesInRepo(): Promise<Language[]> {
* If no languages could be detected from either the workflow or the repository
* then throw an error.
*/
async function getLanguages(): Promise<Language[]> {
async function getLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
// Obtain from action input 'languages' if set
let languages = core.getInput('languages', { required: false })
let languages = (languagesInput || "")
.split(',')
.map(x => x.trim())
.filter(x => x.length > 0);
core.info("Languages from configuration: " + JSON.stringify(languages));
logger.info("Languages from configuration: " + JSON.stringify(languages));
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
languages = await getLanguagesInRepo();
core.info("Automatically detected languages: " + JSON.stringify(languages));
languages = await getLanguagesInRepo(
repository,
githubAuth,
githubUrl,
logger);
logger.info("Automatically detected languages: " + JSON.stringify(languages));
}
// If the languages parameter was not given and no languages were
@ -525,19 +539,30 @@ async function getLanguages(): Promise<Language[]> {
return parsedLanguages;
}
async function addQueriesFromWorkflowIfRequired(
async function addQueriesFromWorkflow(
codeQL: CodeQL,
queriesInput: string,
languages: string[],
resultMap: { [language: string]: string[] },
tempDir: string
) {
let queryUses = core.getInput('queries');
if (queryUses) {
queryUses = queryUses.trim();
queryUses = queryUses.replace(/^\+/, ''); // "+" means "don't override config file" - see shouldAddConfigFileQueries
for (const query of queryUses.split(',')) {
await parseQueryUses(languages, codeQL, resultMap, query, tempDir);
}
tempDir: string,
checkoutPath: string,
githubUrl: string,
logger: Logger) {
queriesInput = queriesInput.trim();
// "+" means "don't override config file" - see shouldAddConfigFileQueries
queriesInput = queriesInput.replace(/^\+/, '');
for (const query of queriesInput.split(',')) {
await parseQueryUses(
languages,
codeQL,
resultMap,
query,
tempDir,
checkoutPath,
githubUrl,
logger);
}
}
@ -545,10 +570,9 @@ async function addQueriesFromWorkflowIfRequired(
// or if the queries in the workflow were provided in "additive" mode,
// indicating that they shouldn't override the config queries but
// should instead be added in addition
function shouldAddConfigFileQueries(): boolean {
const queryUses = core.getInput('queries');
if (queryUses) {
return queryUses.trimStart().substr(0, 1) === '+';
function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
if (queriesInput) {
return queriesInput.trimStart().substr(0, 1) === '+';
}
return true;
@ -557,11 +581,37 @@ function shouldAddConfigFileQueries(): boolean {
/**
* Get the default config for when the user has not supplied one.
*/
export async function getDefaultConfig(tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
const languages = await getLanguages();
export async function getDefaultConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
const queries = {};
await addDefaultQueries(codeQL, languages, queries);
await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
if (queriesInput) {
await addQueriesFromWorkflow(
codeQL,
queriesInput,
languages,
queries,
tempDir,
checkoutPath,
githubUrl,
logger);
}
return {
languages: languages,
@ -578,17 +628,30 @@ export async function getDefaultConfig(tempDir: string, toolCacheDir: string, co
/**
* Load the config from the given file.
*/
async function loadConfig(configFile: string, tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
async function loadConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
let parsedYAML: UserConfig;
if (isLocal(configFile)) {
// Treat the config file as relative to the workspace
const workspacePath = util.getRequiredEnvParam('GITHUB_WORKSPACE');
configFile = path.resolve(workspacePath, configFile);
parsedYAML = getLocalConfig(configFile, workspacePath);
configFile = path.resolve(checkoutPath, configFile);
parsedYAML = getLocalConfig(configFile, checkoutPath);
} else {
parsedYAML = await getRemoteConfig(configFile);
parsedYAML = await getRemoteConfig(
configFile,
githubAuth,
githubUrl);
}
// Validate that the 'name' property is syntactically correct,
@ -602,7 +665,12 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
}
}
const languages = await getLanguages();
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
const queries = {};
const pathsIgnore: string[] = [];
@ -623,8 +691,18 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
// they should take precedence over the queries in the config file
// unless they're prefixed with "+", in which case they supplement those
// in the config file.
await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
if (shouldAddConfigFileQueries() && QUERIES_PROPERTY in parsedYAML) {
if (queriesInput) {
await addQueriesFromWorkflow(
codeQL,
queriesInput,
languages,
queries,
tempDir,
checkoutPath,
githubUrl,
logger);
}
if (shouldAddConfigFileQueries(queriesInput) && QUERIES_PROPERTY in parsedYAML) {
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
throw new Error(getQueriesInvalid(configFile));
}
@ -632,7 +710,16 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
throw new Error(getQueryUsesInvalid(configFile));
}
await parseQueryUses(languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir, configFile);
await parseQueryUses(
languages,
codeQL,
queries,
query[QUERIES_USES_PROPERTY],
tempDir,
checkoutPath,
githubUrl,
logger,
configFile);
}
}
@ -644,7 +731,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (typeof path !== "string" || path === '') {
throw new Error(getPathsIgnoreInvalid(configFile));
}
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile));
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile, logger));
});
}
@ -656,7 +743,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (typeof path !== "string" || path === '') {
throw new Error(getPathsInvalid(configFile));
}
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile));
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile, logger));
});
}
@ -687,20 +774,52 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
* This will parse the config from the user input if present, or generate
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
const configFile = core.getInput('config-file');
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
let config: Config;
// If no config file was provided create an empty one
if (configFile === '') {
core.debug('No configuration file was provided');
config = await getDefaultConfig(tempDir, toolCacheDir, codeQL);
if (!configFile) {
logger.debug('No configuration file was provided');
config = await getDefaultConfig(
languagesInput,
queriesInput,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
} else {
config = await loadConfig(configFile, tempDir, toolCacheDir, codeQL);
config = await loadConfig(
languagesInput,
queriesInput,
configFile,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config);
await saveConfig(config, logger);
return config;
}
@ -713,9 +832,9 @@ function isLocal(configPath: string): boolean {
return (configPath.indexOf("@") === -1);
}
function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
// Error if the config file is now outside of the workspace
if (!(configFile + path.sep).startsWith(workspacePath + path.sep)) {
if (!(configFile + path.sep).startsWith(checkoutPath + path.sep)) {
throw new Error(getConfigFileOutsideWorkspaceErrorMessage(configFile));
}
@ -727,7 +846,11 @@ function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
return yaml.safeLoad(fs.readFileSync(configFile, 'utf8'));
}
async function getRemoteConfig(configFile: string): Promise<UserConfig> {
async function getRemoteConfig(
configFile: string,
githubAuth: string,
githubUrl: string): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp('(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)');
const pieces = format.exec(configFile);
@ -736,7 +859,7 @@ async function getRemoteConfig(configFile: string): Promise<UserConfig> {
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
}
const response = await api.getActionsApiClient(true).repos.getContents({
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.getContents({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
@ -765,30 +888,26 @@ export function getPathToParsedConfigFile(tempDir: string): string {
/**
* Store the given config to the path returned from getPathToParsedConfigFile.
*/
async function saveConfig(config: Config) {
async function saveConfig(config: Config, logger: Logger) {
const configString = JSON.stringify(config);
const configFile = getPathToParsedConfigFile(config.tempDir);
fs.mkdirSync(path.dirname(configFile), { recursive: true });
fs.writeFileSync(configFile, configString, 'utf8');
core.debug('Saved config:');
core.debug(configString);
logger.debug('Saved config:');
logger.debug(configString);
}
/**
* Get the config.
*
* If this is the first time in a workflow that this is being called then
* this will parse the config from the user input. The parsed config is then
* stored to a known location. On the second and further calls, this will
* return the contents of the parsed config from the known location.
* Get the config that has been saved to the given temp dir.
* If the config could not be found then returns undefined.
*/
export async function getConfig(tempDir: string): Promise<Config> {
export async function getConfig(tempDir: string, logger: Logger): Promise<Config | undefined> {
const configFile = getPathToParsedConfigFile(tempDir);
if (!fs.existsSync(configFile)) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
return undefined;
}
const configString = fs.readFileSync(configFile, 'utf8');
core.debug('Loaded config:');
core.debug(configString);
logger.debug('Loaded config:');
logger.debug(configString);
return JSON.parse(configString);
}

5
src/external-queries.test.ts
View file

@ -3,6 +3,7 @@ import * as fs from "fs";
import * as path from "path";
import * as externalQueries from "./external-queries";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from "./util";
@ -14,7 +15,9 @@ test("checkoutExternalQueries", async t => {
await externalQueries.checkoutExternalRepository(
"github/codeql-go",
ref,
tmpDir);
'https://github.com',
tmpDir,
getRunnerLogger(true));
// COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in the default branch
t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", ref, "COPYRIGHT")));

23
src/external-queries.ts
View file

@ -1,13 +1,20 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from 'fs';
import * as path from 'path';
import { Logger } from './logging';
/**
* Check out repository at the given ref, and return the directory of the checkout.
*/
export async function checkoutExternalRepository(repository: string, ref: string, tempDir: string): Promise<string> {
core.info('Checking out ' + repository);
export async function checkoutExternalRepository(
repository: string,
ref: string,
githubUrl: string,
tempDir: string,
logger: Logger): Promise<string> {
logger.info('Checking out ' + repository);
const checkoutLocation = path.join(tempDir, repository, ref);
@ -17,13 +24,13 @@ export async function checkoutExternalRepository(repository: string, ref: string
}
if (!fs.existsSync(checkoutLocation)) {
const repoURL = 'https://github.com/' + repository + '.git';
await exec.exec('git', ['clone', repoURL, checkoutLocation]);
await exec.exec('git', [
const repoURL = githubUrl + '/' + repository + '.git';
await new toolrunnner.ToolRunner('git', ['clone', repoURL, checkoutLocation]).exec();
await new toolrunnner.ToolRunner('git', [
'--work-tree=' + checkoutLocation,
'--git-dir=' + checkoutLocation + '/.git',
'checkout', ref,
]);
]).exec();
}
return checkoutLocation;

12
src/fingerprints.test.ts
View file

@ -116,7 +116,7 @@ test('hash', (t: ava.Assertions) => {
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
const location = { "uri": uri, "index": index };
const artifacts = artifactsURIs.map(uri => ({ "location": { "uri": uri } }));
return fingerprints.resolveUriToFile(location, artifacts, getRunnerLogger());
return fingerprints.resolveUriToFile(location, artifacts, process.cwd(), getRunnerLogger(true));
}
test('resolveUriToFile', t => {
@ -129,8 +129,6 @@ test('resolveUriToFile', t => {
t.true(filepath.startsWith(cwd + '/'));
const relativeFilepaht = filepath.substring(cwd.length + 1);
process.env['GITHUB_WORKSPACE'] = cwd;
// Absolute paths are unmodified
t.is(testResolveUriToFile(filepath, undefined, []), filepath);
t.is(testResolveUriToFile('file://' + filepath, undefined, []), filepath);
@ -173,9 +171,9 @@ test('addFingerprints', t => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input, getRunnerLogger()), expected);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)), expected);
});
test('missingRegions', t => {
@ -188,7 +186,7 @@ test('missingRegions', t => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input, getRunnerLogger()), expected);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)), expected);
});

17
src/fingerprints.ts
View file

@ -161,7 +161,12 @@ function locationUpdateCallback(result: any, location: any, logger: Logger): has
// the source file so we can hash it.
// If possible returns a absolute file path for the source file,
// or if not possible then returns undefined.
export function resolveUriToFile(location: any, artifacts: any[], logger: Logger): string | undefined {
export function resolveUriToFile(
location: any,
artifacts: any[],
checkoutPath: string,
logger: Logger): string | undefined {
// This may be referencing an artifact
if (!location.uri && location.index !== undefined) {
if (typeof location.index !== 'number' ||
@ -192,7 +197,7 @@ export function resolveUriToFile(location: any, artifacts: any[], logger: Logger
}
// Discard any absolute paths that aren't in the src root
const srcRootPrefix = process.env['GITHUB_WORKSPACE'] + '/';
const srcRootPrefix = checkoutPath + '/';
if (uri.startsWith('/') && !uri.startsWith(srcRootPrefix)) {
logger.debug(`Ignoring location URI "${uri}" as it is outside of the src root`);
return undefined;
@ -216,7 +221,7 @@ export function resolveUriToFile(location: any, artifacts: any[], logger: Logger
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(sarifContents: string, logger: Logger): string {
export function addFingerprints(sarifContents: string, checkoutPath: string, logger: Logger): string {
let sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
@ -234,7 +239,11 @@ export function addFingerprints(sarifContents: string, logger: Logger): string {
continue;
}
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, logger);
const filepath = resolveUriToFile(
primaryLocation.physicalLocation.artifactLocation,
artifacts,
checkoutPath,
logger);
if (!filepath) {
continue;
}

63
src/init-action.ts
View file

@ -1,12 +1,10 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { getCombinedTracerConfig } from './tracer-config';
import { initCodeQL, initConfig, runInit } from './init';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as util from './util';
interface InitSuccessStatusReport extends util.StatusReportBase {
@ -49,8 +47,8 @@ async function sendSuccessStatusReport(startedAt: Date, config: configUtils.Conf
}
async function run() {
const startedAt = new Date();
const logger = getActionsLogger();
let config: configUtils.Config;
let codeql: CodeQL;
@ -60,18 +58,26 @@ async function run() {
return;
}
core.startGroup('Setup CodeQL tools');
codeql = await setupCodeQL();
await codeql.printVersion();
core.endGroup();
core.startGroup('Load language configuration');
config = await configUtils.initConfig(
codeql = await initCodeQL(
core.getInput('tools'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql);
analysisPaths.includeAndExcludeAnalysisPaths(config);
core.endGroup();
'actions',
logger);
config = await initConfig(
core.getInput('languages'),
core.getInput('queries'),
core.getInput('config-file'),
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql,
util.getRequiredEnvParam('GITHUB_WORKSPACE'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
logger);
} catch (e) {
core.setFailed(e.message);
@ -82,8 +88,6 @@ async function run() {
try {
const sourceRoot = path.resolve();
// Forward Go flags
const goFlags = process.env['GOFLAGS'];
if (goFlags) {
@ -95,27 +99,8 @@ async function run() {
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
// Init language database
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
const tracerConfig = await getCombinedTracerConfig(config, codeql);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig !== undefined) {
if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
}
// NB: in CLI mode these will be output to a file rather than exported with core.exportVariable
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
}

104
src/init.ts Normal file
View file

@ -0,0 +1,104 @@
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import { getCombinedTracerConfig, TracerConfig } from './tracer-config';
import * as util from './util';
export async function initCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
logger.startGroup('Setup CodeQL tools');
const codeql = await setupCodeQL(
codeqlURL,
githubAuth,
githubUrl,
tempDir,
toolsDir,
mode,
logger);
await codeql.printVersion();
logger.endGroup();
return codeql;
}
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<configUtils.Config> {
logger.startGroup('Load language configuration');
const config = await configUtils.initConfig(
languagesInput,
queriesInput,
configFile,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
analysisPaths.printPathFiltersWarning(config, logger);
logger.endGroup();
return config;
}
export async function runInit(
codeql: CodeQL,
config: configUtils.Config): Promise<TracerConfig | undefined> {
const sourceRoot = path.resolve();
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
// Init language database
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
const tracerConfig = await getCombinedTracerConfig(config, codeql);
if (tracerConfig !== undefined && process.platform === 'win32') {
const injectTracerPath = path.join(config.tempDir, 'inject-tracer.ps1');
fs.writeFileSync(injectTracerPath, `
Param(
[Parameter(Position=0)]
[String]
$tracer
)
Get-Process -Name Runner.Worker
$process=Get-Process -Name Runner.Worker
$id=$process.Id
Invoke-Expression "&$tracer --inject=$id"`);
await new toolrunnner.ToolRunner(
'powershell',
[
injectTracerPath,
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } }).exec();
}
return tracerConfig;
}

9
src/inject-tracer.ps1
View file

@ -1,9 +0,0 @@
Param(
[Parameter(Position=0)]
[String]
$tracer
)
Get-Process -Name Runner.Worker
$process=Get-Process -Name Runner.Worker
$id=$process.Id
Invoke-Expression "&$tracer --inject=$id"

4
src/logging.ts
View file

@ -14,9 +14,9 @@ export function getActionsLogger(): Logger {
return core;
}
export function getRunnerLogger(): Logger {
export function getRunnerLogger(debugMode: boolean): Logger {
return {
debug: console.debug,
debug: debugMode ? console.debug : () => undefined,
info: console.info,
warning: console.warn,
error: console.error,

328
src/runner.ts
View file

@ -1,36 +1,35 @@
import { Command } from 'commander';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { runAnalyze } from './analyze';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import { CodeQL, getCodeQL } from './codeql';
import { Config, getConfig } from './config-utils';
import { initCodeQL, initConfig, runInit } from './init';
import { Language, parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import { getMemoryFlag, getThreadsFlag } from './util';
const program = new Command();
program.version('0.0.1');
interface UploadArgs {
sarifFile: string;
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
}
function parseGithubApiUrl(inputUrl: string): string {
function parseGithubUrl(inputUrl: string): string {
try {
const url = new URL(inputUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
return 'https://github.com';
}
// Add the API path if it's not already present.
if (url.pathname.indexOf('/api/v3') === -1) {
url.pathname = path.join(url.pathname, 'api', 'v3');
// Remove the API prefix if it's present
if (url.pathname.indexOf('/api/v3') !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf('/api/v3'));
}
return url.toString();
@ -40,32 +39,301 @@ function parseGithubApiUrl(inputUrl: string): string {
}
}
const logger = getRunnerLogger();
function getTempDir(userInput: string | undefined): string {
const tempDir = path.join(userInput || process.cwd(), 'codeql-runner');
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
return tempDir;
}
function getToolsDir(userInput: string | undefined): string {
const toolsDir = userInput || path.join(os.homedir(), 'codeql-runner-tools');
if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true });
}
return toolsDir;
}
const codeqlEnvJsonFilename = 'codeql-env.json';
// Imports the environment from codeqlEnvJsonFilename if not already present
function importTracerEnvironment(config: Config) {
if (!('ODASA_TRACER_CONFIGURATION' in process.env)) {
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString('utf-8'));
Object.keys(env).forEach(key => process.env[key] = env[key]);
}
}
// Allow the user to specify refs in full refs/heads/branch format
// or just the short branch name and prepend "refs/heads/" to it.
function parseRef(userInput: string): string {
if (userInput.startsWith('refs/')) {
return userInput;
} else {
return 'refs/heads/' + userInput;
}
}
interface InitArgs {
languages: string | undefined;
queries: string | undefined;
configFile: string | undefined;
codeqlPath: string | undefined;
tempDir: string | undefined;
toolsDir: string | undefined;
checkoutPath: string | undefined;
repository: string;
githubUrl: string;
githubAuth: string;
debug: boolean;
}
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload; can also be a directory for uploading multiple')
.requiredOption('--repository <repository>', 'Repository name')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed')
.requiredOption('--github-url <url>', 'URL of GitHub instance')
.requiredOption('--github-auth <auth>', 'GitHub Apps token, or of the form "username:token" if using a personal access token')
.option('--checkout-path <path>', 'Checkout path (default: current working directory)')
.action(async (cmd: UploadArgs) => {
.command('init')
.description('Initializes CodeQL')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--languages <languages>', 'Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo.')
.option('--queries <queries>', 'Comma-separated list of additional queries to run. This overrides the same setting in a configuration file.')
.option('--config-file <file>', 'Path to config file.')
.option('--codeql-path <path>', 'Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--tools-dir <dir>', 'Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory.')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: InitArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
await upload_lib.upload(
cmd.sarifFile,
const tempDir = getTempDir(cmd.tempDir);
const toolsDir = getToolsDir(cmd.toolsDir);
// Wipe the temp dir
logger.info(`Cleaning temp directory ${tempDir}`);
fs.rmdirSync(tempDir, { recursive: true });
fs.mkdirSync(tempDir, { recursive: true });
let codeql: CodeQL;
if (cmd.codeqlPath !== undefined) {
codeql = getCodeQL(cmd.codeqlPath);
} else {
codeql = await initCodeQL(
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
tempDir,
toolsDir,
'runner',
logger);
}
const config = await initConfig(
cmd.languages,
cmd.queries,
cmd.configFile,
parseRepositoryNwo(cmd.repository),
tempDir,
toolsDir,
codeql,
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
logger);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig === undefined) {
return;
}
// Always output a json file of the env that can be consumed programatically
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
fs.writeFileSync(jsonEnvFile, JSON.stringify(tracerConfig.env));
if (process.platform === 'win32') {
const batEnvFile = path.join(config.tempDir, 'codeql-env.bat');
const batEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `Set ${key}=${value}`)
.join('\n');
fs.writeFileSync(batEnvFile, batEnvFileContents);
const powershellEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const powershellEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `$env:${key}="${value}"`)
.join('\n');
fs.writeFileSync(powershellEnvFile, powershellEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`);
} else {
// Assume that anything that's not windows is using a unix-style shell
const shEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const shEnvFileContents = Object.entries(tracerConfig.env)
// Some vars contain ${LIB} that we do not want to be expanded when executing this script
.map(([key, value]) => `export ${key}="${value.replace('$', '\\$')}"`)
.join('\n');
fs.writeFileSync(shEnvFile, shEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`for example by running ". ${shEnvFile}".`);
}
} catch (e) {
logger.error('Init failed');
logger.error(e);
process.exitCode = 1;
}
});
interface AutobuildArgs {
language: string;
tempDir: string | undefined;
debug: boolean;
}
program
.command('autobuild')
.description('Attempts to automatically build code')
.option('--language <language>', 'The language to build. Otherwise will detect the dominant compiled language.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: AutobuildArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
}
importTracerEnvironment(config);
let language: Language | undefined = undefined;
if (cmd.language !== undefined) {
language = parseLanguage(cmd.language);
if (language === undefined || !config.languages.includes(language)) {
throw new Error(`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(', ')}.`);
}
} else {
language = determineAutobuildLanguage(config, logger);
}
if (language !== undefined) {
await runAutobuild(language, config, logger);
}
} catch (e) {
logger.error('Autobuild failed');
logger.error(e);
process.exitCode = 1;
}
});
interface AnalyzeArgs {
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
upload: boolean;
outputDir: string | undefined;
ram: string | undefined;
threads: string | undefined;
tempDir: string | undefined;
debug: boolean;
}
program
.command('analyze')
.description('Finishes extracting code and runs CodeQL queries')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--no-upload', 'Do not upload results after analysis.', false)
.option('--output-dir <dir>', 'Directory to output SARIF files to. Default is in the temp directory.')
.option('--ram <ram>', 'Amount of memory to use when running queries. Default is to use all available memory.')
.option('--threads <threads>', 'Number of threads to use when running queries. ' +
'Default is to use all available cores.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: AnalyzeArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const tempDir = getTempDir(cmd.tempDir);
const outputDir = cmd.outputDir || path.join(tempDir, 'codeql-sarif');
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
}
await runAnalyze(
parseRepositoryNwo(cmd.repository),
cmd.commit,
cmd.ref,
parseRef(cmd.ref),
undefined,
undefined,
undefined,
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubApiUrl(cmd.githubUrl),
parseGithubUrl(cmd.githubUrl),
cmd.upload,
'runner',
outputDir,
getMemoryFlag(cmd.ram),
getThreadsFlag(cmd.threads, logger),
config,
logger);
} catch (e) {
logger.error('Analyze failed');
logger.error(e);
process.exitCode = 1;
}
});
interface UploadArgs {
sarifFile: string;
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
debug: boolean;
}
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload, or a directory containing multiple SARIF files. (Required)')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: UploadArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
await upload_lib.upload(
cmd.sarifFile,
parseRepositoryNwo(cmd.repository),
cmd.commit,
parseRef(cmd.ref),
undefined,
undefined,
undefined,
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
'runner',
logger);
} catch (e) {

4
src/testing-utils.ts
View file

@ -55,10 +55,6 @@ export function setupTests(test: TestInterface<any>) {
// process.env only has strings fields, so a shallow copy is fine.
t.context.env = {};
Object.assign(t.context.env, process.env);
// Any test that runs code that expects to only be run on actions
// will depend on various environment variables.
process.env['GITHUB_API_URL'] = 'https://github.localhost/api/v3';
});
typedTest.afterEach.always(t => {

17
src/tracer-config.test.ts
View file

@ -299,13 +299,20 @@ test('getCombinedTracerConfig - valid spec file', async t => {
});
const result = await getCombinedTracerConfig(config, codeQL);
const expectedEnv = {
'foo': 'bar',
'ODASA_TRACER_CONFIGURATION': result!.spec,
};
if (process.platform === 'darwin') {
expectedEnv['DYLD_INSERT_LIBRARIES'] = path.join(path.dirname(codeQL.getPath()), 'tools', 'osx64', 'libtrace.dylib');
} else if (process.platform !== 'win32') {
expectedEnv['LD_PRELOAD'] = path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so');
}
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'foo': 'bar',
'ODASA_TRACER_CONFIGURATION': result!.spec,
'LD_PRELOAD': path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so'),
}
env: expectedEnv,
});
});
});

12
src/tracer-env.ts
View file

@ -1,12 +0,0 @@
import * as fs from 'fs';
const env = {};
for (let entry of Object.entries(process.env)) {
const key = entry[0];
const value = entry[1];
if (typeof value !== 'undefined' && key !== '_' && !key.startsWith('JAVA_MAIN_CLASS_')) {
env[key] = value;
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');

4
src/upload-lib.test.ts
View file

@ -8,10 +8,10 @@ setupTests(test);
test('validateSarifFileSchema - valid', t => {
const inputFile = __dirname + '/../src/testdata/valid-sarif.sarif';
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger()));
t.notThrows(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)));
});
test('validateSarifFileSchema - invalid', t => {
const inputFile = __dirname + '/../src/testdata/invalid-sarif.sarif';
t.throws(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger()));
t.throws(() => uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)));
});

22
src/upload-lib.ts
View file

@ -12,8 +12,6 @@ import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as util from './util';
type UploadMode = 'actions' | 'runner';
// Takes a list of paths to sarif files and combines them together,
// returning the contents of the combined sarif file.
export function combineSarifFiles(sarifFiles: string[]): string {
@ -43,8 +41,8 @@ async function uploadPayload(
payload: any,
repositoryNwo: RepositoryNwo,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger) {
logger.info('Uploading results');
@ -61,7 +59,7 @@ async function uploadPayload(
// minutes, but just waiting a little bit could maybe help.
const backoffPeriods = [1, 5, 15];
const client = api.getApiClient(githubAuth, githubApiUrl);
const client = api.getApiClient(githubAuth, githubUrl);
for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
const reqURL = mode === 'actions'
@ -134,8 +132,8 @@ export async function upload(
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
const sarifFiles: string[] = [];
@ -165,7 +163,7 @@ export async function upload(
checkoutPath,
environment,
githubAuth,
githubApiUrl,
githubUrl,
mode,
logger);
}
@ -214,8 +212,8 @@ async function uploadFiles(
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
logger.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
@ -235,7 +233,7 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload, logger);
sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutURI = fileUrl(checkoutPath);
@ -275,7 +273,7 @@ async function uploadFiles(
logger.debug("Number of results in upload: " + numResultInSarif);
// Make the upload
await uploadPayload(payload, repositoryNwo, githubAuth, githubApiUrl, mode, logger);
await uploadPayload(payload, repositoryNwo, githubAuth, githubUrl, mode, logger);
return {
raw_upload_size_bytes: rawUploadSizeBytes,

2
src/upload-sarif-action.ts
View file

@ -34,7 +34,7 @@ async function run() {
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
'actions',
getActionsLogger());
await sendSuccessStatusReport(startedAt, uploadStats);

17
src/util.test.ts
View file

@ -2,6 +2,7 @@ import test from 'ava';
import * as fs from 'fs';
import * as os from "os";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
@ -23,18 +24,14 @@ test('getMemoryFlag() should return the correct --ram flag', t => {
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_RAM'] = input;
const flag = util.getMemoryFlag();
const flag = util.getMemoryFlag(input);
t.deepEqual(flag, expectedFlag);
}
});
test('getMemoryFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["-1", "hello!"]) {
process.env['INPUT_RAM'] = input;
t.throws(util.getMemoryFlag);
t.throws(() => util.getMemoryFlag(input));
}
});
@ -50,17 +47,13 @@ test('getThreadsFlag() should return the correct --threads flag', t => {
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_THREADS'] = input;
const flag = util.getThreadsFlag();
const flag = util.getThreadsFlag(input, getRunnerLogger(true));
t.deepEqual(flag, expectedFlag);
}
});
test('getThreadsFlag() throws if the threads input is not an integer', t => {
process.env['INPUT_THREADS'] = "hello!";
t.throws(util.getThreadsFlag);
t.throws(() => util.getThreadsFlag("hello!", getRunnerLogger(true)));
});
test('getRef() throws on the empty string', t => {

48
src/util.ts
View file

@ -1,32 +1,24 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolrunnner from '@actions/exec/lib/toolrunner';
import * as fs from "fs";
import * as os from 'os';
import * as path from 'path';
import * as api from './api-client';
import { Language } from './languages';
import { Logger } from './logging';
import * as sharedEnv from './shared-environment';
/**
* The API URL for github.com.
* Are we running on actions, or not.
*/
export const GITHUB_DOTCOM_API_URL = "https://api.github.com";
export type Mode = 'actions' | 'runner';
/**
* Get the API URL for the GitHub instance we are connected to.
* May be for github.com or for an enterprise instance.
* The URL for github.com.
*/
export function getInstanceAPIURL(): string {
return process.env["GITHUB_API_URL"] || GITHUB_DOTCOM_API_URL;
}
export const GITHUB_DOTCOM_URL = "https://github.com";
/**
* Are we running against a GitHub Enterpise instance, as opposed to github.com.
*/
export function isEnterprise(): boolean {
return getInstanceAPIURL() !== GITHUB_DOTCOM_API_URL;
}
/**
* Get an environment parameter, but throw an error if it is not set.
@ -93,13 +85,13 @@ export async function getCommitOid(): Promise<string> {
// reported on the merge commit.
try {
let commitOid = '';
await exec.exec('git', ['rev-parse', 'HEAD'], {
await new toolrunnner.ToolRunner('git', ['rev-parse', 'HEAD'], {
silent: true,
listeners: {
stdout: (data) => { commitOid += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
}).exec();
return commitOid.trim();
} catch (e) {
core.info("Failed to call git to get current commit. Continuing with data from environment: " + e);
@ -297,7 +289,7 @@ export async function sendStatusReport<S extends StatusReportBase>(
statusReport: S,
ignoreFailures?: boolean): Promise<boolean> {
if (isEnterprise()) {
if (getRequiredEnvParam("GITHUB_SERVER_URL") !== GITHUB_DOTCOM_URL) {
core.debug("Not sending status report to GitHub Enterprise");
return true;
}
@ -378,13 +370,12 @@ export async function withTmpDir<T>(body: (tmpDir: string) => Promise<T>): Promi
*
* @returns string
*/
export function getMemoryFlag(): string {
export function getMemoryFlag(userInput: string | undefined): string {
let memoryToUseMegaBytes: number;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
throw new Error("Invalid RAM setting \"" + userInput + "\", specified.");
}
} else {
const totalMemoryBytes = os.totalmem();
@ -403,22 +394,21 @@ export function getMemoryFlag(): string {
*
* @returns string
*/
export function getThreadsFlag(): string {
export function getThreadsFlag(userInput: string | undefined, logger: Logger): string {
let numThreads: number;
const numThreadsString = core.getInput("threads");
const maxThreads = os.cpus().length;
if (numThreadsString) {
numThreads = Number(numThreadsString);
if (userInput) {
numThreads = Number(userInput);
if (Number.isNaN(numThreads)) {
throw new Error(`Invalid threads setting "${numThreadsString}", specified.`);
throw new Error(`Invalid threads setting "${userInput}", specified.`);
}
if (numThreads > maxThreads) {
core.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
logger.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
numThreads = maxThreads;
}
const minThreads = -maxThreads;
if (numThreads < minThreads) {
core.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
logger.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
numThreads = minThreads;
}
} else {