robojerk/codeql-action

Merge branch 'main' into chrisgavin/default-bundle-version-json

Browse source
This commit is contained in:
Chris Gavin 2020-08-11 17:52:34 +01:00
commit 9c5f7640e5
No known key found for this signature in database
GPG key ID: 07F950B80C27E4DA
27 changed files with 107 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/autobuild.ts
View file

@ -1,7 +1,7 @@
import * as core from '@actions/core';
import { getCodeQL } from './codeql';
import * as sharedEnv from './shared-environment';
import { getCodeQL, isTracedLanguage } from './codeql';
import * as config_utils from './config-utils';
import * as util from './util';
interface AutobuildStatusReport extends util.StatusReportBase {
@ -37,16 +37,17 @@ async function run() {
let language;
try {
util.prepareLocalRunEnvironment();
if (util.should_abort('autobuild', true) ||
!await util.sendStatusReport(await util.createStatusReportBase('autobuild', 'starting', startedAt), true)) {
if (!await util.sendStatusReport(await util.createStatusReportBase('autobuild', 'starting', startedAt), true)) {
return;
}
const config = await config_utils.getConfig();
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = process.env[sharedEnv.CODEQL_ACTION_TRACED_LANGUAGES]?.split(',') || [];
const autobuildLanguages = config.languages.filter(isTracedLanguage);
language = autobuildLanguages[0];
if (!language) {

11
src/codeql.ts
View file

@ -2,7 +2,6 @@ import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as http from '@actions/http-client';
import { IHeaders } from '@actions/http-client/interfaces';
import * as io from '@actions/io';
import * as toolcache from '@actions/tool-cache';
import * as fs from 'fs';
import * as path from 'path';
@ -155,7 +154,7 @@ async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<s
throw err;
}
const pipeline = globalutil.promisify(stream.pipeline);
await io.mkdirP(path.dirname(dest));
fs.mkdirSync(path.dirname(dest), { recursive: true });
await pipeline(response.message, fs.createWriteStream(dest));
return dest;
}
@ -391,3 +390,11 @@ function getCodeQLForCmd(cmd: string): CodeQL {
}
};
}
export function isTracedLanguage(language: string): boolean {
return ['cpp', 'java', 'csharp'].includes(language);
}
export function isScannedLanguage(language: string): boolean {
return !isTracedLanguage(language);
}

15
src/config-utils.ts
View file

@ -1,5 +1,4 @@
import * as core from '@actions/core';
import * as io from '@actions/io';
import * as fs from 'fs';
import * as yaml from 'js-yaml';
import * as path from 'path';
@ -709,18 +708,11 @@ async function getRemoteConfig(configFile: string): Promise<UserConfig> {
return yaml.safeLoad(Buffer.from(fileContents, 'base64').toString('binary'));
}
/**
* Get the directory where the parsed config will be stored.
*/
function getPathToParsedConfigFolder(): string {
return util.getRequiredEnvParam('RUNNER_TEMP');
}
/**
* Get the file path where the parsed config will be stored.
*/
export function getPathToParsedConfigFile(): string {
return path.join(getPathToParsedConfigFolder(), 'config');
return path.join(util.getRequiredEnvParam('RUNNER_TEMP'), 'config');
}
/**
@ -728,8 +720,9 @@ export function getPathToParsedConfigFile(): string {
*/
async function saveConfig(config: Config) {
const configString = JSON.stringify(config);
await io.mkdirP(getPathToParsedConfigFolder());
fs.writeFileSync(getPathToParsedConfigFile(), configString, 'utf8');
const configFile = getPathToParsedConfigFile();
fs.mkdirSync(path.dirname(configFile), { recursive: true });
fs.writeFileSync(configFile, configString, 'utf8');
core.debug('Saved config:');
core.debug(configString);
}

21
src/finalize-db.ts
View file

@ -1,9 +1,8 @@
import * as core from '@actions/core';
import * as io from '@actions/io';
import * as fs from 'fs';
import * as path from 'path';
import { getCodeQL } from './codeql';
import { getCodeQL, isScannedLanguage } from './codeql';
import * as configUtils from './config-utils';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
@ -56,11 +55,10 @@ async function sendStatusReport(
await util.sendStatusReport(statusReport);
}
async function createdDBForScannedLanguages(databaseFolder: string) {
const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
if (scannedLanguages) {
const codeql = getCodeQL();
for (const language of scannedLanguages.split(',')) {
async function createdDBForScannedLanguages(databaseFolder: string, config: configUtils.Config) {
const codeql = getCodeQL();
for (const language of config.languages) {
if (isScannedLanguage(language)) {
core.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
core.endGroup();
@ -69,7 +67,7 @@ async function createdDBForScannedLanguages(databaseFolder: string) {
}
async function finalizeDatabaseCreation(databaseFolder: string, config: configUtils.Config) {
await createdDBForScannedLanguages(databaseFolder);
await createdDBForScannedLanguages(databaseFolder, config);
const codeql = getCodeQL();
for (const language of config.languages) {
@ -126,8 +124,7 @@ async function run() {
let uploadStats: upload_lib.UploadStatusReport | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (util.should_abort('finish', true) ||
!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
if (!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
return;
}
const config = await configUtils.getConfig();
@ -135,10 +132,10 @@ async function run() {
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const databaseFolder = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_DATABASE_DIR);
const databaseFolder = util.getCodeQLDatabasesDir();
const sarifFolder = core.getInput('output');
await io.mkdirP(sarifFolder);
fs.mkdirSync(sarifFolder, { recursive: true });
core.info('Finalizing database creation');
await finalizeDatabaseCreation(databaseFolder, config);

37
src/setup-tracer.ts
View file

@ -1,13 +1,11 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as io from '@actions/io';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import { CodeQL, isTracedLanguage, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import * as sharedEnv from './shared-environment';
import * as util from './util';
type TracerConfig = {
@ -54,7 +52,7 @@ async function tracerConfig(
return info;
}
function concatTracerConfigs(configs: { [lang: string]: TracerConfig }): TracerConfig {
function concatTracerConfigs(configs: TracerConfig[]): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
@ -62,7 +60,7 @@ function concatTracerConfigs(configs: { [lang: string]: TracerConfig }): TracerC
const env: { [key: string]: string; } = {};
let copyExecutables = false;
let envSize = 0;
for (let v of Object.values(configs)) {
for (const v of configs) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
@ -178,8 +176,7 @@ async function run() {
try {
util.prepareLocalRunEnvironment();
if (util.should_abort('init', false) ||
!await util.sendStatusReport(await util.createStatusReportBase('init', 'starting', startedAt), true)) {
if (!await util.sendStatusReport(await util.createStatusReportBase('init', 'starting', startedAt), true)) {
return;
}
@ -215,11 +212,10 @@ async function run() {
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
const databaseFolder = path.resolve(util.getRequiredEnvParam('RUNNER_TEMP'), 'codeql_databases');
await io.mkdirP(databaseFolder);
const databaseFolder = util.getCodeQLDatabasesDir();
fs.mkdirSync(databaseFolder, { recursive: true });
let tracedLanguages: { [key: string]: TracerConfig } = {};
let scannedLanguages: string[] = [];
let tracedLanguageConfigs: TracerConfig[] = [];
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
const languageDatabase = path.join(databaseFolder, language);
@ -227,16 +223,13 @@ async function run() {
// Init language database
await codeql.databaseInit(languageDatabase, language, sourceRoot);
// TODO: add better detection of 'traced languages' instead of using a hard coded list
if (['cpp', 'java', 'csharp'].includes(language)) {
if (isTracedLanguage(language)) {
const config: TracerConfig = await tracerConfig(codeql, languageDatabase);
tracedLanguages[language] = config;
} else {
scannedLanguages.push(language);
tracedLanguageConfigs.push(config);
}
}
const tracedLanguageKeys = Object.keys(tracedLanguages);
if (tracedLanguageKeys.length > 0) {
const mainTracerConfig = concatTracerConfigs(tracedLanguages);
if (tracedLanguageConfigs.length > 0) {
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs);
if (mainTracerConfig.spec) {
for (let entry of Object.entries(mainTracerConfig.env)) {
core.exportVariable(entry[0], entry[1]);
@ -260,13 +253,6 @@ async function run() {
}
}
}
core.exportVariable(sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES, scannedLanguages.join(','));
core.exportVariable(sharedEnv.CODEQL_ACTION_TRACED_LANGUAGES, tracedLanguageKeys.join(','));
// TODO: make this a "private" environment variable of the action
core.exportVariable(sharedEnv.CODEQL_ACTION_DATABASE_DIR, databaseFolder);
} catch (error) {
core.setFailed(error.message);
console.log(error);
@ -279,7 +265,6 @@ async function run() {
return;
}
await sendSuccessStatusReport(startedAt, config);
core.exportVariable(sharedEnv.CODEQL_ACTION_INIT_COMPLETED, 'true');
}
run().catch(e => {

6
src/shared-environment.ts
View file

@ -1,13 +1,7 @@
export const CODEQL_ACTION_DATABASE_DIR = 'CODEQL_ACTION_DATABASE_DIR';
export const CODEQL_ACTION_ANALYSIS_KEY = 'CODEQL_ACTION_ANALYSIS_KEY';
export const ODASA_TRACER_CONFIGURATION = 'ODASA_TRACER_CONFIGURATION';
export const CODEQL_ACTION_SCANNED_LANGUAGES = 'CODEQL_ACTION_SCANNED_LANGUAGES';
export const CODEQL_ACTION_TRACED_LANGUAGES = 'CODEQL_ACTION_TRACED_LANGUAGES';
// The time at which the first action (normally init) started executing.
// If a workflow invokes a different action without first invoking the init
// action (i.e. the upload action is being used by a third-party integrator)
// then this variable will be assigned the start time of the action invoked
// rather that the init action.
export const CODEQL_WORKFLOW_STARTED_AT = 'CODEQL_WORKFLOW_STARTED_AT';
// Populated when the init action completes successfully
export const CODEQL_ACTION_INIT_COMPLETED = 'CODEQL_ACTION_INIT_COMPLETED';

3
src/upload-sarif.ts
View file

@ -16,8 +16,7 @@ async function sendSuccessStatusReport(startedAt: Date, uploadStats: upload_lib.
async function run() {
const startedAt = new Date();
if (util.should_abort('upload-sarif', false) ||
!await util.sendStatusReport(await util.createStatusReportBase('upload-sarif', 'starting', startedAt), true)) {
if (!await util.sendStatusReport(await util.createStatusReportBase('upload-sarif', 'starting', startedAt), true)) {
return;
}

38
src/util.ts
View file

@ -27,31 +27,6 @@ export function isEnterprise(): boolean {
return getInstanceAPIURL() !== GITHUB_DOTCOM_API_URL;
}
/**
* Should the current action be aborted?
*
* This method should be called at the start of all CodeQL actions and they
* should abort cleanly if this returns true without failing the action.
* This method will call `core.setFailed` if necessary.
*/
export function should_abort(actionName: string, requireInitActionHasRun: boolean): boolean {
// Check that required aspects of the environment are present
const ref = process.env['GITHUB_REF'];
if (ref === undefined) {
core.setFailed('GITHUB_REF must be set.');
return true;
}
// If the init action is required, then check the it completed successfully.
if (requireInitActionHasRun && process.env[sharedEnv.CODEQL_ACTION_INIT_COMPLETED] === undefined) {
core.setFailed('The CodeQL ' + actionName + ' action cannot be used unless the CodeQL init action is run first. Aborting.');
return true;
}
return false;
}
/**
* Get an environment parameter, but throw an error if it is not set.
*/
@ -144,7 +119,9 @@ async function getWorkflowPath(): Promise<string> {
* the github API, but after that the result will be cached.
*/
export async function getAnalysisKey(): Promise<string> {
let analysisKey = process.env[sharedEnv.CODEQL_ACTION_ANALYSIS_KEY];
const analysisKeyEnvVar = 'CODEQL_ACTION_ANALYSIS_KEY';
let analysisKey = process.env[analysisKeyEnvVar];
if (analysisKey !== undefined) {
return analysisKey;
}
@ -153,7 +130,7 @@ export async function getAnalysisKey(): Promise<string> {
const jobName = getRequiredEnvParam('GITHUB_JOB');
analysisKey = workflowPath + ':' + jobName;
core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey);
core.exportVariable(analysisKeyEnvVar, analysisKey);
return analysisKey;
}
@ -413,3 +390,10 @@ export function getThreadsFlag(): string {
}
return `--threads=${numThreads}`;
}
/**
* Get the directory where CodeQL databases should be placed.
*/
export function getCodeQLDatabasesDir() {
return path.resolve(getRequiredEnvParam('RUNNER_TEMP'), 'codeql_databases');
}