Add --ram and --threads args

This commit is contained in:
Robert Brignull 2020-09-01 14:13:10 +01:00
parent 09fb3ec514
commit 4c00c68d14
18 changed files with 81 additions and 65 deletions

View file

@ -49,6 +49,8 @@ async function run() {
core.getInput('upload') === 'true',
'actions',
core.getInput('output'),
util.getMemoryFlag(core.getInput('ram')),
util.getThreadsFlag(core.getInput('threads'), logger),
config,
logger);

View file

@ -77,6 +77,8 @@ async function finalizeDatabaseCreation(
// Runs queries and creates sarif files in the given folder
async function runQueries(
sarifFolder: string,
memoryFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<QueriesStatusReport> {
@ -100,7 +102,7 @@ async function runQueries(
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, threadsFlag);
logger.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
logger.endGroup();
@ -130,6 +132,8 @@ export async function runAnalyze(
doUpload: boolean,
mode: util.Mode,
outputDir: string,
memoryFlag: string,
threadsFlag: string,
config: configUtils.Config,
logger: Logger): Promise<AnalysisStatusReport> {
@ -142,7 +146,7 @@ export async function runAnalyze(
await finalizeDatabaseCreation(config, logger);
logger.info('Analyzing database');
const queriesStats = await runQueries(outputDir, config, logger);
const queriesStats = await runQueries(outputDir, memoryFlag, threadsFlag, config, logger);
if (!doUpload) {
logger.info('Not uploading results');

View file

@ -74,7 +74,12 @@ export interface CodeQL {
/**
* Run 'codeql database analyze'.
*/
databaseAnalyze(databasePath: string, sarifFile: string, querySuite: string): Promise<void>;
databaseAnalyze(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
threadsFlag: string): Promise<void>;
}
export interface ResolveQueriesOutput {
@ -456,12 +461,18 @@ function getCodeQLForCmd(cmd: string): CodeQL {
return JSON.parse(output);
},
databaseAnalyze: async function(databasePath: string, sarifFile: string, querySuite: string) {
databaseAnalyze: async function(
databasePath: string,
sarifFile: string,
querySuite: string,
memoryFlag: string,
threadsFlag: string) {
await new toolrunnner.ToolRunner(cmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
memoryFlag,
threadsFlag,
databasePath,
'--format=sarif-latest',
'--output=' + sarifFile,

View file

@ -12,6 +12,7 @@ import { Language, parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import { getMemoryFlag, getThreadsFlag } from './util';
const program = new Command();
program.version('0.0.1');
@ -229,6 +230,8 @@ interface AnalyzeArgs {
checkoutPath: string | undefined;
upload: boolean;
outputDir: string | undefined;
ram: string | undefined;
threads: string | undefined;
tempDir: string | undefined;
debug: boolean;
}
@ -244,6 +247,9 @@ program
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--no-upload', 'Do not upload results after analysis.', false)
.option('--output-dir <dir>', 'Directory to output SARIF files to. Default is in the temp directory.')
.option('--ram <ram>', 'Amount of memory to use when running queries. Default is to use all available memory.')
.option('--threads <threads>', 'Number of threads to use when running queries. ' +
'Default is to use all available cores.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.action(async (cmd: AnalyzeArgs) => {
@ -270,6 +276,8 @@ program
cmd.upload,
'runner',
outputDir,
getMemoryFlag(cmd.ram),
getThreadsFlag(cmd.threads, logger),
config,
logger);
} catch (e) {

View file

@ -2,6 +2,7 @@ import test from 'ava';
import * as fs from 'fs';
import * as os from "os";
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
@ -23,18 +24,14 @@ test('getMemoryFlag() should return the correct --ram flag', t => {
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_RAM'] = input;
const flag = util.getMemoryFlag();
const flag = util.getMemoryFlag(input);
t.deepEqual(flag, expectedFlag);
}
});
test('getMemoryFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["-1", "hello!"]) {
process.env['INPUT_RAM'] = input;
t.throws(util.getMemoryFlag);
t.throws(() => util.getMemoryFlag(input));
}
});
@ -50,17 +47,13 @@ test('getThreadsFlag() should return the correct --threads flag', t => {
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_THREADS'] = input;
const flag = util.getThreadsFlag();
const flag = util.getThreadsFlag(input, getRunnerLogger(true));
t.deepEqual(flag, expectedFlag);
}
});
test('getThreadsFlag() throws if the threads input is not an integer', t => {
process.env['INPUT_THREADS'] = "hello!";
t.throws(util.getThreadsFlag);
t.throws(() => util.getThreadsFlag("hello!", getRunnerLogger(true)));
});
test('getRef() throws on the empty string', t => {

View file

@ -6,6 +6,7 @@ import * as path from 'path';
import * as api from './api-client';
import { Language } from './languages';
import { Logger } from './logging';
import * as sharedEnv from './shared-environment';
/**
@ -369,13 +370,12 @@ export async function withTmpDir<T>(body: (tmpDir: string) => Promise<T>): Promi
*
* @returns string
*/
export function getMemoryFlag(): string {
export function getMemoryFlag(userInput: string | undefined): string {
let memoryToUseMegaBytes: number;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
throw new Error("Invalid RAM setting \"" + userInput + "\", specified.");
}
} else {
const totalMemoryBytes = os.totalmem();
@ -394,22 +394,21 @@ export function getMemoryFlag(): string {
*
* @returns string
*/
export function getThreadsFlag(): string {
export function getThreadsFlag(userInput: string | undefined, logger: Logger): string {
let numThreads: number;
const numThreadsString = core.getInput("threads");
const maxThreads = os.cpus().length;
if (numThreadsString) {
numThreads = Number(numThreadsString);
if (userInput) {
numThreads = Number(userInput);
if (Number.isNaN(numThreads)) {
throw new Error(`Invalid threads setting "${numThreadsString}", specified.`);
throw new Error(`Invalid threads setting "${userInput}", specified.`);
}
if (numThreads > maxThreads) {
core.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
logger.info(`Clamping desired number of threads (${numThreads}) to max available (${maxThreads}).`);
numThreads = maxThreads;
}
const minThreads = -maxThreads;
if (numThreads < minThreads) {
core.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
logger.info(`Clamping desired number of free threads (${numThreads}) to max available (${minThreads}).`);
numThreads = minThreads;
}
} else {