robojerk/codeql-action

Merge pull request #73 from github/add-analyze-threads-flag

Browse source 
Add threads flag to analyze action
This commit is contained in:
Alex Kalyvitis 2020-06-23 15:03:08 +02:00 • committed by GitHub
commit 98f8945cfb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 207 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

7
analyze/action.yml
View file

@ -4,6 +4,7 @@ author: 'GitHub'
inputs:
check_name:
description: The name of the check run to add text to.
required: false
output:
description: The path of the directory in which to save the SARIF results
required: false
@ -11,10 +12,14 @@ inputs:
upload:
description: Upload the SARIF file
required: false
default: true
default: "true"
ram:
description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
required: false
threads:
description: The number of threads to be used by CodeQL.
required: false
default: "1"
token:
default: ${{ github.token }}
matrix:

21
lib/finalize-db.js generated
View file

@ -11,7 +11,6 @@ const core = __importStar(require("@actions/core"));
const exec = __importStar(require("@actions/exec"));
const io = __importStar(require("@actions/io"));
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
const configUtils = __importStar(require("./config-utils"));
const externalQueries = __importStar(require("./external-queries"));
@ -37,23 +36,6 @@ function queryIsDisabled(language, query) {
return (DISABLED_BUILTIN_QUERIES[language] || [])
.some(disabledQuery => query.endsWith(disabledQuery));
}
function getMemoryFlag() {
let memoryToUseMegaBytes;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
}
}
else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
}
return "--ram=" + Math.floor(memoryToUseMegaBytes);
}
async function createdDBForScannedLanguages(codeqlCmd, databaseFolder) {
const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
if (scannedLanguages) {
@ -163,7 +145,8 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
await exec.exec(codeqlCmd, [
'database',
'analyze',
getMemoryFlag(),
util.getMemoryFlag(),
util.getThreadsFlag(),
path.join(databaseFolder, database),
'--format=sarif-latest',
'--output=' + sarifFile,

2
lib/finalize-db.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/finalize-db.test.js generated Normal file
View file

@ -0,0 +1,2 @@
"use strict";
//# sourceMappingURL=finalize-db.test.js.map

1
lib/finalize-db.test.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"finalize-db.test.js","sourceRoot":"","sources":["../src/finalize-db.test.ts"],"names":[],"mappings":""}

50
lib/util.js generated
View file

@ -354,4 +354,54 @@ async function withTmpDir(body) {
return result;
}
exports.withTmpDir = withTmpDir;
/**
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
* specified, the total available memory will be used minus 256 MB.
*
* @returns string
*/
function getMemoryFlag() {
let memoryToUseMegaBytes;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
}
}
else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
}
return "--ram=" + Math.floor(memoryToUseMegaBytes);
}
exports.getMemoryFlag = getMemoryFlag;
/**
* Get the codeql `--threads` value specified for the `threads` input. The value
* defaults to 1. The value will be capped to the number of available CPUs.
*
* @returns string
*/
function getThreadsFlag() {
let numThreads = 1;
const numThreadsString = core.getInput("threads");
if (numThreadsString) {
numThreads = Number(numThreadsString);
if (Number.isNaN(numThreads)) {
throw new Error(`Invalid threads setting "${numThreadsString}", specified.`);
}
const maxThreads = os.cpus().length;
if (numThreads > maxThreads) {
numThreads = maxThreads;
}
const minThreads = -maxThreads;
if (numThreads < minThreads) {
numThreads = minThreads;
}
}
return `--threads=${numThreads}`;
}
exports.getThreadsFlag = getThreadsFlag;
//# sourceMappingURL=util.js.map

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

39
lib/util.test.js generated
View file

@ -12,10 +12,49 @@ var __importStar = (this && this.__importStar) || function (mod) {
Object.defineProperty(exports, "__esModule", { value: true });
const ava_1 = __importDefault(require("ava"));
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const util = __importStar(require("./util"));
ava_1.default('getToolNames', t => {
const input = fs.readFileSync(__dirname + '/../src/testdata/tool-names.sarif', 'utf8');
const toolNames = util.getToolNames(input);
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
ava_1.default('getMemoryFlag() should return the correct --ram flag', t => {
const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
const tests = {
"": `--ram=${totalMem - 256}`,
"512": "--ram=512",
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_RAM'] = input;
const flag = util.getMemoryFlag();
t.deepEqual(flag, expectedFlag);
}
});
ava_1.default('getMemoryFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["-1", "hello!"]) {
process.env['INPUT_RAM'] = input;
t.throws(util.getMemoryFlag);
}
});
ava_1.default('getThreadsFlag() should return the correct --threads flag', t => {
const numCpus = os.cpus().length;
const tests = {
"0": "--threads=0",
"1": "--threads=1",
[`${numCpus + 1}`]: `--threads=${numCpus}`,
[`${-numCpus - 1}`]: `--threads=${-numCpus}`
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_THREADS'] = input;
const flag = util.getThreadsFlag();
t.deepEqual(flag, expectedFlag);
}
});
ava_1.default('getThreadsFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["hello!"]) {
process.env['INPUT_THREADS'] = input;
t.throws(util.getThreadsFlag);
}
});
//# sourceMappingURL=util.test.js.map

2
lib/util.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"util.test.js","sourceRoot":"","sources":["../src/util.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAuB;AACvB,uCAAyB;AAEzB,6CAA+B;AAE/B,aAAI,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE;IACvB,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,GAAG,mCAAmC,EAAE,MAAM,CAAC,CAAC;IACvF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,+BAA+B,EAAE,QAAQ,CAAC,CAAC,CAAC;AACtE,CAAC,CAAC,CAAC"}
{"version":3,"file":"util.test.js","sourceRoot":"","sources":["../src/util.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAuB;AACvB,uCAAyB;AACzB,uCAAyB;AAEzB,6CAA+B;AAE/B,aAAI,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE;IACvB,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,GAAG,mCAAmC,EAAE,MAAM,CAAC,CAAC;IACvF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,+BAA+B,EAAE,QAAQ,CAAC,CAAC,CAAC;AACtE,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAE/D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IAE3D,MAAM,KAAK,GAAG;QACZ,EAAE,EAAE,SAAS,QAAQ,GAAG,GAAG,EAAE;QAC7B,KAAK,EAAE,WAAW;KACnB,CAAC;IAEF,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAEzD,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;KACjC;AACH,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IAChE,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC;QACjC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;KAC9B;AACH,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,2DAA2D,EAAE,CAAC,CAAC,EAAE;IAEpE,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IAEjC,MAAM,KAAK,GAAG;QACZ,GAAG,EAAE,aAAa;QAClB,GAAG,EAAE,aAAa;QAClB,CAAC,GAAG,OAAO,GAAG,CAAC,EAAE,CAAC,EAAE,aAAa,OAAO,EAAE;QAC1C,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,OAAO,EAAE;KAC7C,CAAC;IAEF,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAEzD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC;QAErC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACnC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;KACjC;AACH,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IACjE,KAAK,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,EAAE;QAC9B,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC;QACrC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;KAC/B;AACH,CAAC,CAAC,CAAC"}

21
src/finalize-db.ts
View file

@ -2,7 +2,6 @@ import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as io from '@actions/io';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import * as configUtils from './config-utils';
@ -32,23 +31,6 @@ function queryIsDisabled(language, query): boolean {
.some(disabledQuery => query.endsWith(disabledQuery));
}
function getMemoryFlag(): string {
let memoryToUseMegaBytes: number;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
}
} else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
}
return "--ram=" + Math.floor(memoryToUseMegaBytes);
}
async function createdDBForScannedLanguages(codeqlCmd: string, databaseFolder: string) {
const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
if (scannedLanguages) {
@ -200,7 +182,8 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
await exec.exec(codeqlCmd, [
'database',
'analyze',
getMemoryFlag(),
util.getMemoryFlag(),
util.getThreadsFlag(),
path.join(databaseFolder, database),
'--format=sarif-latest',
'--output=' + sarifFile,

53
src/util.test.ts
View file

@ -1,5 +1,6 @@
import test from 'ava';
import * as fs from 'fs';
import * as os from "os";
import * as util from './util';
@ -8,3 +9,55 @@ test('getToolNames', t => {
const toolNames = util.getToolNames(input);
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
test('getMemoryFlag() should return the correct --ram flag', t => {
const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
const tests = {
"": `--ram=${totalMem - 256}`,
"512": "--ram=512",
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_RAM'] = input;
const flag = util.getMemoryFlag();
t.deepEqual(flag, expectedFlag);
}
});
test('getMemoryFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["-1", "hello!"]) {
process.env['INPUT_RAM'] = input;
t.throws(util.getMemoryFlag);
}
});
test('getThreadsFlag() should return the correct --threads flag', t => {
const numCpus = os.cpus().length;
const tests = {
"0": "--threads=0",
"1": "--threads=1",
[`${numCpus + 1}`]: `--threads=${numCpus}`,
[`${-numCpus - 1}`]: `--threads=${-numCpus}`
};
for (const [input, expectedFlag] of Object.entries(tests)) {
process.env['INPUT_THREADS'] = input;
const flag = util.getThreadsFlag();
t.deepEqual(flag, expectedFlag);
}
});
test('getThreadsFlag() throws if the ram input is < 0 or NaN', t => {
for (const input of ["hello!"]) {
process.env['INPUT_THREADS'] = input;
t.throws(util.getThreadsFlag);
}
});

49
src/util.ts
View file

@ -393,3 +393,52 @@ export async function withTmpDir<T>(body: (tmpDir: string) => Promise<T>): Promi
fs.rmdirSync(tmpDir, { recursive: true });
return result;
}
/**
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
* specified, the total available memory will be used minus 256 MB.
*
* @returns string
*/
export function getMemoryFlag(): string {
let memoryToUseMegaBytes: number;
const memoryToUseString = core.getInput("ram");
if (memoryToUseString) {
memoryToUseMegaBytes = Number(memoryToUseString);
if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
}
} else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
}
return "--ram=" + Math.floor(memoryToUseMegaBytes);
}
/**
* Get the codeql `--threads` value specified for the `threads` input. The value
* defaults to 1. The value will be capped to the number of available CPUs.
*
* @returns string
*/
export function getThreadsFlag(): string {
let numThreads = 1;
const numThreadsString = core.getInput("threads");
if (numThreadsString) {
numThreads = Number(numThreadsString);
if (Number.isNaN(numThreads)) {
throw new Error(`Invalid threads setting "${numThreadsString}", specified.`);
}
const maxThreads = os.cpus().length;
if (numThreads > maxThreads) {
numThreads = maxThreads;
}
const minThreads = -maxThreads;
if (numThreads < minThreads) {
numThreads = minThreads;
}
}
return `--threads=${numThreads}`;
}