robojerk/codeql-action

Merge pull request #93 from github/codeql_mock

Browse source 
Introduce wrapper around codeql
This commit is contained in:
Robert 2020-07-06 10:38:41 +01:00 • committed by GitHub
commit 9da537eb33
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 563 additions and 295 deletions
Download patch file Download diff file Expand all files Collapse all files

17
lib/autobuild.js generated
View file

@ -8,8 +8,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(require("@actions/core"));
const exec = __importStar(require("@actions/exec"));
const path = __importStar(require("path"));
const codeql_1 = require("./codeql");
const sharedEnv = __importStar(require("./shared-environment"));
const util = __importStar(require("./util"));
async function run() {
@ -33,18 +32,8 @@ async function run() {
core.warning(`We will only automatically build ${language} code. If you wish to scan ${autobuildLanguages.slice(1).join(' and ')}, you must replace this block with custom build steps.`);
}
core.startGroup(`Attempting to automatically build ${language} code`);
// TODO: share config accross actions better via env variables
const codeqlCmd = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_CMD);
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
const autobuildCmd = path.join(path.dirname(codeqlCmd), language, 'tools', cmdName);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
await exec.exec(autobuildCmd);
const codeQL = codeql_1.getCodeQL();
await codeQL.runAutobuild(language);
core.endGroup();
}
catch (error) {

2
lib/autobuild.js.map
View file

@ -1 +1 @@
{"version":3,"file":"autobuild.js","sourceRoot":"","sources":["../src/autobuild.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AACtC,oDAAsC;AACtC,2CAA6B;AAE7B,gEAAkD;AAClD,6CAA+B;AAE/B,KAAK,UAAU,GAAG;;IAChB,IAAI;QACF,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,EAAE;YACzF,OAAO;SACR;QAED,0CAA0C;QAC1C,mFAAmF;QACnF,oFAAoF;QACpF,4EAA4E;QAC5E,MAAM,kBAAkB,GAAG,OAAA,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,8BAA8B,CAAC,0CAAE,KAAK,CAAC,GAAG,MAAK,EAAE,CAAC;QACnG,MAAM,QAAQ,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;QAEvC,IAAI,CAAC,QAAQ,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;YAC7E,OAAO;SACR;QAED,IAAI,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;QAE7D,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;YACjC,IAAI,CAAC,OAAO,CAAC,oCAAoC,QAAQ,8BAA8B,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC;SAC3L;QAED,IAAI,CAAC,UAAU,CAAC,qCAAqC,QAAQ,OAAO,CAAC,CAAC;QACtE,8DAA8D;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAExE,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC;QAChF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAGpF,+DAA+D;QAC/D,0FAA0F;QAC1F,qDAAqD;QACrD,8EAA8E;QAC9E,gHAAgH;QAChH,IAAI,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,wBAAwB,EAAE,+BAA+B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE1I,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ,EAAE,CAAC;KAEjB;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,kIAAkI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QACnK,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO;KACR;IAED,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;AAChD,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,4BAA4B,GAAG,CAAC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
{"version":3,"file":"autobuild.js","sourceRoot":"","sources":["../src/autobuild.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAEtC,qCAAqC;AACrC,gEAAkD;AAClD,6CAA+B;AAE/B,KAAK,UAAU,GAAG;;IAChB,IAAI;QACF,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,EAAE;YACzF,OAAO;SACR;QAED,0CAA0C;QAC1C,mFAAmF;QACnF,oFAAoF;QACpF,4EAA4E;QAC5E,MAAM,kBAAkB,GAAG,OAAA,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,8BAA8B,CAAC,0CAAE,KAAK,CAAC,GAAG,MAAK,EAAE,CAAC;QACnG,MAAM,QAAQ,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;QAEvC,IAAI,CAAC,QAAQ,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;YAC7E,OAAO;SACR;QAED,IAAI,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;QAE7D,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;YACjC,IAAI,CAAC,OAAO,CAAC,oCAAoC,QAAQ,8BAA8B,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC;SAC3L;QAED,IAAI,CAAC,UAAU,CAAC,qCAAqC,QAAQ,OAAO,CAAC,CAAC;QACtE,MAAM,MAAM,GAAG,kBAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAEpC,IAAI,CAAC,QAAQ,EAAE,CAAC;KAEjB;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,kIAAkI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QACnK,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO;KACR;IAED,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;AAChD,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,4BAA4B,GAAG,CAAC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}

184
lib/codeql.js generated Normal file
View file

@ -0,0 +1,184 @@
"use strict";
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(require("@actions/core"));
const exec = __importStar(require("@actions/exec"));
const toolcache = __importStar(require("@actions/tool-cache"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const semver = __importStar(require("semver"));
const util = __importStar(require("./util"));
/**
* Environment variable used to store the location of the CodeQL CLI executable.
* Value is set by setupCodeQL and read by getCodeQL.
*/
const CODEQL_ACTION_CMD = "CODEQL_ACTION_CMD";
async function setupCodeQL() {
try {
const codeqlURL = core.getInput('tools', { required: true });
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
}
else {
const codeqlPath = await toolcache.downloadTool(codeqlURL);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
}
let codeqlCmd = path.join(codeqlFolder, 'codeql', 'codeql');
if (process.platform === 'win32') {
codeqlCmd += ".exe";
}
else if (process.platform !== 'linux' && process.platform !== 'darwin') {
throw new Error("Unsupported plaform: " + process.platform);
}
core.exportVariable(CODEQL_ACTION_CMD, codeqlCmd);
return getCodeQLForCmd(codeqlCmd);
}
catch (e) {
core.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
exports.setupCodeQL = setupCodeQL;
function getCodeQLURLVersion(url) {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Version could not be inferred`);
}
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`);
}
return s;
}
exports.getCodeQLURLVersion = getCodeQLURLVersion;
function getCodeQL() {
const codeqlCmd = util.getRequiredEnvParam(CODEQL_ACTION_CMD);
return getCodeQLForCmd(codeqlCmd);
}
exports.getCodeQL = getCodeQL;
function getCodeQLForCmd(cmd) {
return {
getDir: function () {
return path.dirname(cmd);
},
printVersion: async function () {
await exec.exec(cmd, [
'version',
'--format=json'
]);
},
getTracerEnv: async function (databasePath, compilerSpec) {
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
await exec.exec(cmd, [
'database',
'trace-command',
databasePath,
...compilerSpecArg,
process.execPath,
path.resolve(__dirname, 'tracer-env.js'),
envFile
]);
return JSON.parse(fs.readFileSync(envFile, 'utf-8'));
},
databaseInit: async function (databasePath, language, sourceRoot) {
await exec.exec(cmd, [
'database',
'init',
databasePath,
'--language=' + language,
'--source-root=' + sourceRoot,
]);
},
runAutobuild: async function (language) {
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
const autobuildCmd = path.join(path.dirname(cmd), language, 'tools', cmdName);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
await exec.exec(autobuildCmd);
},
extractScannedLanguage: async function (databasePath, language) {
// Get extractor location
let extractorPath = '';
await exec.exec(cmd, [
'resolve',
'extractor',
'--format=json',
'--language=' + language
], {
silent: true,
listeners: {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
const traceCommand = path.resolve(JSON.parse(extractorPath), 'tools', 'autobuild' + ext);
// Run trace command
await exec.exec(cmd, [
'database',
'trace-command',
databasePath,
'--',
traceCommand
]);
},
finalizeDatabase: async function (databasePath) {
await exec.exec(cmd, [
'database',
'finalize',
databasePath
]);
},
resolveQueries: async function (queries) {
let output = '';
await exec.exec(cmd, [
'resolve',
'queries',
...queries,
'--format=bylanguage'
], {
listeners: {
stdout: (data) => {
output += data.toString();
}
}
});
return JSON.parse(output);
},
databaseAnalyze: async function (databasePath, sarifFile, querySuite) {
await exec.exec(cmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
databasePath,
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
querySuite
]);
}
};
}
//# sourceMappingURL=codeql.js.map

1
lib/codeql.js.map Normal file
View file

File diff suppressed because one or more lines are too long

60
lib/codeql.test.js generated Normal file
View file

@ -0,0 +1,60 @@
"use strict";
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const toolcache = __importStar(require("@actions/tool-cache"));
const ava_1 = __importDefault(require("ava"));
const nock_1 = __importDefault(require("nock"));
const path = __importStar(require("path"));
const codeql = __importStar(require("./codeql"));
const testing_utils_1 = require("./testing-utils");
const util = __importStar(require("./util"));
testing_utils_1.silenceDebugOutput(ava_1.default);
ava_1.default('download codeql bundle cache', async (t) => {
await util.withTmpDir(async (tmpDir) => {
process.env['GITHUB_WORKSPACE'] = tmpDir;
process.env['RUNNER_TEMP'] = path.join(tmpDir, 'temp');
process.env['RUNNER_TOOL_CACHE'] = path.join(tmpDir, 'cache');
const versions = ['20200601', '20200610'];
for (let i = 0; i < versions.length; i++) {
const version = versions[i];
nock_1.default('https://example.com')
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(200, path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`));
process.env['INPUT_TOOLS'] = `https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
await codeql.setupCodeQL();
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
}
const cachedVersions = toolcache.findAllVersions('CodeQL');
t.is(cachedVersions.length, 2);
});
});
ava_1.default('parse codeql bundle url version', t => {
const tests = {
'20200601': '0.0.0-20200601',
'20200601.0': '0.0.0-20200601.0',
'20200601.0.0': '20200601.0.0',
'1.2.3': '1.2.3',
'1.2.3-alpha': '1.2.3-alpha',
'1.2.3-beta.1': '1.2.3-beta.1',
};
for (const [version, expectedVersion] of Object.entries(tests)) {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = codeql.getCodeQLURLVersion(url);
t.deepEqual(parsedVersion, expectedVersion);
}
catch (e) {
t.fail(e.message);
}
}
});
//# sourceMappingURL=codeql.test.js.map

1
lib/codeql.test.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"codeql.test.js","sourceRoot":"","sources":["../src/codeql.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+DAAiD;AACjD,8CAAuB;AACvB,gDAAwB;AACxB,2CAA6B;AAE7B,iDAAmC;AACnC,mDAAmD;AACnD,6CAA+B;AAE/B,kCAAkB,CAAC,aAAI,CAAC,CAAC;AAEzB,aAAI,CAAC,8BAA8B,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAE7C,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;QAEnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,MAAM,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE9D,MAAM,QAAQ,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAE5B,cAAI,CAAC,qBAAqB,CAAC;iBACxB,GAAG,CAAC,2BAA2B,OAAO,uBAAuB,CAAC;iBAC9D,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uCAAuC,CAAC,CAAC,CAAC;YAGrF,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,8CAA8C,OAAO,uBAAuB,CAAC;YAE1G,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC;YAE3B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAC,CAAC,CAAC;SACxD;QAED,MAAM,cAAc,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAE3D,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,iCAAiC,EAAE,CAAC,CAAC,EAAE;IAE1C,MAAM,KAAK,GAAG;QACZ,UAAU,EAAE,gBAAgB;QAC5B,YAAY,EAAE,kBAAkB;QAChC,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,OAAO;QAChB,aAAa,EAAE,aAAa;QAC5B,cAAc,EAAE,cAAc;KAC/B,CAAC;IAEF,KAAK,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAC9D,MAAM,GAAG,GAAG,wCAAwC,OAAO,MAAM,CAAC;QAElE,IAAI;YACF,MAAM,aAAa,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YACtD,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;SAC7C;QAAC,OAAO,CAAC,EAAE;YACV,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACnB;KACF;AACH,CAAC,CAAC,CAAC"}

73
lib/finalize-db.js generated
View file

@ -8,10 +8,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(require("@actions/core"));
const exec = __importStar(require("@actions/exec"));
const io = __importStar(require("@actions/io"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const codeql_1 = require("./codeql");
const configUtils = __importStar(require("./config-utils"));
const externalQueries = __importStar(require("./external-queries"));
const sharedEnv = __importStar(require("./shared-environment"));
@ -36,57 +36,30 @@ function queryIsDisabled(language, query) {
return (DISABLED_BUILTIN_QUERIES[language] || [])
.some(disabledQuery => query.endsWith(disabledQuery));
}
async function createdDBForScannedLanguages(codeqlCmd, databaseFolder) {
async function createdDBForScannedLanguages(databaseFolder) {
const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
if (scannedLanguages) {
const codeql = codeql_1.getCodeQL();
for (const language of scannedLanguages.split(',')) {
core.startGroup('Extracting ' + language);
// Get extractor location
let extractorPath = '';
await exec.exec(codeqlCmd, ['resolve', 'extractor', '--format=json', '--language=' + language], {
silent: true,
listeners: {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
const traceCommand = path.resolve(JSON.parse(extractorPath), 'tools', 'autobuild' + ext);
// Run trace command
await exec.exec(codeqlCmd, ['database', 'trace-command', path.join(databaseFolder, language), '--', traceCommand]);
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(codeqlCmd, databaseFolder) {
await createdDBForScannedLanguages(codeqlCmd, databaseFolder);
async function finalizeDatabaseCreation(databaseFolder) {
await createdDBForScannedLanguages(databaseFolder);
const languages = process.env[sharedEnv.CODEQL_ACTION_LANGUAGES] || '';
const codeql = codeql_1.getCodeQL();
for (const language of languages.split(',')) {
core.startGroup('Finalizing ' + language);
await exec.exec(codeqlCmd, ['database', 'finalize', path.join(databaseFolder, language)]);
await codeql.finalizeDatabase(path.join(databaseFolder, language));
core.endGroup();
}
}
async function runResolveQueries(codeqlCmd, queries) {
let output = '';
const options = {
listeners: {
stdout: (data) => {
output += data.toString();
}
}
};
await exec.exec(codeqlCmd, [
'resolve',
'queries',
...queries,
'--format=bylanguage'
], options);
return JSON.parse(output);
}
async function resolveQueryLanguages(codeqlCmd, config) {
async function resolveQueryLanguages(config) {
let res = new Map();
const codeql = codeql_1.getCodeQL();
if (!config.disableDefaultQueries || config.additionalSuites.length !== 0) {
const suites = [];
for (const language of await util.getLanguages()) {
@ -97,7 +70,7 @@ async function resolveQueryLanguages(codeqlCmd, config) {
suites.push(language + '-' + additionalSuite + '.qls');
}
}
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, suites);
const resolveQueriesOutputObject = await codeql.resolveQueries(suites);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
if (res[language] === undefined) {
res[language] = [];
@ -106,7 +79,7 @@ async function resolveQueryLanguages(codeqlCmd, config) {
}
}
if (config.additionalQueries.length !== 0) {
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, config.additionalQueries);
const resolveQueriesOutputObject = await codeql.resolveQueries(config.additionalQueries);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
if (res[language] === undefined) {
res[language] = [];
@ -127,8 +100,9 @@ async function resolveQueryLanguages(codeqlCmd, config) {
return res;
}
// Runs queries and creates sarif files in the given folder
async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
const queriesPerLanguage = await resolveQueryLanguages(codeqlCmd, config);
async function runQueries(databaseFolder, sarifFolder, config) {
const queriesPerLanguage = await resolveQueryLanguages(config);
const codeql = codeql_1.getCodeQL();
for (let database of fs.readdirSync(databaseFolder)) {
core.startGroup('Analyzing ' + database);
const queries = queriesPerLanguage[database] || [];
@ -142,17 +116,7 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + database + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, database + '.sarif');
await exec.exec(codeqlCmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
path.join(databaseFolder, database),
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
querySuite
]);
await codeql.databaseAnalyze(path.join(databaseFolder, database), sarifFile, querySuite);
core.debug('SARIF results for database ' + database + ' created at "' + sarifFile + '"');
core.endGroup();
}
@ -165,15 +129,14 @@ async function run() {
const config = await configUtils.loadConfig();
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const codeqlCmd = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_CMD);
const databaseFolder = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_DATABASE_DIR);
const sarifFolder = core.getInput('output');
await io.mkdirP(sarifFolder);
core.info('Finalizing database creation');
await finalizeDatabaseCreation(codeqlCmd, databaseFolder);
await finalizeDatabaseCreation(databaseFolder);
await externalQueries.checkoutExternalQueries(config);
core.info('Analyzing database');
await runQueries(codeqlCmd, databaseFolder, sarifFolder, config);
await runQueries(databaseFolder, sarifFolder, config);
if ('true' === core.getInput('upload')) {
if (!await upload_lib.upload(sarifFolder)) {
await util.reportActionFailed('finish', 'upload');

2
lib/finalize-db.js.map
View file

File diff suppressed because one or more lines are too long

30
lib/setup-tracer.js generated
View file

@ -13,8 +13,8 @@ const io = __importStar(require("@actions/io"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const analysisPaths = __importStar(require("./analysis-paths"));
const codeql_1 = require("./codeql");
const configUtils = __importStar(require("./config-utils"));
const setuptools = __importStar(require("./setup-tools"));
const sharedEnv = __importStar(require("./shared-environment"));
const util = __importStar(require("./util"));
const CRITICAL_TRACER_VARS = new Set(['SEMMLE_PRELOAD_libtrace',
@ -28,12 +28,7 @@ const CRITICAL_TRACER_VARS = new Set(['SEMMLE_PRELOAD_libtrace',
'SEMMLE_JAVA_TOOL_OPTIONS'
]);
async function tracerConfig(codeql, database, compilerSpec) {
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
let envFile = path.resolve(database, 'working', 'env.tmp');
await exec.exec(codeql.cmd, ['database', 'trace-command', database,
...compilerSpecArg,
process.execPath, path.resolve(__dirname, 'tracer-env.js'), envFile]);
const env = JSON.parse(fs.readFileSync(envFile, 'utf-8'));
const env = await codeql.getTracerEnv(database, compilerSpec);
const config = env['ODASA_TRACER_CONFIGURATION'];
const info = { spec: config, env: {} };
// Extract critical tracer variables from the environment
@ -151,8 +146,8 @@ async function run() {
try {
const sourceRoot = path.resolve();
core.startGroup('Setup CodeQL tools');
const codeqlSetup = await setuptools.setupCodeQL();
await exec.exec(codeqlSetup.cmd, ['version', '--format=json']);
const codeql = await codeql_1.setupCodeQL();
await codeql.printVersion();
core.endGroup();
// Forward Go flags
const goFlags = process.env['GOFLAGS'];
@ -171,16 +166,10 @@ async function run() {
for (let language of languages) {
const languageDatabase = path.join(databaseFolder, language);
// Init language database
await exec.exec(codeqlSetup.cmd, [
'database',
'init',
languageDatabase,
'--language=' + language,
'--source-root=' + sourceRoot,
]);
await codeql.databaseInit(languageDatabase, language, sourceRoot);
// TODO: add better detection of 'traced languages' instead of using a hard coded list
if (['cpp', 'java', 'csharp'].includes(language)) {
const config = await tracerConfig(codeqlSetup, languageDatabase);
const config = await tracerConfig(codeql, languageDatabase);
tracedLanguages[language] = config;
}
else {
@ -196,16 +185,16 @@ async function run() {
}
core.exportVariable('ODASA_TRACER_CONFIGURATION', mainTracerConfig.spec);
if (process.platform === 'darwin') {
core.exportVariable('DYLD_INSERT_LIBRARIES', path.join(codeqlSetup.tools, 'osx64', 'libtrace.dylib'));
core.exportVariable('DYLD_INSERT_LIBRARIES', path.join(codeql.getDir(), 'tools', 'osx64', 'libtrace.dylib'));
}
else if (process.platform === 'win32') {
await exec.exec('powershell', [
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(codeqlSetup.tools, 'win64', 'tracer.exe'),
path.resolve(codeql.getDir(), 'tools', 'win64', 'tracer.exe'),
], { env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
}
else {
core.exportVariable('LD_PRELOAD', path.join(codeqlSetup.tools, 'linux64', '${LIB}trace.so'));
core.exportVariable('LD_PRELOAD', path.join(codeql.getDir(), 'tools', 'linux64', '${LIB}trace.so'));
}
}
}
@ -213,7 +202,6 @@ async function run() {
core.exportVariable(sharedEnv.CODEQL_ACTION_TRACED_LANGUAGES, tracedLanguageKeys.join(','));
// TODO: make this a "private" environment variable of the action
core.exportVariable(sharedEnv.CODEQL_ACTION_DATABASE_DIR, databaseFolder);
core.exportVariable(sharedEnv.CODEQL_ACTION_CMD, codeqlSetup.cmd);
}
catch (error) {
core.setFailed(error.message);

2
lib/setup-tracer.js.map
View file

File diff suppressed because one or more lines are too long

1
lib/shared-environment.js generated
View file

@ -1,6 +1,5 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.CODEQL_ACTION_CMD = 'CODEQL_ACTION_CMD';
exports.CODEQL_ACTION_DATABASE_DIR = 'CODEQL_ACTION_DATABASE_DIR';
exports.CODEQL_ACTION_LANGUAGES = 'CODEQL_ACTION_LANGUAGES';
exports.CODEQL_ACTION_ANALYSIS_KEY = 'CODEQL_ACTION_ANALYSIS_KEY';

2
lib/shared-environment.js.map
View file

@ -1 +1 @@
{"version":3,"file":"shared-environment.js","sourceRoot":"","sources":["../src/shared-environment.ts"],"names":[],"mappings":";;AAAa,QAAA,iBAAiB,GAAG,mBAAmB,CAAC;AACxC,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAC1D,QAAA,uBAAuB,GAAG,yBAAyB,CAAC;AACpD,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAC1D,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAC1D,QAAA,+BAA+B,GAAG,iCAAiC,CAAC;AACpE,QAAA,8BAA8B,GAAG,gCAAgC,CAAC;AAC/E,wEAAwE;AACxE,2EAA2E;AAC3E,4EAA4E;AAC5E,2EAA2E;AAC3E,+BAA+B;AAClB,QAAA,wBAAwB,GAAG,0BAA0B,CAAC;AACnE,wDAAwD;AAC3C,QAAA,4BAA4B,GAAG,8BAA8B,CAAC"}
{"version":3,"file":"shared-environment.js","sourceRoot":"","sources":["../src/shared-environment.ts"],"names":[],"mappings":";;AAAa,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAC1D,QAAA,uBAAuB,GAAG,yBAAyB,CAAC;AACpD,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAC1D,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAC1D,QAAA,+BAA+B,GAAG,iCAAiC,CAAC;AACpE,QAAA,8BAA8B,GAAG,gCAAgC,CAAC;AAC/E,wEAAwE;AACxE,2EAA2E;AAC3E,4EAA4E;AAC5E,2EAA2E;AAC3E,+BAA+B;AAClB,QAAA,wBAAwB,GAAG,0BAA0B,CAAC;AACnE,wDAAwD;AAC3C,QAAA,4BAA4B,GAAG,8BAA8B,CAAC"}

20
src/autobuild.ts
View file

@ -1,7 +1,6 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as path from 'path';
import { getCodeQL } from './codeql';
import * as sharedEnv from './shared-environment';
import * as util from './util';
@ -30,22 +29,9 @@ async function run() {
}
core.startGroup(`Attempting to automatically build ${language} code`);
// TODO: share config accross actions better via env variables
const codeqlCmd = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_CMD);
const codeQL = getCodeQL();
await codeQL.runAutobuild(language);
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
const autobuildCmd = path.join(path.dirname(codeqlCmd), language, 'tools', cmdName);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
await exec.exec(autobuildCmd);
core.endGroup();
} catch (error) {

6
src/setup-tools.test.ts → src/codeql.test.ts
View file

@ -3,7 +3,7 @@ import test from 'ava';
import nock from 'nock';
import * as path from 'path';
import * as setupTools from './setup-tools';
import * as codeql from './codeql';
import {silenceDebugOutput} from './testing-utils';
import * as util from './util';
@ -30,7 +30,7 @@ test('download codeql bundle cache', async t => {
process.env['INPUT_TOOLS'] = `https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
await setupTools.setupCodeQL();
await codeql.setupCodeQL();
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
}
@ -56,7 +56,7 @@ test('parse codeql bundle url version', t => {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = setupTools.getCodeQLURLVersion(url);
const parsedVersion = codeql.getCodeQLURLVersion(url);
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e.message);

249
src/codeql.ts Normal file
View file

@ -0,0 +1,249 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as toolcache from '@actions/tool-cache';
import * as fs from 'fs';
import * as path from 'path';
import * as semver from 'semver';
import * as util from './util';
export interface CodeQL {
/**
* Get the directory where the CodeQL executable is located.
*/
getDir(): string;
/**
* Print version information about CodeQL.
*/
printVersion(): Promise<void>;
/**
* Run 'codeql database trace-command' on 'tracer-env.js' and parse
* the result to get environment variables set by CodeQL.
*/
getTracerEnv(databasePath: string, compilerSpec: string | undefined): Promise<{ [key: string]: string }>;
/**
* Run 'codeql database init'.
*/
databaseInit(databasePath: string, language: string, sourceRoot: string): Promise<void>;
/**
* Runs the autobuilder for the given language.
*/
runAutobuild(language: string): Promise<void>;
/**
* Extract code for a scanned language using 'codeql database trace-command'
* and running the language extracter.
*/
extractScannedLanguage(database: string, language: string): Promise<void>;
/**
* Finalize a database using 'codeql database finalize'.
*/
finalizeDatabase(databasePath: string): Promise<void>;
/**
* Run 'codeql resolve queries'.
*/
resolveQueries(queries: string[]): Promise<ResolveQueriesOutput>;
/**
* Run 'codeql database analyze'.
*/
databaseAnalyze(databasePath: string, sarifFile: string, querySuite: string): Promise<void>;
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
[queryPath: string]: {}
}
};
noDeclaredLanguage: {
[queryPath: string]: {}
};
multipleDeclaredLanguages: {
[queryPath: string]: {}
};
}
/**
* Environment variable used to store the location of the CodeQL CLI executable.
* Value is set by setupCodeQL and read by getCodeQL.
*/
const CODEQL_ACTION_CMD = "CODEQL_ACTION_CMD";
export async function setupCodeQL(): Promise<CodeQL> {
try {
const codeqlURL = core.getInput('tools', { required: true });
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
const codeqlPath = await toolcache.downloadTool(codeqlURL);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
}
let codeqlCmd = path.join(codeqlFolder, 'codeql', 'codeql');
if (process.platform === 'win32') {
codeqlCmd += ".exe";
} else if (process.platform !== 'linux' && process.platform !== 'darwin') {
throw new Error("Unsupported plaform: " + process.platform);
}
core.exportVariable(CODEQL_ACTION_CMD, codeqlCmd);
return getCodeQLForCmd(codeqlCmd);
} catch (e) {
core.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Version could not be inferred`);
}
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`);
}
return s;
}
export function getCodeQL(): CodeQL {
const codeqlCmd = util.getRequiredEnvParam(CODEQL_ACTION_CMD);
return getCodeQLForCmd(codeqlCmd);
}
function getCodeQLForCmd(cmd: string): CodeQL {
return {
getDir: function() {
return path.dirname(cmd);
},
printVersion: async function() {
await exec.exec(cmd, [
'version',
'--format=json'
]);
},
getTracerEnv: async function(databasePath: string, compilerSpec: string | undefined) {
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
await exec.exec(cmd, [
'database',
'trace-command',
databasePath,
...compilerSpecArg,
process.execPath,
path.resolve(__dirname, 'tracer-env.js'),
envFile
]);
return JSON.parse(fs.readFileSync(envFile, 'utf-8'));
},
databaseInit: async function(databasePath: string, language: string, sourceRoot: string) {
await exec.exec(cmd, [
'database',
'init',
databasePath,
'--language=' + language,
'--source-root=' + sourceRoot,
]);
},
runAutobuild: async function(language: string) {
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
const autobuildCmd = path.join(path.dirname(cmd), language, 'tools', cmdName);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
let javaToolOptions = process.env['JAVA_TOOL_OPTIONS'] || "";
process.env['JAVA_TOOL_OPTIONS'] = [...javaToolOptions.split(/\s+/), '-Dhttp.keepAlive=false', '-Dmaven.wagon.http.pool=false'].join(' ');
await exec.exec(autobuildCmd);
},
extractScannedLanguage: async function(databasePath: string, language: string) {
// Get extractor location
let extractorPath = '';
await exec.exec(
cmd,
[
'resolve',
'extractor',
'--format=json',
'--language=' + language
],
{
silent: true,
listeners: {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
const traceCommand = path.resolve(JSON.parse(extractorPath), 'tools', 'autobuild' + ext);
// Run trace command
await exec.exec(cmd, [
'database',
'trace-command',
databasePath,
'--',
traceCommand
]);
},
finalizeDatabase: async function(databasePath: string) {
await exec.exec(cmd, [
'database',
'finalize',
databasePath
]);
},
resolveQueries: async function(queries: string[]) {
let output = '';
await exec.exec(
cmd,
[
'resolve',
'queries',
...queries,
'--format=bylanguage'
],
{
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
}
}
});
return JSON.parse(output);
},
databaseAnalyze: async function(databasePath: string, sarifFile: string, querySuite: string) {
await exec.exec(cmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
databasePath,
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
querySuite
]);
}
};
}

98
src/finalize-db.ts
View file

@ -1,9 +1,9 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as io from '@actions/io';
import * as fs from 'fs';
import * as path from 'path';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import * as externalQueries from "./external-queries";
import * as sharedEnv from './shared-environment';
@ -31,85 +31,33 @@ function queryIsDisabled(language, query): boolean {
.some(disabledQuery => query.endsWith(disabledQuery));
}
async function createdDBForScannedLanguages(codeqlCmd: string, databaseFolder: string) {
async function createdDBForScannedLanguages(databaseFolder: string) {
const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
if (scannedLanguages) {
const codeql = getCodeQL();
for (const language of scannedLanguages.split(',')) {
core.startGroup('Extracting ' + language);
// Get extractor location
let extractorPath = '';
await exec.exec(codeqlCmd, ['resolve', 'extractor', '--format=json', '--language=' + language], {
silent: true,
listeners: {
stdout: (data) => { extractorPath += data.toString(); },
stderr: (data) => { process.stderr.write(data); }
}
});
// Set trace command
const ext = process.platform === 'win32' ? '.cmd' : '.sh';
const traceCommand = path.resolve(JSON.parse(extractorPath), 'tools', 'autobuild' + ext);
// Run trace command
await exec.exec(
codeqlCmd,
['database', 'trace-command', path.join(databaseFolder, language), '--', traceCommand]);
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(codeqlCmd: string, databaseFolder: string) {
await createdDBForScannedLanguages(codeqlCmd, databaseFolder);
async function finalizeDatabaseCreation(databaseFolder: string) {
await createdDBForScannedLanguages(databaseFolder);
const languages = process.env[sharedEnv.CODEQL_ACTION_LANGUAGES] || '';
const codeql = getCodeQL();
for (const language of languages.split(',')) {
core.startGroup('Finalizing ' + language);
await exec.exec(codeqlCmd, ['database', 'finalize', path.join(databaseFolder, language)]);
await codeql.finalizeDatabase(path.join(databaseFolder, language));
core.endGroup();
}
}
interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
[queryPath: string]: {}
}
};
noDeclaredLanguage: {
[queryPath: string]: {}
};
multipleDeclaredLanguages: {
[queryPath: string]: {}
};
}
async function runResolveQueries(codeqlCmd: string, queries: string[]): Promise<ResolveQueriesOutput> {
let output = '';
const options = {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
}
}
};
await exec.exec(
codeqlCmd, [
'resolve',
'queries',
...queries,
'--format=bylanguage'
],
options);
return JSON.parse(output);
}
async function resolveQueryLanguages(codeqlCmd: string, config: configUtils.Config): Promise<Map<string, string[]>> {
async function resolveQueryLanguages(config: configUtils.Config): Promise<Map<string, string[]>> {
let res = new Map();
const codeql = getCodeQL();
if (!config.disableDefaultQueries || config.additionalSuites.length !== 0) {
const suites: string[] = [];
@ -122,7 +70,7 @@ async function resolveQueryLanguages(codeqlCmd: string, config: configUtils.Conf
}
}
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, suites);
const resolveQueriesOutputObject = await codeql.resolveQueries(suites);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
if (res[language] === undefined) {
@ -133,7 +81,7 @@ async function resolveQueryLanguages(codeqlCmd: string, config: configUtils.Conf
}
if (config.additionalQueries.length !== 0) {
const resolveQueriesOutputObject = await runResolveQueries(codeqlCmd, config.additionalQueries);
const resolveQueriesOutputObject = await codeql.resolveQueries(config.additionalQueries);
for (const [language, queries] of Object.entries(resolveQueriesOutputObject.byLanguage)) {
if (res[language] === undefined) {
@ -159,8 +107,9 @@ async function resolveQueryLanguages(codeqlCmd: string, config: configUtils.Conf
}
// Runs queries and creates sarif files in the given folder
async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder: string, config: configUtils.Config) {
const queriesPerLanguage = await resolveQueryLanguages(codeqlCmd, config);
async function runQueries(databaseFolder: string, sarifFolder: string, config: configUtils.Config) {
const queriesPerLanguage = await resolveQueryLanguages(config);
const codeql = getCodeQL();
for (let database of fs.readdirSync(databaseFolder)) {
core.startGroup('Analyzing ' + database);
@ -179,17 +128,7 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
const sarifFile = path.join(sarifFolder, database + '.sarif');
await exec.exec(codeqlCmd, [
'database',
'analyze',
util.getMemoryFlag(),
util.getThreadsFlag(),
path.join(databaseFolder, database),
'--format=sarif-latest',
'--output=' + sarifFile,
'--no-sarif-add-snippets',
querySuite
]);
await codeql.databaseAnalyze(path.join(databaseFolder, database), sarifFile, querySuite);
core.debug('SARIF results for database ' + database + ' created at "' + sarifFile + '"');
core.endGroup();
@ -206,19 +145,18 @@ async function run() {
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const codeqlCmd = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_CMD);
const databaseFolder = util.getRequiredEnvParam(sharedEnv.CODEQL_ACTION_DATABASE_DIR);
const sarifFolder = core.getInput('output');
await io.mkdirP(sarifFolder);
core.info('Finalizing database creation');
await finalizeDatabaseCreation(codeqlCmd, databaseFolder);
await finalizeDatabaseCreation(databaseFolder);
await externalQueries.checkoutExternalQueries(config);
core.info('Analyzing database');
await runQueries(codeqlCmd, databaseFolder, sarifFolder, config);
await runQueries(databaseFolder, sarifFolder, config);
if ('true' === core.getInput('upload')) {
if (!await upload_lib.upload(sarifFolder)) {

73
src/setup-tools.ts
View file

@ -1,73 +0,0 @@
import * as core from '@actions/core';
import * as toolcache from '@actions/tool-cache';
import * as path from 'path';
import * as semver from 'semver';
export class CodeQLSetup {
public dist: string;
public tools: string;
public cmd: string;
public platform: string;
constructor(codeqlDist: string) {
this.dist = codeqlDist;
this.tools = path.join(this.dist, 'tools');
this.cmd = path.join(codeqlDist, 'codeql');
// TODO check process.arch ?
if (process.platform === 'win32') {
this.platform = 'win64';
if (this.cmd.endsWith('codeql')) {
this.cmd += ".exe";
}
} else if (process.platform === 'linux') {
this.platform = 'linux64';
} else if (process.platform === 'darwin') {
this.platform = 'osx64';
} else {
throw new Error("Unsupported plaform: " + process.platform);
}
}
}
export async function setupCodeQL(): Promise<CodeQLSetup> {
try {
const codeqlURL = core.getInput('tools', { required: true });
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
const codeqlPath = await toolcache.downloadTool(codeqlURL);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
}
return new CodeQLSetup(path.join(codeqlFolder, 'codeql'));
} catch (e) {
core.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Version could not be inferred`);
}
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`);
}
return s;
}

36
src/setup-tracer.ts
View file

@ -5,8 +5,8 @@ import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import * as setuptools from './setup-tools';
import * as sharedEnv from './shared-environment';
import * as util from './util';
@ -24,19 +24,11 @@ const CRITICAL_TRACER_VARS = new Set(
]);
async function tracerConfig(
codeql: setuptools.CodeQLSetup,
codeql: CodeQL,
database: string,
compilerSpec?: string): Promise<TracerConfig> {
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
let envFile = path.resolve(database, 'working', 'env.tmp');
await exec.exec(codeql.cmd, ['database', 'trace-command', database,
...compilerSpecArg,
process.execPath, path.resolve(__dirname, 'tracer-env.js'), envFile]
);
const env: { [key: string]: string } = JSON.parse(fs.readFileSync(envFile, 'utf-8'));
const env = await codeql.getTracerEnv(database, compilerSpec);
const config = env['ODASA_TRACER_CONFIGURATION'];
const info: TracerConfig = { spec: config, env: {} };
@ -174,8 +166,8 @@ async function run() {
const sourceRoot = path.resolve();
core.startGroup('Setup CodeQL tools');
const codeqlSetup = await setuptools.setupCodeQL();
await exec.exec(codeqlSetup.cmd, ['version', '--format=json']);
const codeql = await setupCodeQL();
await codeql.printVersion();
core.endGroup();
// Forward Go flags
@ -194,22 +186,15 @@ async function run() {
let tracedLanguages: { [key: string]: TracerConfig } = {};
let scannedLanguages: string[] = [];
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of languages) {
const languageDatabase = path.join(databaseFolder, language);
// Init language database
await exec.exec(codeqlSetup.cmd, [
'database',
'init',
languageDatabase,
'--language=' + language,
'--source-root=' + sourceRoot,
]);
await codeql.databaseInit(languageDatabase, language, sourceRoot);
// TODO: add better detection of 'traced languages' instead of using a hard coded list
if (['cpp', 'java', 'csharp'].includes(language)) {
const config: TracerConfig = await tracerConfig(codeqlSetup, languageDatabase);
const config: TracerConfig = await tracerConfig(codeql, languageDatabase);
tracedLanguages[language] = config;
} else {
scannedLanguages.push(language);
@ -227,17 +212,17 @@ async function run() {
if (process.platform === 'darwin') {
core.exportVariable(
'DYLD_INSERT_LIBRARIES',
path.join(codeqlSetup.tools, 'osx64', 'libtrace.dylib'));
path.join(codeql.getDir(), 'tools', 'osx64', 'libtrace.dylib'));
} else if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(codeqlSetup.tools, 'win64', 'tracer.exe'),
path.resolve(codeql.getDir(), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
} else {
core.exportVariable('LD_PRELOAD', path.join(codeqlSetup.tools, 'linux64', '${LIB}trace.so'));
core.exportVariable('LD_PRELOAD', path.join(codeql.getDir(), 'tools', 'linux64', '${LIB}trace.so'));
}
}
}
@ -247,7 +232,6 @@ async function run() {
// TODO: make this a "private" environment variable of the action
core.exportVariable(sharedEnv.CODEQL_ACTION_DATABASE_DIR, databaseFolder);
core.exportVariable(sharedEnv.CODEQL_ACTION_CMD, codeqlSetup.cmd);
} catch (error) {
core.setFailed(error.message);

1
src/shared-environment.ts
View file

@ -1,4 +1,3 @@
export const CODEQL_ACTION_CMD = 'CODEQL_ACTION_CMD';
export const CODEQL_ACTION_DATABASE_DIR = 'CODEQL_ACTION_DATABASE_DIR';
export const CODEQL_ACTION_LANGUAGES = 'CODEQL_ACTION_LANGUAGES';
export const CODEQL_ACTION_ANALYSIS_KEY = 'CODEQL_ACTION_ANALYSIS_KEY';