robojerk/codeql-action

Break out tracer-config.ts

Browse source
This commit is contained in:
Robert Brignull 2020-08-24 12:53:09 +01:00
parent bd54c20d3e
commit 407ef0ac11
26 changed files with 953 additions and 489 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/codeql.js generated
View file

@ -231,14 +231,12 @@ function getCodeQLForCmd(cmd) {
'--format=json'
]);
},
getTracerEnv: async function (databasePath, compilerSpec) {
getTracerEnv: async function (databasePath) {
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
await exec.exec(cmd, [
'database',
'trace-command',
databasePath,
...compilerSpecArg,
...getExtraOptionsFromEnv(['database', 'trace-command']),
process.execPath,
path.resolve(__dirname, 'tracer-env.js'),

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

24
lib/finalize-db.js generated
View file

@ -29,43 +29,44 @@ async function sendStatusReport(startedAt, queriesStats, uploadStats, error) {
};
await util.sendStatusReport(statusReport);
}
async function createdDBForScannedLanguages(databaseFolder, config) {
async function createdDBForScannedLanguages(config) {
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (languages_1.isScannedLanguage(language)) {
core.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(databaseFolder, config) {
await createdDBForScannedLanguages(databaseFolder, config);
async function finalizeDatabaseCreation(config) {
await createdDBForScannedLanguages(config);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
core.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(path.join(databaseFolder, language));
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
core.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(databaseFolder, sarifFolder, config) {
async function runQueries(sarifFolder, config) {
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
for (let language of fs.readdirSync(databaseFolder)) {
for (let language of config.languages) {
core.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = path.join(databaseFolder, language + '-queries.qls');
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(path.join(databaseFolder, language), sarifFile, querySuite);
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
core.endGroup();
}
@ -90,13 +91,12 @@ async function run() {
const config = await configUtils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
const sarifFolder = core.getInput('output');
fs.mkdirSync(sarifFolder, { recursive: true });
core.info('Finalizing database creation');
await finalizeDatabaseCreation(databaseFolder, config);
await finalizeDatabaseCreation(config);
core.info('Analyzing database');
queriesStats = await runQueries(databaseFolder, sarifFolder, config);
queriesStats = await runQueries(sarifFolder, config);
if ('true' === core.getInput('upload')) {
uploadStats = await upload_lib.upload(sarifFolder, repository_1.parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')), await util.getCommitOid(), util.getRef(), await util.getAnalysisKey(), util.getRequiredEnvParam('GITHUB_WORKFLOW'), util.getWorkflowRunID(), core.getInput('checkout_path'), core.getInput('matrix'), core.getInput('token'), util.getRequiredEnvParam('GITHUB_API_URL'), 'actions', logging_1.getActionsLogger());
}

2
lib/finalize-db.js.map
View file

@ -1 +1 @@
{"version":3,"file":"finalize-db.js","sourceRoot":"","sources":["../src/finalize-db.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,qCAAqC;AACrC,4DAA8C;AAC9C,2CAAgD;AAChD,uCAA6C;AAC7C,6CAAkD;AAClD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAiC/B,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,YAA6C,EAC7C,WAAsD,EACtD,KAAa;;IAEb,MAAM,MAAM,GAAG,OAAA,YAAY,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IACnH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,QAAE,KAAK,0CAAE,OAAO,QAAE,KAAK,0CAAE,KAAK,CAAC,CAAC;IACtH,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QACvB,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,cAAsB,EAAE,MAA0B;IAC5F,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YACnF,IAAI,CAAC,QAAQ,EAAE,CAAC;SACjB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,cAAsB,EAAE,MAA0B;IACxF,MAAM,4BAA4B,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;QAC1C,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,EAAE,CAAC;KACjB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,cAAsB,EACtB,WAAmB,EACnB,MAA0B;IAE1B,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,QAAQ,IAAI,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE;QACnD,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,QAAQ,GAAG,gDAAgD,CAAC,CAAC;SACrG;QAED,IAAI;YACF,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,cAAc,CAAC,CAAC;YACxE,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,uBAAuB,GAAG,QAAQ,GAAG,OAAO,GAAG,kBAAkB,CAAC,CAAC;YAE9E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAEzF,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,QAAQ,GAAG,eAAe,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;YACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;SAEjB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,YAAY,GAAoC,SAAS,CAAC;IAC9D,IAAI,WAAW,GAA8C,SAAS,CAAC;IACvE,IAAI;QACF,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE;YAC1G,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;QAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;QAEzD,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC1C,MAAM,wBAAwB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAEvD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAChC,YAAY,GAAG,MAAM,UAAU,CAAC,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QAErE,IAAI,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACtC,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACnC,WAAW,EACX,+BAAkB,CAAC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACjE,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,IAAI,CAAC,MAAM,EAAE,EACb,MAAM,IAAI,CAAC,cAAc,EAAE,EAC3B,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,EAC3C,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAC9B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAC1C,SAAS,EACT,0BAAgB,EAAE,CAAC,CAAC;SACvB;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO;KACR;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;AAC/D,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
{"version":3,"file":"finalize-db.js","sourceRoot":"","sources":["../src/finalize-db.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,qCAAqC;AACrC,4DAA8C;AAC9C,2CAAgD;AAChD,uCAA6C;AAC7C,6CAAkD;AAClD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAiC/B,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,YAA6C,EAC7C,WAAsD,EACtD,KAAa;;IAEb,MAAM,MAAM,GAAG,OAAA,YAAY,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IACnH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,QAAE,KAAK,0CAAE,OAAO,QAAE,KAAK,0CAAE,KAAK,CAAC,CAAC;IACtH,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QACvB,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,MAA0B;IACpE,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YACpG,IAAI,CAAC,QAAQ,EAAE,CAAC;SACjB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,MAA0B;IAChE,MAAM,4BAA4B,CAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;QAC1C,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,QAAQ,EAAE,CAAC;KACjB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,WAAmB,EACnB,MAA0B;IAE1B,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACrC,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,QAAQ,GAAG,gDAAgD,CAAC,CAAC;SACrG;QAED,IAAI;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC1E,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,YAAY,GAAG,cAAc,CAAC;YACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,uBAAuB,GAAG,QAAQ,GAAG,OAAO,GAAG,kBAAkB,CAAC,CAAC;YAE9E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAElE,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,QAAQ,GAAG,eAAe,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;YACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;SAEjB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,YAAY,GAAoC,SAAS,CAAC;IAC9D,IAAI,WAAW,GAA8C,SAAS,CAAC;IACvE,IAAI;QACF,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE;YAC1G,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;QAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;QAEzD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC1C,MAAM,wBAAwB,CAAC,MAAM,CAAC,CAAC;QAEvC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAChC,YAAY,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAErD,IAAI,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACtC,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACnC,WAAW,EACX,+BAAkB,CAAC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACjE,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,IAAI,CAAC,MAAM,EAAE,EACb,MAAM,IAAI,CAAC,cAAc,EAAE,EAC3B,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,EAC3C,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAC9B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAC1C,SAAS,EACT,0BAAgB,EAAE,CAAC,CAAC;SACvB;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO;KACR;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;AAC/D,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}

2
lib/finalize-db.test.js generated
View file

@ -1,2 +0,0 @@
"use strict";
//# sourceMappingURL=finalize-db.test.js.map

1
lib/finalize-db.test.js.map
View file

@ -1 +0,0 @@
{"version":3,"file":"finalize-db.test.js","sourceRoot":"","sources":["../src/finalize-db.test.ts"],"names":[],"mappings":""}

76
lib/setup-tools.js generated
View file

@ -1,76 +0,0 @@
"use strict";
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(require("@actions/core"));
const toolcache = __importStar(require("@actions/tool-cache"));
const path = __importStar(require("path"));
const semver = __importStar(require("semver"));
class CodeQLSetup {
constructor(codeqlDist) {
this.dist = codeqlDist;
this.tools = path.join(this.dist, 'tools');
this.cmd = path.join(codeqlDist, 'codeql');
// TODO check process.arch ?
if (process.platform === 'win32') {
this.platform = 'win64';
if (this.cmd.endsWith('codeql')) {
this.cmd += ".exe";
}
}
else if (process.platform === 'linux') {
this.platform = 'linux64';
}
else if (process.platform === 'darwin') {
this.platform = 'osx64';
}
else {
throw new Error("Unsupported plaform: " + process.platform);
}
}
}
exports.CodeQLSetup = CodeQLSetup;
async function setupCodeQL() {
try {
const codeqlURL = core.getInput('tools', { required: true });
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
}
else {
const codeqlPath = await toolcache.downloadTool(codeqlURL);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
}
return new CodeQLSetup(path.join(codeqlFolder, 'codeql'));
}
catch (e) {
core.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
exports.setupCodeQL = setupCodeQL;
function getCodeQLURLVersion(url) {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Version could not be inferred`);
}
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Malformed tools url ${url}. Version should be in SemVer format but have ${version} instead`);
}
return s;
}
exports.getCodeQLURLVersion = getCodeQLURLVersion;
//# sourceMappingURL=setup-tools.js.map

1
lib/setup-tools.js.map
View file

@ -1 +0,0 @@
{"version":3,"file":"setup-tools.js","sourceRoot":"","sources":["../src/setup-tools.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AACtC,+DAAiD;AACjD,2CAA6B;AAC7B,+CAAiC;AAEjC,MAAa,WAAW;IAMtB,YAAY,UAAkB;QAC5B,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC3C,4BAA4B;QAC5B,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAC/B,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC;aACpB;SACF;aAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YACvC,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;SAC3B;aAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;YACxC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;SACzB;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;SAC7D;IACH,CAAC;CACF;AAxBD,kCAwBC;AAEM,KAAK,UAAU,WAAW;IAC/B,IAAI;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAExD,IAAI,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAC9D,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,KAAK,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAC;SACrD;aAAM;YACL,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/D,YAAY,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;SACtF;QACD,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;KAE3D;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;AACH,CAAC;AAnBD,kCAmBC;AAED,SAAgB,mBAAmB,CAAC,GAAW;IAE7C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAClD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACtC,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,iCAAiC,CAAC,CAAC;KAC/E;IAED,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAEvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE;QAC1B,IAAI,CAAC,KAAK,CAAC,kBAAkB,OAAO,gEAAgE,OAAO,GAAG,CAAC,CAAC;QAChH,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC;KAC9B;IAED,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,CAAC,CAAC,EAAE;QACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,iDAAiD,OAAO,UAAU,CAAC,CAAC;KAC/G;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AApBD,kDAoBC"}

60
lib/setup-tools.test.js generated
View file

@ -1,60 +0,0 @@
"use strict";
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const toolcache = __importStar(require("@actions/tool-cache"));
const ava_1 = __importDefault(require("ava"));
const nock_1 = __importDefault(require("nock"));
const path = __importStar(require("path"));
const setupTools = __importStar(require("./setup-tools"));
const testing_utils_1 = require("./testing-utils");
const util = __importStar(require("./util"));
testing_utils_1.silenceDebugOutput(ava_1.default);
ava_1.default('download codeql bundle cache', async (t) => {
await util.withTmpDir(async (tmpDir) => {
process.env['GITHUB_WORKSPACE'] = tmpDir;
process.env['RUNNER_TEMP'] = path.join(tmpDir, 'temp');
process.env['RUNNER_TOOL_CACHE'] = path.join(tmpDir, 'cache');
const versions = ['20200601', '20200610'];
for (let i = 0; i < versions.length; i++) {
const version = versions[i];
nock_1.default('https://example.com')
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(200, path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`));
process.env['INPUT_TOOLS'] = `https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
await setupTools.setupCodeQL();
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
}
const cachedVersions = toolcache.findAllVersions('CodeQL');
t.is(cachedVersions.length, 2);
});
});
ava_1.default('parse codeql bundle url version', t => {
const tests = {
'20200601': '0.0.0-20200601',
'20200601.0': '0.0.0-20200601.0',
'20200601.0.0': '20200601.0.0',
'1.2.3': '1.2.3',
'1.2.3-alpha': '1.2.3-alpha',
'1.2.3-beta.1': '1.2.3-beta.1',
};
for (const [version, expectedVersion] of Object.entries(tests)) {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = setupTools.getCodeQLURLVersion(url);
t.deepEqual(parsedVersion, expectedVersion);
}
catch (e) {
t.fail(e.message);
}
}
});
//# sourceMappingURL=setup-tools.test.js.map

1
lib/setup-tools.test.js.map
View file

@ -1 +0,0 @@
{"version":3,"file":"setup-tools.test.js","sourceRoot":"","sources":["../src/setup-tools.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+DAAiD;AACjD,8CAAuB;AACvB,gDAAwB;AACxB,2CAA6B;AAE7B,0DAA4C;AAC5C,mDAAmD;AACnD,6CAA+B;AAE/B,kCAAkB,CAAC,aAAI,CAAC,CAAC;AAEzB,aAAI,CAAC,8BAA8B,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAE7C,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;QAEnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,MAAM,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE9D,MAAM,QAAQ,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAE5B,cAAI,CAAC,qBAAqB,CAAC;iBACxB,GAAG,CAAC,2BAA2B,OAAO,uBAAuB,CAAC;iBAC9D,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uCAAuC,CAAC,CAAC,CAAC;YAGrF,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,8CAA8C,OAAO,uBAAuB,CAAC;YAE1G,MAAM,UAAU,CAAC,WAAW,EAAE,CAAC;YAE/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAC,CAAC,CAAC;SACxD;QAED,MAAM,cAAc,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAE3D,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,iCAAiC,EAAE,CAAC,CAAC,EAAE;IAE1C,MAAM,KAAK,GAAG;QACZ,UAAU,EAAE,gBAAgB;QAC5B,YAAY,EAAE,kBAAkB;QAChC,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,OAAO;QAChB,aAAa,EAAE,aAAa;QAC5B,cAAc,EAAE,cAAc;KAC/B,CAAC;IAEF,KAAK,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAC9D,MAAM,GAAG,GAAG,wCAAwC,OAAO,MAAM,CAAC;QAElE,IAAI;YACF,MAAM,aAAa,GAAG,UAAU,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC1D,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;SAC7C;QAAC,OAAO,CAAC,EAAE;YACV,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACnB;KACF;AACH,CAAC,CAAC,CAAC"}

146
lib/setup-tracer.js generated
View file

@ -14,111 +14,8 @@ const path = __importStar(require("path"));
const analysisPaths = __importStar(require("./analysis-paths"));
const codeql_1 = require("./codeql");
const configUtils = __importStar(require("./config-utils"));
const languages_1 = require("./languages");
const tracer_config_1 = require("./tracer-config");
const util = __importStar(require("./util"));
const CRITICAL_TRACER_VARS = new Set(['SEMMLE_PRELOAD_libtrace',
,
'SEMMLE_RUNNER',
,
'SEMMLE_COPY_EXECUTABLES_ROOT',
,
'SEMMLE_DEPTRACE_SOCKET',
,
'SEMMLE_JAVA_TOOL_OPTIONS'
]);
async function tracerConfig(codeql, database, compilerSpec) {
const env = await codeql.getTracerEnv(database, compilerSpec);
const config = env['ODASA_TRACER_CONFIGURATION'];
const info = { spec: config, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
info.env[key] = value;
}
}
return info;
}
function concatTracerConfigs(tracerConfigs, config) {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env = {};
let copyExecutables = false;
let envSize = 0;
for (const v of tracerConfigs) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
copyExecutables = true;
}
else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
}
}
else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
fs.writeFileSync(envPath, buffer);
return { env, spec };
}
async function sendSuccessStatusReport(startedAt, config) {
const statusReportBase = await util.createStatusReportBase('init', 'success', startedAt);
const languages = config.languages.join(',');
@ -173,41 +70,22 @@ async function run() {
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
fs.mkdirSync(databaseFolder, { recursive: true });
let tracedLanguageConfigs = [];
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
const languageDatabase = path.join(databaseFolder, language);
// Init language database
await codeql.databaseInit(languageDatabase, language, sourceRoot);
// TODO: add better detection of 'traced languages' instead of using a hard coded list
if (languages_1.isTracedLanguage(language)) {
const config = await tracerConfig(codeql, languageDatabase);
tracedLanguageConfigs.push(config);
}
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
if (tracedLanguageConfigs.length > 0) {
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
if (mainTracerConfig.spec) {
for (let entry of Object.entries(mainTracerConfig.env)) {
core.exportVariable(entry[0], entry[1]);
}
core.exportVariable('ODASA_TRACER_CONFIGURATION', mainTracerConfig.spec);
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
core.exportVariable('DYLD_INSERT_LIBRARIES', path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib'));
}
else if (process.platform === 'win32') {
await exec.exec('powershell', [
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(codeQLDir, 'tools', 'win64', 'tracer.exe'),
], { env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
}
else {
core.exportVariable('LD_PRELOAD', path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so'));
}
const tracerConfig = await tracer_config_1.getTracerConfig(config, codeql);
if (tracerConfig !== undefined) {
if (process.platform === 'win32') {
await exec.exec('powershell', [
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
], { env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
}
// NB: in CLI mode these will be outputted to a file instead of exported with core.exportVariable
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
}
}
catch (error) {

2
lib/setup-tracer.js.map
View file

File diff suppressed because one or more lines are too long

22
lib/test-utils.js generated
View file

@ -1,22 +0,0 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
function silenceDebugOutput(test) {
const typedTest = test;
typedTest.beforeEach(t => {
const processStdoutWrite = process.stdout.write.bind(process.stdout);
t.context.write = processStdoutWrite;
process.stdout.write = (str, encoding, cb) => {
// Core library will directly call process.stdout.write for commands
// We don't want :: commands to be executed by the runner during tests
if (!str.match(/^::/)) {
processStdoutWrite(str, encoding, cb);
}
return true;
};
});
typedTest.afterEach(t => {
process.stdout.write = t.context.write;
});
}
exports.silenceDebugOutput = silenceDebugOutput;
//# sourceMappingURL=test-utils.js.map

1
lib/test-utils.js.map
View file

@ -1 +0,0 @@
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../src/test-utils.ts"],"names":[],"mappings":";;AAEA,SAAgB,kBAAkB,CAAC,IAAwB;IACzD,MAAM,SAAS,GAAG,IAAmC,CAAC;IAEtD,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QACrB,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrE,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,kBAAkB,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,GAAQ,EAAE,QAAc,EAAE,EAA0B,EAAE,EAAE;YAC5E,oEAAoE;YACpE,sEAAsE;YACtE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACnB,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;aACzC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC;AAnBD,gDAmBC"}

143
lib/tracer-config.js generated Normal file
View file

@ -0,0 +1,143 @@
"use strict";
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const languages_1 = require("./languages");
const util = __importStar(require("./util"));
const CRITICAL_TRACER_VARS = new Set(['SEMMLE_PRELOAD_libtrace',
,
'SEMMLE_RUNNER',
,
'SEMMLE_COPY_EXECUTABLES_ROOT',
,
'SEMMLE_DEPTRACE_SOCKET',
,
'SEMMLE_JAVA_TOOL_OPTIONS'
]);
async function tracerConfig(codeql, config, language) {
const env = await codeql.getTracerEnv(util.getCodeQLDatabasePath(config.tempDir, language));
const spec = env['ODASA_TRACER_CONFIGURATION'];
const info = { spec, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
info.env[key] = value;
}
}
return info;
}
exports.tracerConfig = tracerConfig;
function concatTracerConfigs(tracerConfigs, config) {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env = {};
let copyExecutables = false;
let envSize = 0;
for (const v of Object.values(tracerConfigs)) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
copyExecutables = true;
}
else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
}
}
else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
fs.writeFileSync(envPath, buffer);
return { env, spec };
}
exports.concatTracerConfigs = concatTracerConfigs;
async function getTracerConfig(config, codeql) {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter(languages_1.isTracedLanguage);
if (tracedLanguages.length === 0) {
return undefined;
}
// Get all the tracer configs and combine them together
let tracedLanguageConfigs = {};
for (let language of tracedLanguages) {
tracedLanguageConfigs[language] = await tracerConfig(codeql, config, language);
}
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
// Add a couple more variables
mainTracerConfig.env['ODASA_TRACER_CONFIGURATION'] = mainTracerConfig.spec;
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
mainTracerConfig.env['DYLD_INSERT_LIBRARIES'] = path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib');
}
else if (process.platform !== 'win32') {
mainTracerConfig.env['LD_PRELOAD'] = path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so');
}
return mainTracerConfig;
}
exports.getTracerConfig = getTracerConfig;
//# sourceMappingURL=tracer-config.js.map

1
lib/tracer-config.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"tracer-config.js","sourceRoot":"","sources":["../src/tracer-config.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAI7B,2CAAyD;AACzD,6CAA+B;AAO/B,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAClC,CAAC,yBAAyB;IACxB,AADyB;IACvB,eAAe;IACjB,AADkB;IAChB,8BAA8B;IAChC,AADiC;IAC/B,wBAAwB;IAC1B,AAD2B;IACzB,0BAA0B;CAC7B,CAAC,CAAC;AAEE,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,MAA0B,EAC1B,QAAkB;IAElB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAE5F,MAAM,IAAI,GAAG,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAE7C,yDAAyD;IACzD,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACrC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,8DAA8D;QAC9D,IAAI,GAAG,KAAK,4BAA4B,EAAE;YACxC,SAAS;SACV;QACD,wBAAwB;QACxB,IAAI,OAAO,KAAK,KAAK,WAAW,EAAE;YAChC,SAAS;SACV;QACD,mFAAmF;QACnF,iCAAiC;QACjC,IAAI,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,WAAW,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YACzG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACvB;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AA7BD,oCA6BC;AAED,SAAgB,mBAAmB,CACjC,aAA+C,EAC/C,MAA0B;IAE1B,iGAAiG;IACjG,0FAA0F;IAE1F,yBAAyB;IACzB,MAAM,GAAG,GAA+B,EAAE,CAAC;IAC3C,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QAC5C,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE;YACnC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,gEAAgE;YAChE,IAAI,IAAI,KAAK,8BAA8B,EAAE;gBAC3C,eAAe,GAAG,IAAI,CAAC;aACxB;iBAAM,IAAI,IAAI,IAAI,GAAG,EAAE;gBACtB,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE;oBACvB,MAAM,KAAK,CAAC,+CAA+C;wBACzD,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC;iBAC9C;aACF;iBAAM;gBACL,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;gBAClB,OAAO,IAAI,CAAC,CAAC;aACd;SACF;KACF;IAED,8CAA8C;IAC9C,IAAI,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1C,yFAAyF;IACzF,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE;QACnB,IAAI,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;QACtD,SAAS,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC;KAChC;IAED,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,IAAI,IAAI,SAAS,EAAE;QAC1B,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,UAAU,IAAI,KAAK,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KACpC;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAC3D,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACzE,MAAM,cAAc,GAAG,CAAC,cAAc,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC;IAEhF,IAAI,eAAe,EAAE;QACnB,GAAG,CAAC,8BAA8B,CAAC,GAAG,kBAAkB,CAAC;QACzD,OAAO,IAAI,CAAC,CAAC;KACd;IAED,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAElD,uDAAuD;IACvD,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAChC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACjC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,UAAU,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,GAAG,GAAG,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,UAAU,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;KAC1D;IACD,iCAAiC;IACjC,MAAM,OAAO,GAAG,IAAI,GAAG,cAAc,CAAC;IACtC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAElC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AACvB,CAAC;AA7ED,kDA6EC;AAEM,KAAK,UAAU,eAAe,CACnC,MAA0B,EAC1B,MAAc;IAEd,kEAAkE;IAClE,MAAM,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,4BAAgB,CAAC,CAAC;IAClE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;QAChC,OAAO,SAAS,CAAC;KAClB;IAED,uDAAuD;IACvD,IAAI,qBAAqB,GAAqC,EAAE,CAAC;IACjE,KAAK,IAAI,QAAQ,IAAI,eAAe,EAAE;QACpC,qBAAqB,CAAC,QAAQ,CAAC,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KAChF;IACD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAE5E,8BAA8B;IAC9B,gBAAgB,CAAC,GAAG,CAAC,4BAA4B,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC;IAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,gBAAgB,CAAC,GAAG,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;KAC1G;SAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QACvC,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;KACjG;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AA3BD,0CA2BC"}

268
lib/tracer-config.test.js generated Normal file
View file

@ -0,0 +1,268 @@
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const ava_1 = __importDefault(require("ava"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const codeql_1 = require("./codeql");
const languages_1 = require("./languages");
const testing_utils_1 = require("./testing-utils");
const tracer_config_1 = require("./tracer-config");
const util = __importStar(require("./util"));
testing_utils_1.setupTests(ava_1.default);
function getTestConfig(tmpDir) {
return {
languages: [languages_1.Language.java],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: '',
};
}
// A very minimal setup
ava_1.default('tracerConfig - minimal', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const codeQL = codeql_1.setCodeQL({
getTracerEnv: async function () {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
};
},
});
const result = await tracer_config_1.tracerConfig(codeQL, config, languages_1.Language.javascript);
t.deepEqual(result, { spec: 'abc', env: { 'foo': 'bar' } });
});
});
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
ava_1.default('tracerConfig - existing / critical vars', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const codeQL = codeql_1.setCodeQL({
getTracerEnv: async function () {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar',
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
};
},
});
process.env['foo'] = 'abc';
process.env['SEMMLE_PRELOAD_libtrace'] = 'abc';
process.env['SEMMLE_RUNNER'] = 'abc';
process.env['SEMMLE_COPY_EXECUTABLES_ROOT'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['SEMMLE_JAVA_TOOL_OPTIONS'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['CODEQL_VAR'] = 'abc';
const result = await tracer_config_1.tracerConfig(codeQL, config, languages_1.Language.javascript);
t.deepEqual(result, {
spec: 'abc',
env: {
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
}
});
});
});
ava_1.default('concatTracerConfigs - minimal', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
};
const result = tracer_config_1.concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
});
t.true(fs.existsSync(result.spec));
t.deepEqual(fs.readFileSync(result.spec, 'utf8'), path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nabc\ndef\nghi');
});
});
ava_1.default('concatTracerConfigs - conflicting env vars', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
// Ok if env vars have the same name and the same value
t.deepEqual(tracer_config_1.concatTracerConfigs({
'javascript': { spec: spec, env: { 'a': 'a', 'b': 'b' } },
'python': { spec: spec, env: { 'b': 'b', 'c': 'c' } },
}, config).env, {
'a': 'a',
'b': 'b',
'c': 'c',
});
// Throws if env vars have same name but different values
const e = t.throws(() => tracer_config_1.concatTracerConfigs({
'javascript': { spec: spec, env: { 'a': 'a', 'b': 'b' } },
'python': { spec: spec, env: { 'b': 'c' } },
}, config));
t.deepEqual(e.message, 'Incompatible values in environment parameter b: b and c');
});
});
// If cpp is present then it's spec lines always come at the end
ava_1.default('concatTracerConfigs - cpp comes last', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
};
const result = tracer_config_1.concatTracerConfigs({ 'cpp': tc1, 'python': tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
});
t.true(fs.existsSync(result.spec));
t.deepEqual(fs.readFileSync(result.spec, 'utf8'), path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nghi\nabc\ndef');
});
});
ava_1.default('concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
const result = tracer_config_1.concatTracerConfigs({
'javascript': { spec: spec, env: { 'a': 'a', 'b': 'b' } },
'python': { spec: spec, env: { 'SEMMLE_COPY_EXECUTABLES_ROOT': 'foo' } },
}, config);
t.deepEqual(result.env, {
'a': 'a',
'b': 'b',
'SEMMLE_COPY_EXECUTABLES_ROOT': path.join(tmpDir, 'compound-temp')
});
});
});
ava_1.default('concatTracerConfigs - compound environment file', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'foo': 'bar_baz',
}
};
const result = tracer_config_1.concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
const envPath = result.spec + '.environment';
t.true(fs.existsSync(envPath));
const buffer = fs.readFileSync(envPath);
t.deepEqual(28, buffer.length);
t.deepEqual(2, buffer.readInt32LE(0));
t.deepEqual(4, buffer.readInt32LE(4));
t.deepEqual('a=a\0', buffer.toString('utf8', 8, 12));
t.deepEqual(12, buffer.readInt32LE(12));
t.deepEqual('foo=bar_baz\0', buffer.toString('utf8', 16, 28));
});
});
ava_1.default('getTracerConfig - no traced languages', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
// No traced languages
config.languages = [languages_1.Language.javascript, languages_1.Language.python];
const codeQL = codeql_1.setCodeQL({
getTracerEnv: async function () {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
};
},
});
t.deepEqual(undefined, await tracer_config_1.getTracerConfig(config, codeQL));
});
});
ava_1.default('getTracerConfig - full', async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n2\nabc\ndef');
const codeQL = codeql_1.setCodeQL({
getTracerEnv: async function () {
return {
'ODASA_TRACER_CONFIGURATION': spec,
'foo': 'bar',
};
},
});
const result = await tracer_config_1.getTracerConfig(config, codeQL);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'foo': 'bar',
'ODASA_TRACER_CONFIGURATION': result.spec,
'LD_PRELOAD': path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so'),
}
});
});
});
//# sourceMappingURL=tracer-config.test.js.map

1
lib/tracer-config.test.js.map Normal file
View file

File diff suppressed because one or more lines are too long

7
lib/util.js generated
View file

@ -378,4 +378,11 @@ function getCodeQLDatabasesDir(tempDir) {
return path.resolve(tempDir, 'codeql_databases');
}
exports.getCodeQLDatabasesDir = getCodeQLDatabasesDir;
/**
* Get the path where the CodeQL database for the given language lives.
*/
function getCodeQLDatabasePath(tempDir, language) {
return path.resolve(getCodeQLDatabasesDir(tempDir), language);
}
exports.getCodeQLDatabasePath = getCodeQLDatabasePath;
//# sourceMappingURL=util.js.map

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

6
src/codeql.ts
View file

@ -49,7 +49,7 @@ export interface CodeQL {
* Run 'codeql database trace-command' on 'tracer-env.js' and parse
* the result to get environment variables set by CodeQL.
*/
getTracerEnv(databasePath: string, compilerSpec: string | undefined): Promise<{ [key: string]: string }>;
getTracerEnv(databasePath: string): Promise<{ [key: string]: string }>;
/**
* Run 'codeql database init'.
*/
@ -316,14 +316,12 @@ function getCodeQLForCmd(cmd: string): CodeQL {
'--format=json'
]);
},
getTracerEnv: async function(databasePath: string, compilerSpec: string | undefined) {
getTracerEnv: async function(databasePath: string) {
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
await exec.exec(cmd, [
'database',
'trace-command',
databasePath,
...compilerSpecArg,
...getExtraOptionsFromEnv(['database', 'trace-command']),
process.execPath,
path.resolve(__dirname, 'tracer-env.js'),

24
src/finalize-db.ts
View file

@ -58,36 +58,35 @@ async function sendStatusReport(
await util.sendStatusReport(statusReport);
}
async function createdDBForScannedLanguages(databaseFolder: string, config: configUtils.Config) {
async function createdDBForScannedLanguages(config: configUtils.Config) {
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
core.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(databaseFolder: string, config: configUtils.Config) {
await createdDBForScannedLanguages(databaseFolder, config);
async function finalizeDatabaseCreation(config: configUtils.Config) {
await createdDBForScannedLanguages(config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
core.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(path.join(databaseFolder, language));
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
core.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
databaseFolder: string,
sarifFolder: string,
config: configUtils.Config): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of fs.readdirSync(databaseFolder)) {
for (let language of config.languages) {
core.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
@ -96,16 +95,17 @@ async function runQueries(
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = path.join(databaseFolder, language + '-queries.qls');
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(path.join(databaseFolder, language), sarifFile, querySuite);
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
core.endGroup();
@ -135,16 +135,14 @@ async function run() {
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
const sarifFolder = core.getInput('output');
fs.mkdirSync(sarifFolder, { recursive: true });
core.info('Finalizing database creation');
await finalizeDatabaseCreation(databaseFolder, config);
await finalizeDatabaseCreation(config);
core.info('Analyzing database');
queriesStats = await runQueries(databaseFolder, sarifFolder, config);
queriesStats = await runQueries(sarifFolder, config);
if ('true' === core.getInput('upload')) {
uploadStats = await upload_lib.upload(

172
src/setup-tracer.ts
View file

@ -6,129 +6,9 @@ import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isTracedLanguage } from './languages';
import { getTracerConfig } from './tracer-config';
import * as util from './util';
type TracerConfig = {
spec: string;
env: { [key: string]: string };
};
const CRITICAL_TRACER_VARS = new Set(
['SEMMLE_PRELOAD_libtrace',
, 'SEMMLE_RUNNER',
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
, 'SEMMLE_DEPTRACE_SOCKET',
, 'SEMMLE_JAVA_TOOL_OPTIONS'
]);
async function tracerConfig(
codeql: CodeQL,
database: string,
compilerSpec?: string): Promise<TracerConfig> {
const env = await codeql.getTracerEnv(database, compilerSpec);
const config = env['ODASA_TRACER_CONFIGURATION'];
const info: TracerConfig = { spec: config, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
info.env[key] = value;
}
}
return info;
}
function concatTracerConfigs(tracerConfigs: TracerConfig[], config: configUtils.Config): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env: { [key: string]: string; } = {};
let copyExecutables = false;
let envSize = 0;
for (const v of tracerConfigs) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
copyExecutables = true;
} else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
}
} else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines: string[] = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
fs.writeFileSync(envPath, buffer);
return { env, spec };
}
interface InitSuccessStatusReport extends util.StatusReportBase {
// Comma-separated list of languages that analysis was run for
// This may be from the workflow file or may be calculated from repository contents
@ -215,48 +95,30 @@ async function run() {
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
fs.mkdirSync(databaseFolder, { recursive: true });
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
let tracedLanguageConfigs: TracerConfig[] = [];
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
const languageDatabase = path.join(databaseFolder, language);
// Init language database
await codeql.databaseInit(languageDatabase, language, sourceRoot);
// TODO: add better detection of 'traced languages' instead of using a hard coded list
if (isTracedLanguage(language)) {
const config: TracerConfig = await tracerConfig(codeql, languageDatabase);
tracedLanguageConfigs.push(config);
}
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
if (tracedLanguageConfigs.length > 0) {
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
if (mainTracerConfig.spec) {
for (let entry of Object.entries(mainTracerConfig.env)) {
core.exportVariable(entry[0], entry[1]);
}
core.exportVariable('ODASA_TRACER_CONFIGURATION', mainTracerConfig.spec);
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
core.exportVariable(
'DYLD_INSERT_LIBRARIES',
path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib'));
} else if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(codeQLDir, 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
} else {
core.exportVariable('LD_PRELOAD', path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so'));
}
const tracerConfig = await getTracerConfig(config, codeql);
if (tracerConfig !== undefined) {
if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
}
// NB: in CLI mode these will be outputted to a file instead of exported with core.exportVariable
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
}
} catch (error) {
core.setFailed(error.message);
console.log(error);

307
src/tracer-config.test.ts Normal file
View file

@ -0,0 +1,307 @@
import test from 'ava';
import * as fs from 'fs';
import * as path from 'path';
import { setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from './languages';
import { setupTests } from './testing-utils';
import { concatTracerConfigs, getTracerConfig, tracerConfig } from './tracer-config';
import * as util from './util';
setupTests(test);
function getTestConfig(tmpDir: string): configUtils.Config {
return {
languages: [Language.java],
queries: {},
pathsIgnore: [],
paths: [],
originalUserInput: {},
tempDir: tmpDir,
toolCacheDir: tmpDir,
codeQLCmd: '',
};
}
// A very minimal setup
test('tracerConfig - minimal', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
};
},
});
const result = await tracerConfig(codeQL, config, Language.javascript);
t.deepEqual(result, { spec: 'abc', env: {'foo': 'bar'} });
});
});
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
test('tracerConfig - existing / critical vars', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar',
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
};
},
});
process.env['foo'] = 'abc';
process.env['SEMMLE_PRELOAD_libtrace'] = 'abc';
process.env['SEMMLE_RUNNER'] = 'abc';
process.env['SEMMLE_COPY_EXECUTABLES_ROOT'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['SEMMLE_JAVA_TOOL_OPTIONS'] = 'abc';
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
process.env['CODEQL_VAR'] = 'abc';
const result = await tracerConfig(codeQL, config, Language.javascript);
t.deepEqual(result, {
spec: 'abc',
env: {
'baz': 'qux',
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
'CODEQL_VAR': 'CODEQL_VAR',
}
});
});
});
test('concatTracerConfigs - minimal', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
};
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, 'utf8'),
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nabc\ndef\nghi');
});
});
test('concatTracerConfigs - conflicting env vars', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
// Ok if env vars have the same name and the same value
t.deepEqual(
concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'b': 'b', 'c': 'c'}},
},
config).env,
{
'a': 'a',
'b': 'b',
'c': 'c',
});
// Throws if env vars have same name but different values
const e = t.throws(() =>
concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'b': 'c'}},
},
config));
t.deepEqual(e.message, 'Incompatible values in environment parameter b: b and c');
});
});
// If cpp is present then it's spec lines always come at the end
test('concatTracerConfigs - cpp comes last', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
'b': 'b',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'c': 'c',
}
};
const result = concatTracerConfigs({ 'cpp': tc1, 'python': tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'a': 'a',
'b': 'b',
'c': 'c',
}
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, 'utf8'),
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nghi\nabc\ndef');
});
});
test('concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n0');
const result = concatTracerConfigs(
{
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
'python': {spec: spec, env: {'SEMMLE_COPY_EXECUTABLES_ROOT': 'foo'}},
},
config);
t.deepEqual(result.env, {
'a': 'a',
'b': 'b',
'SEMMLE_COPY_EXECUTABLES_ROOT': path.join(tmpDir, 'compound-temp')
});
});
});
test('concatTracerConfigs - compound environment file', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, 'spec1');
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
const tc1 = {
spec: spec1,
env: {
'a': 'a',
}
};
const spec2 = path.join(tmpDir, 'spec2');
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
const tc2 = {
spec: spec2,
env: {
'foo': 'bar_baz',
}
};
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
const envPath = result.spec + '.environment';
t.true(fs.existsSync(envPath));
const buffer: Buffer = fs.readFileSync(envPath);
t.deepEqual(28, buffer.length);
t.deepEqual(2, buffer.readInt32LE(0));
t.deepEqual(4, buffer.readInt32LE(4));
t.deepEqual('a=a\0', buffer.toString('utf8', 8, 12));
t.deepEqual(12, buffer.readInt32LE(12));
t.deepEqual('foo=bar_baz\0', buffer.toString('utf8', 16, 28));
});
});
test('getTracerConfig - no traced languages', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
// No traced languages
config.languages = [Language.javascript, Language.python];
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': 'abc',
'foo': 'bar'
};
},
});
t.deepEqual(undefined, await getTracerConfig(config, codeQL));
});
});
test('getTracerConfig - full', async t => {
await util.withTmpDir(async tmpDir => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, 'spec');
fs.writeFileSync(spec, 'foo.log\n2\nabc\ndef');
const codeQL = setCodeQL({
getTracerEnv: async function() {
return {
'ODASA_TRACER_CONFIGURATION': spec,
'foo': 'bar',
};
},
});
const result = await getTracerConfig(config, codeQL);
t.deepEqual(result, {
spec: path.join(tmpDir, 'compound-spec'),
env: {
'foo': 'bar',
'ODASA_TRACER_CONFIGURATION': result!.spec,
'LD_PRELOAD': path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so'),
}
});
});
});

159
src/tracer-config.ts Normal file
View file

@ -0,0 +1,159 @@
import * as fs from 'fs';
import * as path from 'path';
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isTracedLanguage, Language } from './languages';
import * as util from './util';
export type TracerConfig = {
spec: string;
env: { [key: string]: string };
};
const CRITICAL_TRACER_VARS = new Set(
['SEMMLE_PRELOAD_libtrace',
, 'SEMMLE_RUNNER',
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
, 'SEMMLE_DEPTRACE_SOCKET',
, 'SEMMLE_JAVA_TOOL_OPTIONS'
]);
export async function tracerConfig(
codeql: CodeQL,
config: configUtils.Config,
language: Language): Promise<TracerConfig> {
const env = await codeql.getTracerEnv(util.getCodeQLDatabasePath(config.tempDir, language));
const spec = env['ODASA_TRACER_CONFIGURATION'];
const info: TracerConfig = { spec, env: {} };
// Extract critical tracer variables from the environment
for (let entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === 'ODASA_TRACER_CONFIGURATION') {
continue;
}
// skip undefined values
if (typeof value === 'undefined') {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
info.env[key] = value;
}
}
return info;
}
export function concatTracerConfigs(
tracerConfigs: { [lang: string]: TracerConfig },
config: configUtils.Config): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env: { [key: string]: string; } = {};
let copyExecutables = false;
let envSize = 0;
for (const v of Object.values(tracerConfigs)) {
for (let e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
copyExecutables = true;
} else if (name in env) {
if (env[name] !== value) {
throw Error('Incompatible values in environment parameter ' +
name + ': ' + env[name] + ' and ' + value);
}
} else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
let languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf('cpp');
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
let lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
let totalLines: string[] = [];
let totalCount = 0;
for (let lang of languages) {
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
const spec = path.resolve(config.tempDir, 'compound-spec');
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
if (copyExecutables) {
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join('\n'));
// Prepare the content of the compound environment file
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (let e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment
const envPath = spec + '.environment';
fs.writeFileSync(envPath, buffer);
return { env, spec };
}
export async function getTracerConfig(
config: configUtils.Config,
codeql: CodeQL): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter(isTracedLanguage);
if (tracedLanguages.length === 0) {
return undefined;
}
// Get all the tracer configs and combine them together
let tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
for (let language of tracedLanguages) {
tracedLanguageConfigs[language] = await tracerConfig(codeql, config, language);
}
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
// Add a couple more variables
mainTracerConfig.env['ODASA_TRACER_CONFIGURATION'] = mainTracerConfig.spec;
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === 'darwin') {
mainTracerConfig.env['DYLD_INSERT_LIBRARIES'] = path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib');
} else if (process.platform !== 'win32') {
mainTracerConfig.env['LD_PRELOAD'] = path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so');
}
return mainTracerConfig;
}

8
src/util.ts
View file

@ -5,6 +5,7 @@ import * as os from 'os';
import * as path from 'path';
import * as api from './api-client';
import { Language } from './languages';
import * as sharedEnv from './shared-environment';
/**
@ -433,3 +434,10 @@ export function getThreadsFlag(): string {
export function getCodeQLDatabasesDir(tempDir: string) {
return path.resolve(tempDir, 'codeql_databases');
}
/**
* Get the path where the CodeQL database for the given language lives.
*/
export function getCodeQLDatabasePath(tempDir: string, language: Language) {
return path.resolve(getCodeQLDatabasesDir(tempDir), language);
}