Merge branch 'main' into vercel/pkg
This commit is contained in:
Robert Brignull
commit
80a22f43d8
25 changed files with 1400 additions and 355 deletions
|
|
@ -16,6 +16,9 @@ inputs:
|
|||
config-file:
|
||||
description: Path of the config file to use
|
||||
required: false
|
||||
queries:
|
||||
description: Comma-separated list of additional queries to run. By default, this overrides the same setting in a configuration file
|
||||
required: false
|
||||
runs:
|
||||
using: 'node12'
|
||||
main: '../lib/init-action.js'
|
||||
|
|
|
|||
24
lib/analyze-action.js
generated
24
lib/analyze-action.js
generated
|
|
@ -29,43 +29,44 @@ async function sendStatusReport(startedAt, queriesStats, uploadStats, error) {
|
|||
};
|
||||
await util.sendStatusReport(statusReport);
|
||||
}
|
||||
async function createdDBForScannedLanguages(databaseFolder, config) {
|
||||
async function createdDBForScannedLanguages(config) {
|
||||
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
|
||||
for (const language of config.languages) {
|
||||
if (languages_1.isScannedLanguage(language)) {
|
||||
core.startGroup('Extracting ' + language);
|
||||
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
|
||||
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
|
||||
core.endGroup();
|
||||
}
|
||||
}
|
||||
}
|
||||
async function finalizeDatabaseCreation(databaseFolder, config) {
|
||||
await createdDBForScannedLanguages(databaseFolder, config);
|
||||
async function finalizeDatabaseCreation(config) {
|
||||
await createdDBForScannedLanguages(config);
|
||||
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
|
||||
for (const language of config.languages) {
|
||||
core.startGroup('Finalizing ' + language);
|
||||
await codeql.finalizeDatabase(path.join(databaseFolder, language));
|
||||
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
|
||||
core.endGroup();
|
||||
}
|
||||
}
|
||||
// Runs queries and creates sarif files in the given folder
|
||||
async function runQueries(databaseFolder, sarifFolder, config) {
|
||||
async function runQueries(sarifFolder, config) {
|
||||
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
|
||||
for (let language of fs.readdirSync(databaseFolder)) {
|
||||
for (let language of config.languages) {
|
||||
core.startGroup('Analyzing ' + language);
|
||||
const queries = config.queries[language] || [];
|
||||
if (queries.length === 0) {
|
||||
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
|
||||
}
|
||||
try {
|
||||
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
|
||||
// Pass the queries to codeql using a file instead of using the command
|
||||
// line to avoid command line length restrictions, particularly on windows.
|
||||
const querySuite = path.join(databaseFolder, language + '-queries.qls');
|
||||
const querySuite = databasePath + '-queries.qls';
|
||||
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
|
||||
fs.writeFileSync(querySuite, querySuiteContents);
|
||||
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
|
||||
const sarifFile = path.join(sarifFolder, language + '.sarif');
|
||||
await codeql.databaseAnalyze(path.join(databaseFolder, language), sarifFile, querySuite);
|
||||
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
|
||||
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
|
||||
core.endGroup();
|
||||
}
|
||||
|
|
@ -90,13 +91,12 @@ async function run() {
|
|||
const config = await configUtils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
|
||||
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
|
||||
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
|
||||
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
|
||||
const sarifFolder = core.getInput('output');
|
||||
fs.mkdirSync(sarifFolder, { recursive: true });
|
||||
core.info('Finalizing database creation');
|
||||
await finalizeDatabaseCreation(databaseFolder, config);
|
||||
await finalizeDatabaseCreation(config);
|
||||
core.info('Analyzing database');
|
||||
queriesStats = await runQueries(databaseFolder, sarifFolder, config);
|
||||
queriesStats = await runQueries(sarifFolder, config);
|
||||
if ('true' === core.getInput('upload')) {
|
||||
uploadStats = await upload_lib.upload(sarifFolder, repository_1.parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')), await util.getCommitOid(), util.getRef(), await util.getAnalysisKey(), util.getRequiredEnvParam('GITHUB_WORKFLOW'), util.getWorkflowRunID(), core.getInput('checkout_path'), core.getInput('matrix'), core.getInput('token'), util.getRequiredEnvParam('GITHUB_API_URL'), 'actions', logging_1.getActionsLogger());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,qCAAqC;AACrC,4DAA8C;AAC9C,2CAAgD;AAChD,uCAA6C;AAC7C,6CAAkD;AAClD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAiC/B,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,YAA6C,EAC7C,WAAsD,EACtD,KAAa;;IAEb,MAAM,MAAM,GAAG,OAAA,YAAY,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IACnH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,QAAE,KAAK,0CAAE,OAAO,QAAE,KAAK,0CAAE,KAAK,CAAC,CAAC;IACtH,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QACvB,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,cAAsB,EAAE,MAA0B;IAC5F,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YACnF,IAAI,CAAC,QAAQ,EAAE,CAAC;SACjB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,cAAsB,EAAE,MAA0B;IACxF,MAAM,4BAA4B,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;QAC1C,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,EAAE,CAAC;KACjB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,cAAsB,EACtB,WAAmB,EACnB,MAA0B;IAE1B,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,QAAQ,IAAI,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE;QACnD,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,QAAQ,GAAG,gDAAgD,CAAC,CAAC;SACrG;QAED,IAAI;YACF,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,cAAc,CAAC,CAAC;YACxE,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,uBAAuB,GAAG,QAAQ,GAAG,OAAO,GAAG,kBAAkB,CAAC,CAAC;YAE9E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAEzF,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,QAAQ,GAAG,eAAe,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;YACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;SAEjB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,YAAY,GAAoC,SAAS,CAAC;IAC9D,IAAI,WAAW,GAA8C,SAAS,CAAC;IACvE,IAAI;QACF,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE;YAC1G,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;QAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;QAEzD,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC1C,MAAM,wBAAwB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAEvD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAChC,YAAY,GAAG,MAAM,UAAU,CAAC,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QAErE,IAAI,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACtC,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACnC,WAAW,EACX,+BAAkB,CAAC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACjE,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,IAAI,CAAC,MAAM,EAAE,EACb,MAAM,IAAI,CAAC,cAAc,EAAE,EAC3B,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,EAC3C,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAC9B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAC1C,SAAS,EACT,0BAAgB,EAAE,CAAC,CAAC;SACvB;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO;KACR;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;AAC/D,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
||||
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,qCAAqC;AACrC,4DAA8C;AAC9C,2CAAgD;AAChD,uCAA6C;AAC7C,6CAAkD;AAClD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAiC/B,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,YAA6C,EAC7C,WAAsD,EACtD,KAAa;;IAEb,MAAM,MAAM,GAAG,OAAA,YAAY,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IACnH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,QAAE,KAAK,0CAAE,OAAO,QAAE,KAAK,0CAAE,KAAK,CAAC,CAAC;IACtH,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QACvB,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,MAA0B;IACpE,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YACpG,IAAI,CAAC,QAAQ,EAAE,CAAC;SACjB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,MAA0B;IAChE,MAAM,4BAA4B,CAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;QAC1C,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,QAAQ,EAAE,CAAC;KACjB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,WAAmB,EACnB,MAA0B;IAE1B,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACrC,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,QAAQ,GAAG,gDAAgD,CAAC,CAAC;SACrG;QAED,IAAI;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC1E,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,YAAY,GAAG,cAAc,CAAC;YACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,uBAAuB,GAAG,QAAQ,GAAG,OAAO,GAAG,kBAAkB,CAAC,CAAC;YAE9E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAElE,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,QAAQ,GAAG,eAAe,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;YACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;SAEjB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,YAAY,GAAoC,SAAS,CAAC;IAC9D,IAAI,WAAW,GAA8C,SAAS,CAAC;IACvE,IAAI;QACF,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE;YAC1G,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;QAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;QAEzD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC1C,MAAM,wBAAwB,CAAC,MAAM,CAAC,CAAC;QAEvC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAChC,YAAY,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAErD,IAAI,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACtC,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACnC,WAAW,EACX,+BAAkB,CAAC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACjE,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,IAAI,CAAC,MAAM,EAAE,EACb,MAAM,IAAI,CAAC,cAAc,EAAE,EAC3B,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,EAC3C,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAC9B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAC1C,SAAS,EACT,0BAAgB,EAAE,CAAC,CAAC;SACvB;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO;KACR;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;AAC/D,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
||||
4
lib/codeql.js
generated
4
lib/codeql.js
generated
|
|
@ -231,14 +231,12 @@ function getCodeQLForCmd(cmd) {
|
|||
'--format=json'
|
||||
]);
|
||||
},
|
||||
getTracerEnv: async function (databasePath, compilerSpec) {
|
||||
getTracerEnv: async function (databasePath) {
|
||||
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
|
||||
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
|
||||
await exec.exec(cmd, [
|
||||
'database',
|
||||
'trace-command',
|
||||
databasePath,
|
||||
...compilerSpecArg,
|
||||
...getExtraOptionsFromEnv(['database', 'trace-command']),
|
||||
process.execPath,
|
||||
path.resolve(__dirname, 'tracer-env.js'),
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
45
lib/config-utils.js
generated
45
lib/config-utils.js
generated
|
|
@ -89,7 +89,7 @@ const builtinSuites = ['security-extended', 'security-and-quality'];
|
|||
* Determine the set of queries associated with suiteName's suites and add them to resultMap.
|
||||
* Throws an error if suiteName is not a valid builtin suite.
|
||||
*/
|
||||
async function addBuiltinSuiteQueries(configFile, languages, codeQL, resultMap, suiteName) {
|
||||
async function addBuiltinSuiteQueries(languages, codeQL, resultMap, suiteName, configFile) {
|
||||
const suite = builtinSuites.find((suite) => suite === suiteName);
|
||||
if (!suite) {
|
||||
throw new Error(getQueryUsesInvalid(configFile, suiteName));
|
||||
|
|
@ -100,7 +100,7 @@ async function addBuiltinSuiteQueries(configFile, languages, codeQL, resultMap,
|
|||
/**
|
||||
* Retrieve the set of queries at localQueryPath and add them to resultMap.
|
||||
*/
|
||||
async function addLocalQueries(configFile, codeQL, resultMap, localQueryPath) {
|
||||
async function addLocalQueries(codeQL, resultMap, localQueryPath, configFile) {
|
||||
// Resolve the local path against the workspace so that when this is
|
||||
// passed to codeql it resolves to exactly the path we expect it to resolve to.
|
||||
const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE'));
|
||||
|
|
@ -122,7 +122,7 @@ async function addLocalQueries(configFile, codeQL, resultMap, localQueryPath) {
|
|||
/**
|
||||
* Retrieve the set of queries at the referenced remote repo and add them to resultMap.
|
||||
*/
|
||||
async function addRemoteQueries(configFile, codeQL, resultMap, queryUses, tempDir) {
|
||||
async function addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile) {
|
||||
let tok = queryUses.split('@');
|
||||
if (tok.length !== 2) {
|
||||
throw new Error(getQueryUsesInvalid(configFile, queryUses));
|
||||
|
|
@ -155,23 +155,23 @@ async function addRemoteQueries(configFile, codeQL, resultMap, queryUses, tempDi
|
|||
* local paths starting with './', or references to remote repos, or
|
||||
* a finite set of hardcoded terms for builtin suites.
|
||||
*/
|
||||
async function parseQueryUses(configFile, languages, codeQL, resultMap, queryUses, tempDir) {
|
||||
async function parseQueryUses(languages, codeQL, resultMap, queryUses, tempDir, configFile) {
|
||||
queryUses = queryUses.trim();
|
||||
if (queryUses === "") {
|
||||
throw new Error(getQueryUsesInvalid(configFile));
|
||||
}
|
||||
// Check for the local path case before we start trying to parse the repository name
|
||||
if (queryUses.startsWith("./")) {
|
||||
await addLocalQueries(configFile, codeQL, resultMap, queryUses.slice(2));
|
||||
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), configFile);
|
||||
return;
|
||||
}
|
||||
// Check for one of the builtin suites
|
||||
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
|
||||
await addBuiltinSuiteQueries(configFile, languages, codeQL, resultMap, queryUses);
|
||||
await addBuiltinSuiteQueries(languages, codeQL, resultMap, queryUses, configFile);
|
||||
return;
|
||||
}
|
||||
// Otherwise, must be a reference to another repo
|
||||
await addRemoteQueries(configFile, codeQL, resultMap, queryUses, tempDir);
|
||||
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile);
|
||||
}
|
||||
// Regex validating stars in paths or paths-ignore entries.
|
||||
// The intention is to only allow ** to appear when immediately
|
||||
|
|
@ -219,6 +219,8 @@ function validateAndSanitisePath(originalPath, propertyName, configFile) {
|
|||
return path;
|
||||
}
|
||||
exports.validateAndSanitisePath = validateAndSanitisePath;
|
||||
// An undefined configFile in some of these functions indicates that
|
||||
// the property was in a workflow file, not a config file
|
||||
function getNameInvalid(configFile) {
|
||||
return getConfigFilePropertyError(configFile, NAME_PROPERTY, 'must be a non-empty string');
|
||||
}
|
||||
|
|
@ -276,7 +278,12 @@ function getConfigFileDirectoryGivenMessage(configFile) {
|
|||
}
|
||||
exports.getConfigFileDirectoryGivenMessage = getConfigFileDirectoryGivenMessage;
|
||||
function getConfigFilePropertyError(configFile, property, error) {
|
||||
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
|
||||
if (configFile === undefined) {
|
||||
return 'The workflow property "' + property + '" is invalid: ' + error;
|
||||
}
|
||||
else {
|
||||
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
|
||||
}
|
||||
}
|
||||
function getNoLanguagesError() {
|
||||
return "Did not detect any languages to analyze. " +
|
||||
|
|
@ -363,6 +370,20 @@ async function getLanguages() {
|
|||
}
|
||||
return parsedLanguages;
|
||||
}
|
||||
/**
|
||||
* Returns true if queries were provided in the workflow file
|
||||
* (and thus added), otherwise false
|
||||
*/
|
||||
async function addQueriesFromWorkflowIfRequired(codeQL, languages, resultMap, tempDir) {
|
||||
const queryUses = core.getInput('queries');
|
||||
if (queryUses) {
|
||||
for (const query of queryUses.split(',')) {
|
||||
await parseQueryUses(languages, codeQL, resultMap, query, tempDir);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Get the default config for when the user has not supplied one.
|
||||
*/
|
||||
|
|
@ -370,6 +391,7 @@ async function getDefaultConfig(tempDir, toolCacheDir, codeQL) {
|
|||
const languages = await getLanguages();
|
||||
const queries = {};
|
||||
await addDefaultQueries(codeQL, languages, queries);
|
||||
await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
|
||||
return {
|
||||
languages: languages,
|
||||
queries: queries,
|
||||
|
|
@ -420,7 +442,10 @@ async function loadConfig(configFile, tempDir, toolCacheDir, codeQL) {
|
|||
if (!disableDefaultQueries) {
|
||||
await addDefaultQueries(codeQL, languages, queries);
|
||||
}
|
||||
if (QUERIES_PROPERTY in parsedYAML) {
|
||||
// If queries were provided using `with` in the action configuration,
|
||||
// they should take precedence over the queries in the config file
|
||||
const addedQueriesFromAction = await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
|
||||
if (!addedQueriesFromAction && QUERIES_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
|
||||
throw new Error(getQueriesInvalid(configFile));
|
||||
}
|
||||
|
|
@ -428,7 +453,7 @@ async function loadConfig(configFile, tempDir, toolCacheDir, codeQL) {
|
|||
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
|
||||
throw new Error(getQueryUsesInvalid(configFile));
|
||||
}
|
||||
await parseQueryUses(configFile, languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir);
|
||||
await parseQueryUses(languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir, configFile);
|
||||
}
|
||||
}
|
||||
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
159
lib/config-utils.test.js
generated
159
lib/config-utils.test.js
generated
|
|
@ -242,6 +242,165 @@ ava_1.default("default queries are used", async (t) => {
|
|||
t.deepEqual(resolveQueriesArgs[0].extraSearchPath, undefined);
|
||||
});
|
||||
});
|
||||
ava_1.default("Queries can be specified in config file", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
queries:
|
||||
- uses: ./foo`;
|
||||
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
|
||||
setInput('config-file', 'input');
|
||||
fs.mkdirSync(path.join(tmpDir, 'foo'));
|
||||
const resolveQueriesArgs = [];
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
resolveQueries: async function (queries, extraSearchPath) {
|
||||
resolveQueriesArgs.push({ queries, extraSearchPath });
|
||||
// Return what we're given, just in the right format for a resolved query
|
||||
// This way we can test by seeing which returned items are in the final
|
||||
// configuration.
|
||||
const dummyResolvedQueries = {};
|
||||
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': dummyResolvedQueries,
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
setInput('languages', 'javascript');
|
||||
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries
|
||||
// and once for `./foo` from the config file.
|
||||
t.deepEqual(resolveQueriesArgs.length, 2);
|
||||
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
|
||||
t.regex(resolveQueriesArgs[1].queries[0], /.*\/foo$/);
|
||||
// Now check that the end result contains the default queries and the query from config
|
||||
t.deepEqual(config.queries['javascript'].length, 2);
|
||||
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
|
||||
t.regex(config.queries['javascript'][1], /.*\/foo$/);
|
||||
});
|
||||
});
|
||||
ava_1.default("Queries from config file can be overridden in workflow file", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
queries:
|
||||
- uses: ./foo`;
|
||||
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
|
||||
setInput('config-file', 'input');
|
||||
// This config item should take precedence over the config file but shouldn't affect the default queries.
|
||||
setInput('queries', './override');
|
||||
fs.mkdirSync(path.join(tmpDir, 'foo'));
|
||||
fs.mkdirSync(path.join(tmpDir, 'override'));
|
||||
const resolveQueriesArgs = [];
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
resolveQueries: async function (queries, extraSearchPath) {
|
||||
resolveQueriesArgs.push({ queries, extraSearchPath });
|
||||
// Return what we're given, just in the right format for a resolved query
|
||||
// This way we can test overriding by seeing which returned items are in
|
||||
// the final configuration.
|
||||
const dummyResolvedQueries = {};
|
||||
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': dummyResolvedQueries,
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
setInput('languages', 'javascript');
|
||||
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries and once for `./override`,
|
||||
// but won't be called for './foo' from the config file.
|
||||
t.deepEqual(resolveQueriesArgs.length, 2);
|
||||
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
|
||||
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override$/);
|
||||
// Now check that the end result contains only the default queries and the override query
|
||||
t.deepEqual(config.queries['javascript'].length, 2);
|
||||
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
|
||||
t.regex(config.queries['javascript'][1], /.*\/override$/);
|
||||
});
|
||||
});
|
||||
ava_1.default("Multiple queries can be specified in workflow file, no config file required", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
fs.mkdirSync(path.join(tmpDir, 'override1'));
|
||||
fs.mkdirSync(path.join(tmpDir, 'override2'));
|
||||
setInput('queries', './override1,./override2');
|
||||
const resolveQueriesArgs = [];
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
resolveQueries: async function (queries, extraSearchPath) {
|
||||
resolveQueriesArgs.push({ queries, extraSearchPath });
|
||||
// Return what we're given, just in the right format for a resolved query
|
||||
// This way we can test overriding by seeing which returned items are in
|
||||
// the final configuration.
|
||||
const dummyResolvedQueries = {};
|
||||
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': dummyResolvedQueries,
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
setInput('languages', 'javascript');
|
||||
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
// Check resolveQueries was called correctly:
|
||||
// It'll be called once for the default queries,
|
||||
// and then once for each of the two queries from the workflow
|
||||
t.deepEqual(resolveQueriesArgs.length, 3);
|
||||
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
|
||||
t.deepEqual(resolveQueriesArgs[2].queries.length, 1);
|
||||
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override1$/);
|
||||
t.regex(resolveQueriesArgs[2].queries[0], /.*\/override2$/);
|
||||
// Now check that the end result contains both the queries from the workflow, as well as the defaults
|
||||
t.deepEqual(config.queries['javascript'].length, 3);
|
||||
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
|
||||
t.regex(config.queries['javascript'][1], /.*\/override1$/);
|
||||
t.regex(config.queries['javascript'][2], /.*\/override2$/);
|
||||
});
|
||||
});
|
||||
ava_1.default("Invalid queries in workflow file handled correctly", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
setInput('queries', 'foo/bar@v1@v3');
|
||||
setInput('languages', 'javascript');
|
||||
// This function just needs to be type-correct; it doesn't need to do anything,
|
||||
// since we're deliberately passing in invalid data
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
resolveQueries: async function (_queries, _extraSearchPath) {
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': {},
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
try {
|
||||
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
t.fail('initConfig did not throw error');
|
||||
}
|
||||
catch (err) {
|
||||
t.deepEqual(err, new Error(configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3")));
|
||||
}
|
||||
});
|
||||
});
|
||||
ava_1.default("API client used when reading remote config", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
146
lib/init-action.js
generated
146
lib/init-action.js
generated
|
|
@ -14,111 +14,8 @@ const path = __importStar(require("path"));
|
|||
const analysisPaths = __importStar(require("./analysis-paths"));
|
||||
const codeql_1 = require("./codeql");
|
||||
const configUtils = __importStar(require("./config-utils"));
|
||||
const languages_1 = require("./languages");
|
||||
const tracer_config_1 = require("./tracer-config");
|
||||
const util = __importStar(require("./util"));
|
||||
const CRITICAL_TRACER_VARS = new Set(['SEMMLE_PRELOAD_libtrace',
|
||||
,
|
||||
'SEMMLE_RUNNER',
|
||||
,
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
,
|
||||
'SEMMLE_DEPTRACE_SOCKET',
|
||||
,
|
||||
'SEMMLE_JAVA_TOOL_OPTIONS'
|
||||
]);
|
||||
async function tracerConfig(codeql, database, compilerSpec) {
|
||||
const env = await codeql.getTracerEnv(database, compilerSpec);
|
||||
const config = env['ODASA_TRACER_CONFIGURATION'];
|
||||
const info = { spec: config, env: {} };
|
||||
// Extract critical tracer variables from the environment
|
||||
for (let entry of Object.entries(env)) {
|
||||
const key = entry[0];
|
||||
const value = entry[1];
|
||||
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
|
||||
if (key === 'ODASA_TRACER_CONFIGURATION') {
|
||||
continue;
|
||||
}
|
||||
// skip undefined values
|
||||
if (typeof value === 'undefined') {
|
||||
continue;
|
||||
}
|
||||
// Keep variables that do not exist in current environment. In addition always keep
|
||||
// critical and CODEQL_ variables
|
||||
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
|
||||
info.env[key] = value;
|
||||
}
|
||||
}
|
||||
return info;
|
||||
}
|
||||
function concatTracerConfigs(tracerConfigs, config) {
|
||||
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
|
||||
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
|
||||
// Merge the environments
|
||||
const env = {};
|
||||
let copyExecutables = false;
|
||||
let envSize = 0;
|
||||
for (const v of tracerConfigs) {
|
||||
for (let e of Object.entries(v.env)) {
|
||||
const name = e[0];
|
||||
const value = e[1];
|
||||
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
|
||||
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
|
||||
copyExecutables = true;
|
||||
}
|
||||
else if (name in env) {
|
||||
if (env[name] !== value) {
|
||||
throw Error('Incompatible values in environment parameter ' +
|
||||
name + ': ' + env[name] + ' and ' + value);
|
||||
}
|
||||
}
|
||||
else {
|
||||
env[name] = value;
|
||||
envSize += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Concatenate spec files into a new spec file
|
||||
let languages = Object.keys(tracerConfigs);
|
||||
const cppIndex = languages.indexOf('cpp');
|
||||
// Make sure cpp is the last language, if it's present since it must be concatenated last
|
||||
if (cppIndex !== -1) {
|
||||
let lastLang = languages[languages.length - 1];
|
||||
languages[languages.length - 1] = languages[cppIndex];
|
||||
languages[cppIndex] = lastLang;
|
||||
}
|
||||
let totalLines = [];
|
||||
let totalCount = 0;
|
||||
for (let lang of languages) {
|
||||
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
|
||||
const count = parseInt(lines[1], 10);
|
||||
totalCount += count;
|
||||
totalLines.push(...lines.slice(2));
|
||||
}
|
||||
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
|
||||
const spec = path.resolve(config.tempDir, 'compound-spec');
|
||||
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
|
||||
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
|
||||
if (copyExecutables) {
|
||||
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
|
||||
envSize += 1;
|
||||
}
|
||||
fs.writeFileSync(spec, newSpecContent.join('\n'));
|
||||
// Prepare the content of the compound environment file
|
||||
let buffer = Buffer.alloc(4);
|
||||
buffer.writeInt32LE(envSize, 0);
|
||||
for (let e of Object.entries(env)) {
|
||||
const key = e[0];
|
||||
const value = e[1];
|
||||
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
|
||||
const sizeBuffer = Buffer.alloc(4);
|
||||
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
|
||||
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
|
||||
}
|
||||
// Write the compound environment
|
||||
const envPath = spec + '.environment';
|
||||
fs.writeFileSync(envPath, buffer);
|
||||
return { env, spec };
|
||||
}
|
||||
async function sendSuccessStatusReport(startedAt, config) {
|
||||
const statusReportBase = await util.createStatusReportBase('init', 'success', startedAt);
|
||||
const languages = config.languages.join(',');
|
||||
|
|
@ -173,41 +70,22 @@ async function run() {
|
|||
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
|
||||
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
|
||||
core.exportVariable('CODEQL_RAM', codeqlRam);
|
||||
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
|
||||
fs.mkdirSync(databaseFolder, { recursive: true });
|
||||
let tracedLanguageConfigs = [];
|
||||
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
|
||||
// TODO: replace this code once CodeQL supports multi-language tracing
|
||||
for (let language of config.languages) {
|
||||
const languageDatabase = path.join(databaseFolder, language);
|
||||
// Init language database
|
||||
await codeql.databaseInit(languageDatabase, language, sourceRoot);
|
||||
// TODO: add better detection of 'traced languages' instead of using a hard coded list
|
||||
if (languages_1.isTracedLanguage(language)) {
|
||||
const config = await tracerConfig(codeql, languageDatabase);
|
||||
tracedLanguageConfigs.push(config);
|
||||
}
|
||||
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
|
||||
}
|
||||
if (tracedLanguageConfigs.length > 0) {
|
||||
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
|
||||
if (mainTracerConfig.spec) {
|
||||
for (let entry of Object.entries(mainTracerConfig.env)) {
|
||||
core.exportVariable(entry[0], entry[1]);
|
||||
}
|
||||
core.exportVariable('ODASA_TRACER_CONFIGURATION', mainTracerConfig.spec);
|
||||
const codeQLDir = path.dirname(codeql.getPath());
|
||||
if (process.platform === 'darwin') {
|
||||
core.exportVariable('DYLD_INSERT_LIBRARIES', path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib'));
|
||||
}
|
||||
else if (process.platform === 'win32') {
|
||||
await exec.exec('powershell', [
|
||||
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
|
||||
path.resolve(codeQLDir, 'tools', 'win64', 'tracer.exe'),
|
||||
], { env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
|
||||
}
|
||||
else {
|
||||
core.exportVariable('LD_PRELOAD', path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so'));
|
||||
}
|
||||
const tracerConfig = await tracer_config_1.getCombinedTracerConfig(config, codeql);
|
||||
if (tracerConfig !== undefined) {
|
||||
if (process.platform === 'win32') {
|
||||
await exec.exec('powershell', [
|
||||
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
|
||||
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
|
||||
], { env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
|
||||
}
|
||||
// NB: in CLI mode these will be output to a file rather than exported with core.exportVariable
|
||||
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
143
lib/tracer-config.js
generated
Normal file
143
lib/tracer-config.js
generated
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"use strict";
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
|
||||
result["default"] = mod;
|
||||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const languages_1 = require("./languages");
|
||||
const util = __importStar(require("./util"));
|
||||
const CRITICAL_TRACER_VARS = new Set(['SEMMLE_PRELOAD_libtrace',
|
||||
,
|
||||
'SEMMLE_RUNNER',
|
||||
,
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
,
|
||||
'SEMMLE_DEPTRACE_SOCKET',
|
||||
,
|
||||
'SEMMLE_JAVA_TOOL_OPTIONS'
|
||||
]);
|
||||
async function getTracerConfigForLanguage(codeql, config, language) {
|
||||
const env = await codeql.getTracerEnv(util.getCodeQLDatabasePath(config.tempDir, language));
|
||||
const spec = env['ODASA_TRACER_CONFIGURATION'];
|
||||
const info = { spec, env: {} };
|
||||
// Extract critical tracer variables from the environment
|
||||
for (let entry of Object.entries(env)) {
|
||||
const key = entry[0];
|
||||
const value = entry[1];
|
||||
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
|
||||
if (key === 'ODASA_TRACER_CONFIGURATION') {
|
||||
continue;
|
||||
}
|
||||
// skip undefined values
|
||||
if (typeof value === 'undefined') {
|
||||
continue;
|
||||
}
|
||||
// Keep variables that do not exist in current environment. In addition always keep
|
||||
// critical and CODEQL_ variables
|
||||
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
|
||||
info.env[key] = value;
|
||||
}
|
||||
}
|
||||
return info;
|
||||
}
|
||||
exports.getTracerConfigForLanguage = getTracerConfigForLanguage;
|
||||
function concatTracerConfigs(tracerConfigs, config) {
|
||||
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
|
||||
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
|
||||
// Merge the environments
|
||||
const env = {};
|
||||
let copyExecutables = false;
|
||||
let envSize = 0;
|
||||
for (const v of Object.values(tracerConfigs)) {
|
||||
for (let e of Object.entries(v.env)) {
|
||||
const name = e[0];
|
||||
const value = e[1];
|
||||
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
|
||||
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
|
||||
copyExecutables = true;
|
||||
}
|
||||
else if (name in env) {
|
||||
if (env[name] !== value) {
|
||||
throw Error('Incompatible values in environment parameter ' +
|
||||
name + ': ' + env[name] + ' and ' + value);
|
||||
}
|
||||
}
|
||||
else {
|
||||
env[name] = value;
|
||||
envSize += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Concatenate spec files into a new spec file
|
||||
let languages = Object.keys(tracerConfigs);
|
||||
const cppIndex = languages.indexOf('cpp');
|
||||
// Make sure cpp is the last language, if it's present since it must be concatenated last
|
||||
if (cppIndex !== -1) {
|
||||
let lastLang = languages[languages.length - 1];
|
||||
languages[languages.length - 1] = languages[cppIndex];
|
||||
languages[cppIndex] = lastLang;
|
||||
}
|
||||
let totalLines = [];
|
||||
let totalCount = 0;
|
||||
for (let lang of languages) {
|
||||
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
|
||||
const count = parseInt(lines[1], 10);
|
||||
totalCount += count;
|
||||
totalLines.push(...lines.slice(2));
|
||||
}
|
||||
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
|
||||
const spec = path.resolve(config.tempDir, 'compound-spec');
|
||||
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
|
||||
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
|
||||
if (copyExecutables) {
|
||||
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
|
||||
envSize += 1;
|
||||
}
|
||||
fs.writeFileSync(spec, newSpecContent.join('\n'));
|
||||
// Prepare the content of the compound environment file
|
||||
let buffer = Buffer.alloc(4);
|
||||
buffer.writeInt32LE(envSize, 0);
|
||||
for (let e of Object.entries(env)) {
|
||||
const key = e[0];
|
||||
const value = e[1];
|
||||
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
|
||||
const sizeBuffer = Buffer.alloc(4);
|
||||
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
|
||||
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
|
||||
}
|
||||
// Write the compound environment
|
||||
const envPath = spec + '.environment';
|
||||
fs.writeFileSync(envPath, buffer);
|
||||
return { env, spec };
|
||||
}
|
||||
exports.concatTracerConfigs = concatTracerConfigs;
|
||||
async function getCombinedTracerConfig(config, codeql) {
|
||||
// Abort if there are no traced languages as there's nothing to do
|
||||
const tracedLanguages = config.languages.filter(languages_1.isTracedLanguage);
|
||||
if (tracedLanguages.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
// Get all the tracer configs and combine them together
|
||||
const tracedLanguageConfigs = {};
|
||||
for (const language of tracedLanguages) {
|
||||
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(codeql, config, language);
|
||||
}
|
||||
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
|
||||
// Add a couple more variables
|
||||
mainTracerConfig.env['ODASA_TRACER_CONFIGURATION'] = mainTracerConfig.spec;
|
||||
const codeQLDir = path.dirname(codeql.getPath());
|
||||
if (process.platform === 'darwin') {
|
||||
mainTracerConfig.env['DYLD_INSERT_LIBRARIES'] = path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib');
|
||||
}
|
||||
else if (process.platform !== 'win32') {
|
||||
mainTracerConfig.env['LD_PRELOAD'] = path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so');
|
||||
}
|
||||
return mainTracerConfig;
|
||||
}
|
||||
exports.getCombinedTracerConfig = getCombinedTracerConfig;
|
||||
//# sourceMappingURL=tracer-config.js.map
|
||||
1
lib/tracer-config.js.map
Normal file
1
lib/tracer-config.js.map
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"tracer-config.js","sourceRoot":"","sources":["../src/tracer-config.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAI7B,2CAAyD;AACzD,6CAA+B;AAO/B,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAClC,CAAC,yBAAyB;IACxB,AADyB;IACvB,eAAe;IACjB,AADkB;IAChB,8BAA8B;IAChC,AADiC;IAC/B,wBAAwB;IAC1B,AAD2B;IACzB,0BAA0B;CAC7B,CAAC,CAAC;AAEE,KAAK,UAAU,0BAA0B,CAC9C,MAAc,EACd,MAA0B,EAC1B,QAAkB;IAElB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAE5F,MAAM,IAAI,GAAG,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAE7C,yDAAyD;IACzD,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACrC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,8DAA8D;QAC9D,IAAI,GAAG,KAAK,4BAA4B,EAAE;YACxC,SAAS;SACV;QACD,wBAAwB;QACxB,IAAI,OAAO,KAAK,KAAK,WAAW,EAAE;YAChC,SAAS;SACV;QACD,mFAAmF;QACnF,iCAAiC;QACjC,IAAI,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,WAAW,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YACzG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACvB;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AA7BD,gEA6BC;AAED,SAAgB,mBAAmB,CACjC,aAA+C,EAC/C,MAA0B;IAE1B,iGAAiG;IACjG,0FAA0F;IAE1F,yBAAyB;IACzB,MAAM,GAAG,GAA+B,EAAE,CAAC;IAC3C,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QAC5C,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE;YACnC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,gEAAgE;YAChE,IAAI,IAAI,KAAK,8BAA8B,EAAE;gBAC3C,eAAe,GAAG,IAAI,CAAC;aACxB;iBAAM,IAAI,IAAI,IAAI,GAAG,EAAE;gBACtB,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE;oBACvB,MAAM,KAAK,CAAC,+CAA+C;wBACzD,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC;iBAC9C;aACF;iBAAM;gBACL,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;gBAClB,OAAO,IAAI,CAAC,CAAC;aACd;SACF;KACF;IAED,8CAA8C;IAC9C,IAAI,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1C,yFAAyF;IACzF,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE;QACnB,IAAI,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;QACtD,SAAS,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC;KAChC;IAED,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,IAAI,IAAI,SAAS,EAAE;QAC1B,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,UAAU,IAAI,KAAK,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KACpC;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAC3D,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACzE,MAAM,cAAc,GAAG,CAAC,cAAc,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC;IAEhF,IAAI,eAAe,EAAE;QACnB,GAAG,CAAC,8BAA8B,CAAC,GAAG,kBAAkB,CAAC;QACzD,OAAO,IAAI,CAAC,CAAC;KACd;IAED,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAElD,uDAAuD;IACvD,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAChC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACjC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,UAAU,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,GAAG,GAAG,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,UAAU,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;KAC1D;IACD,iCAAiC;IACjC,MAAM,OAAO,GAAG,IAAI,GAAG,cAAc,CAAC;IACtC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAElC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AACvB,CAAC;AA7ED,kDA6EC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,MAA0B,EAC1B,MAAc;IAEd,kEAAkE;IAClE,MAAM,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,4BAAgB,CAAC,CAAC;IAClE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;QAChC,OAAO,SAAS,CAAC;KAClB;IAED,uDAAuD;IACvD,MAAM,qBAAqB,GAAqC,EAAE,CAAC;IACnE,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE;QACtC,qBAAqB,CAAC,QAAQ,CAAC,GAAG,MAAM,0BAA0B,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KAC9F;IACD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAE5E,8BAA8B;IAC9B,gBAAgB,CAAC,GAAG,CAAC,4BAA4B,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC;IAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,gBAAgB,CAAC,GAAG,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;KAC1G;SAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QACvC,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;KACjG;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AA3BD,0DA2BC"}
|
||||
272
lib/tracer-config.test.js
generated
Normal file
272
lib/tracer-config.test.js
generated
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
|
||||
result["default"] = mod;
|
||||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const ava_1 = __importDefault(require("ava"));
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const codeql_1 = require("./codeql");
|
||||
const languages_1 = require("./languages");
|
||||
const testing_utils_1 = require("./testing-utils");
|
||||
const tracer_config_1 = require("./tracer-config");
|
||||
const util = __importStar(require("./util"));
|
||||
testing_utils_1.setupTests(ava_1.default);
|
||||
function getTestConfig(tmpDir) {
|
||||
return {
|
||||
languages: [languages_1.Language.java],
|
||||
queries: {},
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir: tmpDir,
|
||||
toolCacheDir: tmpDir,
|
||||
codeQLCmd: '',
|
||||
};
|
||||
}
|
||||
// A very minimal setup
|
||||
ava_1.default('getTracerConfigForLanguage - minimal setup', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
getTracerEnv: async function () {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': 'abc',
|
||||
'foo': 'bar'
|
||||
};
|
||||
},
|
||||
});
|
||||
const result = await tracer_config_1.getTracerConfigForLanguage(codeQL, config, languages_1.Language.javascript);
|
||||
t.deepEqual(result, { spec: 'abc', env: { 'foo': 'bar' } });
|
||||
});
|
||||
});
|
||||
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
|
||||
ava_1.default('getTracerConfigForLanguage - existing / critical vars', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
// Set up some variables in the environment
|
||||
process.env['foo'] = 'abc';
|
||||
process.env['SEMMLE_PRELOAD_libtrace'] = 'abc';
|
||||
process.env['SEMMLE_RUNNER'] = 'abc';
|
||||
process.env['SEMMLE_COPY_EXECUTABLES_ROOT'] = 'abc';
|
||||
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
|
||||
process.env['SEMMLE_JAVA_TOOL_OPTIONS'] = 'abc';
|
||||
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
|
||||
process.env['CODEQL_VAR'] = 'abc';
|
||||
// Now CodeQL returns all these variables, and one more, with different values
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
getTracerEnv: async function () {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': 'abc',
|
||||
'foo': 'bar',
|
||||
'baz': 'qux',
|
||||
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
|
||||
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
|
||||
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
|
||||
'CODEQL_VAR': 'CODEQL_VAR',
|
||||
};
|
||||
},
|
||||
});
|
||||
const result = await tracer_config_1.getTracerConfigForLanguage(codeQL, config, languages_1.Language.javascript);
|
||||
t.deepEqual(result, {
|
||||
spec: 'abc',
|
||||
env: {
|
||||
// Should contain all variables except 'foo', because that already existed in the
|
||||
// environment with a different value, and is not deemed a "critical" variable.
|
||||
'baz': 'qux',
|
||||
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
|
||||
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
|
||||
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
|
||||
'CODEQL_VAR': 'CODEQL_VAR',
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
ava_1.default('concatTracerConfigs - minimal configs correctly combined', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const spec1 = path.join(tmpDir, 'spec1');
|
||||
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
|
||||
const tc1 = {
|
||||
spec: spec1,
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
}
|
||||
};
|
||||
const spec2 = path.join(tmpDir, 'spec2');
|
||||
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
|
||||
const tc2 = {
|
||||
spec: spec2,
|
||||
env: {
|
||||
'c': 'c',
|
||||
}
|
||||
};
|
||||
const result = tracer_config_1.concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
|
||||
t.deepEqual(result, {
|
||||
spec: path.join(tmpDir, 'compound-spec'),
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'c': 'c',
|
||||
}
|
||||
});
|
||||
t.true(fs.existsSync(result.spec));
|
||||
t.deepEqual(fs.readFileSync(result.spec, 'utf8'), path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nabc\ndef\nghi');
|
||||
});
|
||||
});
|
||||
ava_1.default('concatTracerConfigs - conflicting env vars', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const spec = path.join(tmpDir, 'spec');
|
||||
fs.writeFileSync(spec, 'foo.log\n0');
|
||||
// Ok if env vars have the same name and the same value
|
||||
t.deepEqual(tracer_config_1.concatTracerConfigs({
|
||||
'javascript': { spec: spec, env: { 'a': 'a', 'b': 'b' } },
|
||||
'python': { spec: spec, env: { 'b': 'b', 'c': 'c' } },
|
||||
}, config).env, {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'c': 'c',
|
||||
});
|
||||
// Throws if env vars have same name but different values
|
||||
const e = t.throws(() => tracer_config_1.concatTracerConfigs({
|
||||
'javascript': { spec: spec, env: { 'a': 'a', 'b': 'b' } },
|
||||
'python': { spec: spec, env: { 'b': 'c' } },
|
||||
}, config));
|
||||
t.deepEqual(e.message, 'Incompatible values in environment parameter b: b and c');
|
||||
});
|
||||
});
|
||||
ava_1.default('concatTracerConfigs - cpp spec lines come last if present', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const spec1 = path.join(tmpDir, 'spec1');
|
||||
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
|
||||
const tc1 = {
|
||||
spec: spec1,
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
}
|
||||
};
|
||||
const spec2 = path.join(tmpDir, 'spec2');
|
||||
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
|
||||
const tc2 = {
|
||||
spec: spec2,
|
||||
env: {
|
||||
'c': 'c',
|
||||
}
|
||||
};
|
||||
const result = tracer_config_1.concatTracerConfigs({ 'cpp': tc1, 'python': tc2 }, config);
|
||||
t.deepEqual(result, {
|
||||
spec: path.join(tmpDir, 'compound-spec'),
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'c': 'c',
|
||||
}
|
||||
});
|
||||
t.true(fs.existsSync(result.spec));
|
||||
t.deepEqual(fs.readFileSync(result.spec, 'utf8'), path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nghi\nabc\ndef');
|
||||
});
|
||||
});
|
||||
ava_1.default('concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT is updated to point to compound spec', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const spec = path.join(tmpDir, 'spec');
|
||||
fs.writeFileSync(spec, 'foo.log\n0');
|
||||
const result = tracer_config_1.concatTracerConfigs({
|
||||
'javascript': { spec: spec, env: { 'a': 'a', 'b': 'b' } },
|
||||
'python': { spec: spec, env: { 'SEMMLE_COPY_EXECUTABLES_ROOT': 'foo' } },
|
||||
}, config);
|
||||
t.deepEqual(result.env, {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT': path.join(tmpDir, 'compound-temp')
|
||||
});
|
||||
});
|
||||
});
|
||||
ava_1.default('concatTracerConfigs - compound environment file is created correctly', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const spec1 = path.join(tmpDir, 'spec1');
|
||||
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
|
||||
const tc1 = {
|
||||
spec: spec1,
|
||||
env: {
|
||||
'a': 'a',
|
||||
}
|
||||
};
|
||||
const spec2 = path.join(tmpDir, 'spec2');
|
||||
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
|
||||
const tc2 = {
|
||||
spec: spec2,
|
||||
env: {
|
||||
'foo': 'bar_baz',
|
||||
}
|
||||
};
|
||||
const result = tracer_config_1.concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
|
||||
const envPath = result.spec + '.environment';
|
||||
t.true(fs.existsSync(envPath));
|
||||
const buffer = fs.readFileSync(envPath);
|
||||
// Contents is binary data
|
||||
t.deepEqual(buffer.length, 28);
|
||||
t.deepEqual(buffer.readInt32LE(0), 2); // number of env vars
|
||||
t.deepEqual(buffer.readInt32LE(4), 4); // length of env var definition
|
||||
t.deepEqual(buffer.toString('utf8', 8, 12), 'a=a\0'); // [key]=[value]\0
|
||||
t.deepEqual(buffer.readInt32LE(12), 12); // length of env var definition
|
||||
t.deepEqual(buffer.toString('utf8', 16, 28), 'foo=bar_baz\0'); // [key]=[value]\0
|
||||
});
|
||||
});
|
||||
ava_1.default('getCombinedTracerConfig - return undefined when no languages are traced languages', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
// No traced languages
|
||||
config.languages = [languages_1.Language.javascript, languages_1.Language.python];
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
getTracerEnv: async function () {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': 'abc',
|
||||
'foo': 'bar'
|
||||
};
|
||||
},
|
||||
});
|
||||
t.deepEqual(await tracer_config_1.getCombinedTracerConfig(config, codeQL), undefined);
|
||||
});
|
||||
});
|
||||
ava_1.default('getCombinedTracerConfig - valid spec file', async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
const spec = path.join(tmpDir, 'spec');
|
||||
fs.writeFileSync(spec, 'foo.log\n2\nabc\ndef');
|
||||
const codeQL = codeql_1.setCodeQL({
|
||||
getTracerEnv: async function () {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': spec,
|
||||
'foo': 'bar',
|
||||
};
|
||||
},
|
||||
});
|
||||
const result = await tracer_config_1.getCombinedTracerConfig(config, codeQL);
|
||||
t.deepEqual(result, {
|
||||
spec: path.join(tmpDir, 'compound-spec'),
|
||||
env: {
|
||||
'foo': 'bar',
|
||||
'ODASA_TRACER_CONFIGURATION': result.spec,
|
||||
'LD_PRELOAD': path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so'),
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
//# sourceMappingURL=tracer-config.test.js.map
|
||||
1
lib/tracer-config.test.js.map
Normal file
1
lib/tracer-config.test.js.map
Normal file
File diff suppressed because one or more lines are too long
7
lib/util.js
generated
7
lib/util.js
generated
|
|
@ -378,4 +378,11 @@ function getCodeQLDatabasesDir(tempDir) {
|
|||
return path.resolve(tempDir, 'codeql_databases');
|
||||
}
|
||||
exports.getCodeQLDatabasesDir = getCodeQLDatabasesDir;
|
||||
/**
|
||||
* Get the path where the CodeQL database for the given language lives.
|
||||
*/
|
||||
function getCodeQLDatabasePath(tempDir, language) {
|
||||
return path.resolve(getCodeQLDatabasesDir(tempDir), language);
|
||||
}
|
||||
exports.getCodeQLDatabasePath = getCodeQLDatabasePath;
|
||||
//# sourceMappingURL=util.js.map
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -58,36 +58,35 @@ async function sendStatusReport(
|
|||
await util.sendStatusReport(statusReport);
|
||||
}
|
||||
|
||||
async function createdDBForScannedLanguages(databaseFolder: string, config: configUtils.Config) {
|
||||
async function createdDBForScannedLanguages(config: configUtils.Config) {
|
||||
const codeql = getCodeQL(config.codeQLCmd);
|
||||
for (const language of config.languages) {
|
||||
if (isScannedLanguage(language)) {
|
||||
core.startGroup('Extracting ' + language);
|
||||
await codeql.extractScannedLanguage(path.join(databaseFolder, language), language);
|
||||
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
|
||||
core.endGroup();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeDatabaseCreation(databaseFolder: string, config: configUtils.Config) {
|
||||
await createdDBForScannedLanguages(databaseFolder, config);
|
||||
async function finalizeDatabaseCreation(config: configUtils.Config) {
|
||||
await createdDBForScannedLanguages(config);
|
||||
|
||||
const codeql = getCodeQL(config.codeQLCmd);
|
||||
for (const language of config.languages) {
|
||||
core.startGroup('Finalizing ' + language);
|
||||
await codeql.finalizeDatabase(path.join(databaseFolder, language));
|
||||
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
|
||||
core.endGroup();
|
||||
}
|
||||
}
|
||||
|
||||
// Runs queries and creates sarif files in the given folder
|
||||
async function runQueries(
|
||||
databaseFolder: string,
|
||||
sarifFolder: string,
|
||||
config: configUtils.Config): Promise<QueriesStatusReport> {
|
||||
|
||||
const codeql = getCodeQL(config.codeQLCmd);
|
||||
for (let language of fs.readdirSync(databaseFolder)) {
|
||||
for (let language of config.languages) {
|
||||
core.startGroup('Analyzing ' + language);
|
||||
|
||||
const queries = config.queries[language] || [];
|
||||
|
|
@ -96,16 +95,17 @@ async function runQueries(
|
|||
}
|
||||
|
||||
try {
|
||||
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
|
||||
// Pass the queries to codeql using a file instead of using the command
|
||||
// line to avoid command line length restrictions, particularly on windows.
|
||||
const querySuite = path.join(databaseFolder, language + '-queries.qls');
|
||||
const querySuite = databasePath + '-queries.qls';
|
||||
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
|
||||
fs.writeFileSync(querySuite, querySuiteContents);
|
||||
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
|
||||
|
||||
const sarifFile = path.join(sarifFolder, language + '.sarif');
|
||||
|
||||
await codeql.databaseAnalyze(path.join(databaseFolder, language), sarifFile, querySuite);
|
||||
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
|
||||
|
||||
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
|
||||
core.endGroup();
|
||||
|
|
@ -135,16 +135,14 @@ async function run() {
|
|||
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
|
||||
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
|
||||
|
||||
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
|
||||
|
||||
const sarifFolder = core.getInput('output');
|
||||
fs.mkdirSync(sarifFolder, { recursive: true });
|
||||
|
||||
core.info('Finalizing database creation');
|
||||
await finalizeDatabaseCreation(databaseFolder, config);
|
||||
await finalizeDatabaseCreation(config);
|
||||
|
||||
core.info('Analyzing database');
|
||||
queriesStats = await runQueries(databaseFolder, sarifFolder, config);
|
||||
queriesStats = await runQueries(sarifFolder, config);
|
||||
|
||||
if ('true' === core.getInput('upload')) {
|
||||
uploadStats = await upload_lib.upload(
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ export interface CodeQL {
|
|||
* Run 'codeql database trace-command' on 'tracer-env.js' and parse
|
||||
* the result to get environment variables set by CodeQL.
|
||||
*/
|
||||
getTracerEnv(databasePath: string, compilerSpec: string | undefined): Promise<{ [key: string]: string }>;
|
||||
getTracerEnv(databasePath: string): Promise<{ [key: string]: string }>;
|
||||
/**
|
||||
* Run 'codeql database init'.
|
||||
*/
|
||||
|
|
@ -316,14 +316,12 @@ function getCodeQLForCmd(cmd: string): CodeQL {
|
|||
'--format=json'
|
||||
]);
|
||||
},
|
||||
getTracerEnv: async function(databasePath: string, compilerSpec: string | undefined) {
|
||||
getTracerEnv: async function(databasePath: string) {
|
||||
let envFile = path.resolve(databasePath, 'working', 'env.tmp');
|
||||
const compilerSpecArg = compilerSpec ? ["--compiler-spec=" + compilerSpec] : [];
|
||||
await exec.exec(cmd, [
|
||||
'database',
|
||||
'trace-command',
|
||||
databasePath,
|
||||
...compilerSpecArg,
|
||||
...getExtraOptionsFromEnv(['database', 'trace-command']),
|
||||
process.execPath,
|
||||
path.resolve(__dirname, 'tracer-env.js'),
|
||||
|
|
|
|||
|
|
@ -273,6 +273,195 @@ test("default queries are used", async t => {
|
|||
});
|
||||
});
|
||||
|
||||
test("Queries can be specified in config file", async t => {
|
||||
return await util.withTmpDir(async tmpDir => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
queries:
|
||||
- uses: ./foo`;
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
|
||||
setInput('config-file', 'input');
|
||||
|
||||
fs.mkdirSync(path.join(tmpDir, 'foo'));
|
||||
|
||||
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
|
||||
const codeQL = setCodeQL({
|
||||
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
|
||||
resolveQueriesArgs.push({queries, extraSearchPath});
|
||||
// Return what we're given, just in the right format for a resolved query
|
||||
// This way we can test by seeing which returned items are in the final
|
||||
// configuration.
|
||||
const dummyResolvedQueries = {};
|
||||
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': dummyResolvedQueries,
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
setInput('languages', 'javascript');
|
||||
|
||||
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries
|
||||
// and once for `./foo` from the config file.
|
||||
t.deepEqual(resolveQueriesArgs.length, 2);
|
||||
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
|
||||
t.regex(resolveQueriesArgs[1].queries[0], /.*\/foo$/);
|
||||
|
||||
// Now check that the end result contains the default queries and the query from config
|
||||
t.deepEqual(config.queries['javascript'].length, 2);
|
||||
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
|
||||
t.regex(config.queries['javascript'][1], /.*\/foo$/);
|
||||
});
|
||||
});
|
||||
|
||||
test("Queries from config file can be overridden in workflow file", async t => {
|
||||
return await util.withTmpDir(async tmpDir => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
|
||||
const inputFileContents = `
|
||||
name: my config
|
||||
queries:
|
||||
- uses: ./foo`;
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
|
||||
setInput('config-file', 'input');
|
||||
|
||||
// This config item should take precedence over the config file but shouldn't affect the default queries.
|
||||
setInput('queries', './override');
|
||||
|
||||
fs.mkdirSync(path.join(tmpDir, 'foo'));
|
||||
fs.mkdirSync(path.join(tmpDir, 'override'));
|
||||
|
||||
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
|
||||
const codeQL = setCodeQL({
|
||||
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
|
||||
resolveQueriesArgs.push({queries, extraSearchPath});
|
||||
// Return what we're given, just in the right format for a resolved query
|
||||
// This way we can test overriding by seeing which returned items are in
|
||||
// the final configuration.
|
||||
const dummyResolvedQueries = {};
|
||||
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': dummyResolvedQueries,
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
setInput('languages', 'javascript');
|
||||
|
||||
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries and once for `./override`,
|
||||
// but won't be called for './foo' from the config file.
|
||||
t.deepEqual(resolveQueriesArgs.length, 2);
|
||||
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
|
||||
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override$/);
|
||||
|
||||
// Now check that the end result contains only the default queries and the override query
|
||||
t.deepEqual(config.queries['javascript'].length, 2);
|
||||
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
|
||||
t.regex(config.queries['javascript'][1], /.*\/override$/);
|
||||
});
|
||||
});
|
||||
|
||||
test("Multiple queries can be specified in workflow file, no config file required", async t => {
|
||||
return await util.withTmpDir(async tmpDir => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
|
||||
fs.mkdirSync(path.join(tmpDir, 'override1'));
|
||||
fs.mkdirSync(path.join(tmpDir, 'override2'));
|
||||
|
||||
setInput('queries', './override1,./override2');
|
||||
|
||||
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
|
||||
const codeQL = setCodeQL({
|
||||
resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
|
||||
resolveQueriesArgs.push({queries, extraSearchPath});
|
||||
// Return what we're given, just in the right format for a resolved query
|
||||
// This way we can test overriding by seeing which returned items are in
|
||||
// the final configuration.
|
||||
const dummyResolvedQueries = {};
|
||||
queries.forEach(q => { dummyResolvedQueries[q] = {}; });
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': dummyResolvedQueries,
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
setInput('languages', 'javascript');
|
||||
|
||||
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
|
||||
// Check resolveQueries was called correctly:
|
||||
// It'll be called once for the default queries,
|
||||
// and then once for each of the two queries from the workflow
|
||||
t.deepEqual(resolveQueriesArgs.length, 3);
|
||||
t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
|
||||
t.deepEqual(resolveQueriesArgs[2].queries.length, 1);
|
||||
t.regex(resolveQueriesArgs[1].queries[0], /.*\/override1$/);
|
||||
t.regex(resolveQueriesArgs[2].queries[0], /.*\/override2$/);
|
||||
|
||||
// Now check that the end result contains both the queries from the workflow, as well as the defaults
|
||||
t.deepEqual(config.queries['javascript'].length, 3);
|
||||
t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
|
||||
t.regex(config.queries['javascript'][1], /.*\/override1$/);
|
||||
t.regex(config.queries['javascript'][2], /.*\/override2$/);
|
||||
});
|
||||
});
|
||||
|
||||
test("Invalid queries in workflow file handled correctly", async t => {
|
||||
return await util.withTmpDir(async tmpDir => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
process.env['GITHUB_WORKSPACE'] = tmpDir;
|
||||
|
||||
setInput('queries', 'foo/bar@v1@v3');
|
||||
setInput('languages', 'javascript');
|
||||
|
||||
// This function just needs to be type-correct; it doesn't need to do anything,
|
||||
// since we're deliberately passing in invalid data
|
||||
const codeQL = setCodeQL({
|
||||
resolveQueries: async function(_queries: string[], _extraSearchPath: string | undefined) {
|
||||
return {
|
||||
byLanguage: {
|
||||
'javascript': {},
|
||||
},
|
||||
noDeclaredLanguage: {},
|
||||
multipleDeclaredLanguages: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
|
||||
t.fail('initConfig did not throw error');
|
||||
} catch (err) {
|
||||
t.deepEqual(err, new Error(configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3")));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("API client used when reading remote config", async t => {
|
||||
return await util.withTmpDir(async tmpDir => {
|
||||
process.env['RUNNER_TEMP'] = tmpDir;
|
||||
|
|
|
|||
|
|
@ -160,11 +160,11 @@ const builtinSuites = ['security-extended', 'security-and-quality'] as const;
|
|||
* Throws an error if suiteName is not a valid builtin suite.
|
||||
*/
|
||||
async function addBuiltinSuiteQueries(
|
||||
configFile: string,
|
||||
languages: string[],
|
||||
codeQL: CodeQL,
|
||||
resultMap: { [language: string]: string[] },
|
||||
suiteName: string) {
|
||||
suiteName: string,
|
||||
configFile?: string) {
|
||||
|
||||
const suite = builtinSuites.find((suite) => suite === suiteName);
|
||||
if (!suite) {
|
||||
|
|
@ -179,10 +179,10 @@ async function addBuiltinSuiteQueries(
|
|||
* Retrieve the set of queries at localQueryPath and add them to resultMap.
|
||||
*/
|
||||
async function addLocalQueries(
|
||||
configFile: string,
|
||||
codeQL: CodeQL,
|
||||
resultMap: { [language: string]: string[] },
|
||||
localQueryPath: string) {
|
||||
localQueryPath: string,
|
||||
configFile?: string) {
|
||||
|
||||
// Resolve the local path against the workspace so that when this is
|
||||
// passed to codeql it resolves to exactly the path we expect it to resolve to.
|
||||
|
|
@ -212,11 +212,11 @@ async function addLocalQueries(
|
|||
* Retrieve the set of queries at the referenced remote repo and add them to resultMap.
|
||||
*/
|
||||
async function addRemoteQueries(
|
||||
configFile: string,
|
||||
codeQL: CodeQL,
|
||||
resultMap: { [language: string]: string[] },
|
||||
queryUses: string,
|
||||
tempDir: string) {
|
||||
tempDir: string,
|
||||
configFile?: string) {
|
||||
|
||||
let tok = queryUses.split('@');
|
||||
if (tok.length !== 2) {
|
||||
|
|
@ -257,12 +257,12 @@ async function addRemoteQueries(
|
|||
* a finite set of hardcoded terms for builtin suites.
|
||||
*/
|
||||
async function parseQueryUses(
|
||||
configFile: string,
|
||||
languages: string[],
|
||||
codeQL: CodeQL,
|
||||
resultMap: { [language: string]: string[] },
|
||||
queryUses: string,
|
||||
tempDir: string) {
|
||||
tempDir: string,
|
||||
configFile?: string) {
|
||||
|
||||
queryUses = queryUses.trim();
|
||||
if (queryUses === "") {
|
||||
|
|
@ -271,18 +271,18 @@ async function parseQueryUses(
|
|||
|
||||
// Check for the local path case before we start trying to parse the repository name
|
||||
if (queryUses.startsWith("./")) {
|
||||
await addLocalQueries(configFile, codeQL, resultMap, queryUses.slice(2));
|
||||
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), configFile);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for one of the builtin suites
|
||||
if (queryUses.indexOf('/') === -1 && queryUses.indexOf('@') === -1) {
|
||||
await addBuiltinSuiteQueries(configFile, languages, codeQL, resultMap, queryUses);
|
||||
await addBuiltinSuiteQueries(languages, codeQL, resultMap, queryUses, configFile);
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, must be a reference to another repo
|
||||
await addRemoteQueries(configFile, codeQL, resultMap, queryUses, tempDir);
|
||||
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile);
|
||||
}
|
||||
|
||||
// Regex validating stars in paths or paths-ignore entries.
|
||||
|
|
@ -356,6 +356,9 @@ export function validateAndSanitisePath(
|
|||
return path;
|
||||
}
|
||||
|
||||
// An undefined configFile in some of these functions indicates that
|
||||
// the property was in a workflow file, not a config file
|
||||
|
||||
export function getNameInvalid(configFile: string): string {
|
||||
return getConfigFilePropertyError(configFile, NAME_PROPERTY, 'must be a non-empty string');
|
||||
}
|
||||
|
|
@ -368,7 +371,7 @@ export function getQueriesInvalid(configFile: string): string {
|
|||
return getConfigFilePropertyError(configFile, QUERIES_PROPERTY, 'must be an array');
|
||||
}
|
||||
|
||||
export function getQueryUsesInvalid(configFile: string, queryUses?: string): string {
|
||||
export function getQueryUsesInvalid(configFile: string | undefined, queryUses?: string): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
|
||||
|
|
@ -385,14 +388,14 @@ export function getPathsInvalid(configFile: string): string {
|
|||
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, 'must be an array of non-empty strings');
|
||||
}
|
||||
|
||||
export function getLocalPathOutsideOfRepository(configFile: string, localPath: string): string {
|
||||
export function getLocalPathOutsideOfRepository(configFile: string | undefined, localPath: string): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
|
||||
'is invalid as the local path "' + localPath + '" is outside of the repository');
|
||||
}
|
||||
|
||||
export function getLocalPathDoesNotExist(configFile: string, localPath: string): string {
|
||||
export function getLocalPathDoesNotExist(configFile: string | undefined, localPath: string): string {
|
||||
return getConfigFilePropertyError(
|
||||
configFile,
|
||||
QUERIES_PROPERTY + '.' + QUERIES_USES_PROPERTY,
|
||||
|
|
@ -422,8 +425,12 @@ export function getConfigFileDirectoryGivenMessage(configFile: string): string {
|
|||
return 'The configuration file "' + configFile + '" looks like a directory, not a file';
|
||||
}
|
||||
|
||||
function getConfigFilePropertyError(configFile: string, property: string, error: string): string {
|
||||
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
|
||||
function getConfigFilePropertyError(configFile: string | undefined, property: string, error: string): string {
|
||||
if (configFile === undefined) {
|
||||
return 'The workflow property "' + property + '" is invalid: ' + error;
|
||||
} else {
|
||||
return 'The configuration file "' + configFile + '" is invalid: property "' + property + '" ' + error;
|
||||
}
|
||||
}
|
||||
|
||||
export function getNoLanguagesError(): string {
|
||||
|
|
@ -518,6 +525,27 @@ async function getLanguages(): Promise<Language[]> {
|
|||
return parsedLanguages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if queries were provided in the workflow file
|
||||
* (and thus added), otherwise false
|
||||
*/
|
||||
async function addQueriesFromWorkflowIfRequired(
|
||||
codeQL: CodeQL,
|
||||
languages: string[],
|
||||
resultMap: { [language: string]: string[] },
|
||||
tempDir: string
|
||||
): Promise<boolean> {
|
||||
const queryUses = core.getInput('queries');
|
||||
if (queryUses) {
|
||||
for (const query of queryUses.split(',')) {
|
||||
await parseQueryUses(languages, codeQL, resultMap, query, tempDir);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the default config for when the user has not supplied one.
|
||||
*/
|
||||
|
|
@ -525,6 +553,8 @@ export async function getDefaultConfig(tempDir: string, toolCacheDir: string, co
|
|||
const languages = await getLanguages();
|
||||
const queries = {};
|
||||
await addDefaultQueries(codeQL, languages, queries);
|
||||
await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
|
||||
|
||||
return {
|
||||
languages: languages,
|
||||
queries: queries,
|
||||
|
|
@ -581,7 +611,10 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
|
|||
await addDefaultQueries(codeQL, languages, queries);
|
||||
}
|
||||
|
||||
if (QUERIES_PROPERTY in parsedYAML) {
|
||||
// If queries were provided using `with` in the action configuration,
|
||||
// they should take precedence over the queries in the config file
|
||||
const addedQueriesFromAction = await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
|
||||
if (!addedQueriesFromAction && QUERIES_PROPERTY in parsedYAML) {
|
||||
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
|
||||
throw new Error(getQueriesInvalid(configFile));
|
||||
}
|
||||
|
|
@ -589,7 +622,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
|
|||
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
|
||||
throw new Error(getQueryUsesInvalid(configFile));
|
||||
}
|
||||
await parseQueryUses(configFile, languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir);
|
||||
await parseQueryUses(languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir, configFile);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,129 +6,9 @@ import * as path from 'path';
|
|||
import * as analysisPaths from './analysis-paths';
|
||||
import { CodeQL, setupCodeQL } from './codeql';
|
||||
import * as configUtils from './config-utils';
|
||||
import { isTracedLanguage } from './languages';
|
||||
import { getCombinedTracerConfig } from './tracer-config';
|
||||
import * as util from './util';
|
||||
|
||||
type TracerConfig = {
|
||||
spec: string;
|
||||
env: { [key: string]: string };
|
||||
};
|
||||
|
||||
const CRITICAL_TRACER_VARS = new Set(
|
||||
['SEMMLE_PRELOAD_libtrace',
|
||||
, 'SEMMLE_RUNNER',
|
||||
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
, 'SEMMLE_DEPTRACE_SOCKET',
|
||||
, 'SEMMLE_JAVA_TOOL_OPTIONS'
|
||||
]);
|
||||
|
||||
async function tracerConfig(
|
||||
codeql: CodeQL,
|
||||
database: string,
|
||||
compilerSpec?: string): Promise<TracerConfig> {
|
||||
|
||||
const env = await codeql.getTracerEnv(database, compilerSpec);
|
||||
|
||||
const config = env['ODASA_TRACER_CONFIGURATION'];
|
||||
const info: TracerConfig = { spec: config, env: {} };
|
||||
|
||||
// Extract critical tracer variables from the environment
|
||||
for (let entry of Object.entries(env)) {
|
||||
const key = entry[0];
|
||||
const value = entry[1];
|
||||
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
|
||||
if (key === 'ODASA_TRACER_CONFIGURATION') {
|
||||
continue;
|
||||
}
|
||||
// skip undefined values
|
||||
if (typeof value === 'undefined') {
|
||||
continue;
|
||||
}
|
||||
// Keep variables that do not exist in current environment. In addition always keep
|
||||
// critical and CODEQL_ variables
|
||||
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
|
||||
info.env[key] = value;
|
||||
}
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
function concatTracerConfigs(tracerConfigs: TracerConfig[], config: configUtils.Config): TracerConfig {
|
||||
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
|
||||
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
|
||||
|
||||
// Merge the environments
|
||||
const env: { [key: string]: string; } = {};
|
||||
let copyExecutables = false;
|
||||
let envSize = 0;
|
||||
for (const v of tracerConfigs) {
|
||||
for (let e of Object.entries(v.env)) {
|
||||
const name = e[0];
|
||||
const value = e[1];
|
||||
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
|
||||
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
|
||||
copyExecutables = true;
|
||||
} else if (name in env) {
|
||||
if (env[name] !== value) {
|
||||
throw Error('Incompatible values in environment parameter ' +
|
||||
name + ': ' + env[name] + ' and ' + value);
|
||||
}
|
||||
} else {
|
||||
env[name] = value;
|
||||
envSize += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Concatenate spec files into a new spec file
|
||||
let languages = Object.keys(tracerConfigs);
|
||||
const cppIndex = languages.indexOf('cpp');
|
||||
// Make sure cpp is the last language, if it's present since it must be concatenated last
|
||||
if (cppIndex !== -1) {
|
||||
let lastLang = languages[languages.length - 1];
|
||||
languages[languages.length - 1] = languages[cppIndex];
|
||||
languages[cppIndex] = lastLang;
|
||||
}
|
||||
|
||||
let totalLines: string[] = [];
|
||||
let totalCount = 0;
|
||||
for (let lang of languages) {
|
||||
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
|
||||
const count = parseInt(lines[1], 10);
|
||||
totalCount += count;
|
||||
totalLines.push(...lines.slice(2));
|
||||
}
|
||||
|
||||
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
|
||||
const spec = path.resolve(config.tempDir, 'compound-spec');
|
||||
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
|
||||
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
|
||||
|
||||
if (copyExecutables) {
|
||||
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
|
||||
envSize += 1;
|
||||
}
|
||||
|
||||
fs.writeFileSync(spec, newSpecContent.join('\n'));
|
||||
|
||||
// Prepare the content of the compound environment file
|
||||
let buffer = Buffer.alloc(4);
|
||||
buffer.writeInt32LE(envSize, 0);
|
||||
for (let e of Object.entries(env)) {
|
||||
const key = e[0];
|
||||
const value = e[1];
|
||||
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
|
||||
const sizeBuffer = Buffer.alloc(4);
|
||||
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
|
||||
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
|
||||
}
|
||||
// Write the compound environment
|
||||
const envPath = spec + '.environment';
|
||||
fs.writeFileSync(envPath, buffer);
|
||||
|
||||
return { env, spec };
|
||||
}
|
||||
|
||||
interface InitSuccessStatusReport extends util.StatusReportBase {
|
||||
// Comma-separated list of languages that analysis was run for
|
||||
// This may be from the workflow file or may be calculated from repository contents
|
||||
|
|
@ -215,48 +95,30 @@ async function run() {
|
|||
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
|
||||
core.exportVariable('CODEQL_RAM', codeqlRam);
|
||||
|
||||
const databaseFolder = util.getCodeQLDatabasesDir(config.tempDir);
|
||||
fs.mkdirSync(databaseFolder, { recursive: true });
|
||||
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
|
||||
|
||||
let tracedLanguageConfigs: TracerConfig[] = [];
|
||||
// TODO: replace this code once CodeQL supports multi-language tracing
|
||||
for (let language of config.languages) {
|
||||
const languageDatabase = path.join(databaseFolder, language);
|
||||
|
||||
// Init language database
|
||||
await codeql.databaseInit(languageDatabase, language, sourceRoot);
|
||||
// TODO: add better detection of 'traced languages' instead of using a hard coded list
|
||||
if (isTracedLanguage(language)) {
|
||||
const config: TracerConfig = await tracerConfig(codeql, languageDatabase);
|
||||
tracedLanguageConfigs.push(config);
|
||||
}
|
||||
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
|
||||
}
|
||||
if (tracedLanguageConfigs.length > 0) {
|
||||
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
|
||||
if (mainTracerConfig.spec) {
|
||||
for (let entry of Object.entries(mainTracerConfig.env)) {
|
||||
core.exportVariable(entry[0], entry[1]);
|
||||
}
|
||||
|
||||
core.exportVariable('ODASA_TRACER_CONFIGURATION', mainTracerConfig.spec);
|
||||
const codeQLDir = path.dirname(codeql.getPath());
|
||||
if (process.platform === 'darwin') {
|
||||
core.exportVariable(
|
||||
'DYLD_INSERT_LIBRARIES',
|
||||
path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib'));
|
||||
} else if (process.platform === 'win32') {
|
||||
await exec.exec(
|
||||
'powershell',
|
||||
[
|
||||
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
|
||||
path.resolve(codeQLDir, 'tools', 'win64', 'tracer.exe'),
|
||||
],
|
||||
{ env: { 'ODASA_TRACER_CONFIGURATION': mainTracerConfig.spec } });
|
||||
} else {
|
||||
core.exportVariable('LD_PRELOAD', path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so'));
|
||||
}
|
||||
const tracerConfig = await getCombinedTracerConfig(config, codeql);
|
||||
if (tracerConfig !== undefined) {
|
||||
if (process.platform === 'win32') {
|
||||
await exec.exec(
|
||||
'powershell',
|
||||
[
|
||||
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
|
||||
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
|
||||
],
|
||||
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
|
||||
}
|
||||
|
||||
// NB: in CLI mode these will be output to a file rather than exported with core.exportVariable
|
||||
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
core.setFailed(error.message);
|
||||
console.log(error);
|
||||
|
|
|
|||
311
src/tracer-config.test.ts
Normal file
311
src/tracer-config.test.ts
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
import test from 'ava';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
import { setCodeQL } from './codeql';
|
||||
import * as configUtils from './config-utils';
|
||||
import { Language } from './languages';
|
||||
import { setupTests } from './testing-utils';
|
||||
import { concatTracerConfigs, getCombinedTracerConfig, getTracerConfigForLanguage } from './tracer-config';
|
||||
import * as util from './util';
|
||||
|
||||
setupTests(test);
|
||||
|
||||
function getTestConfig(tmpDir: string): configUtils.Config {
|
||||
return {
|
||||
languages: [Language.java],
|
||||
queries: {},
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir: tmpDir,
|
||||
toolCacheDir: tmpDir,
|
||||
codeQLCmd: '',
|
||||
};
|
||||
}
|
||||
|
||||
// A very minimal setup
|
||||
test('getTracerConfigForLanguage - minimal setup', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const codeQL = setCodeQL({
|
||||
getTracerEnv: async function() {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': 'abc',
|
||||
'foo': 'bar'
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const result = await getTracerConfigForLanguage(codeQL, config, Language.javascript);
|
||||
t.deepEqual(result, { spec: 'abc', env: {'foo': 'bar'} });
|
||||
});
|
||||
});
|
||||
|
||||
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
|
||||
test('getTracerConfigForLanguage - existing / critical vars', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
// Set up some variables in the environment
|
||||
process.env['foo'] = 'abc';
|
||||
process.env['SEMMLE_PRELOAD_libtrace'] = 'abc';
|
||||
process.env['SEMMLE_RUNNER'] = 'abc';
|
||||
process.env['SEMMLE_COPY_EXECUTABLES_ROOT'] = 'abc';
|
||||
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
|
||||
process.env['SEMMLE_JAVA_TOOL_OPTIONS'] = 'abc';
|
||||
process.env['SEMMLE_DEPTRACE_SOCKET'] = 'abc';
|
||||
process.env['CODEQL_VAR'] = 'abc';
|
||||
|
||||
// Now CodeQL returns all these variables, and one more, with different values
|
||||
const codeQL = setCodeQL({
|
||||
getTracerEnv: async function() {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': 'abc',
|
||||
'foo': 'bar',
|
||||
'baz': 'qux',
|
||||
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
|
||||
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
|
||||
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
|
||||
'CODEQL_VAR': 'CODEQL_VAR',
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const result = await getTracerConfigForLanguage(codeQL, config, Language.javascript);
|
||||
t.deepEqual(result, {
|
||||
spec: 'abc',
|
||||
env: {
|
||||
// Should contain all variables except 'foo', because that already existed in the
|
||||
// environment with a different value, and is not deemed a "critical" variable.
|
||||
'baz': 'qux',
|
||||
'SEMMLE_PRELOAD_libtrace': 'SEMMLE_PRELOAD_libtrace',
|
||||
'SEMMLE_RUNNER': 'SEMMLE_RUNNER',
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT': 'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
'SEMMLE_DEPTRACE_SOCKET': 'SEMMLE_DEPTRACE_SOCKET',
|
||||
'SEMMLE_JAVA_TOOL_OPTIONS': 'SEMMLE_JAVA_TOOL_OPTIONS',
|
||||
'CODEQL_VAR': 'CODEQL_VAR',
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('concatTracerConfigs - minimal configs correctly combined', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const spec1 = path.join(tmpDir, 'spec1');
|
||||
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
|
||||
const tc1 = {
|
||||
spec: spec1,
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
}
|
||||
};
|
||||
|
||||
const spec2 = path.join(tmpDir, 'spec2');
|
||||
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
|
||||
const tc2 = {
|
||||
spec: spec2,
|
||||
env: {
|
||||
'c': 'c',
|
||||
}
|
||||
};
|
||||
|
||||
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
|
||||
t.deepEqual(result, {
|
||||
spec: path.join(tmpDir, 'compound-spec'),
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'c': 'c',
|
||||
}
|
||||
});
|
||||
t.true(fs.existsSync(result.spec));
|
||||
t.deepEqual(
|
||||
fs.readFileSync(result.spec, 'utf8'),
|
||||
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nabc\ndef\nghi');
|
||||
});
|
||||
});
|
||||
|
||||
test('concatTracerConfigs - conflicting env vars', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const spec = path.join(tmpDir, 'spec');
|
||||
fs.writeFileSync(spec, 'foo.log\n0');
|
||||
|
||||
// Ok if env vars have the same name and the same value
|
||||
t.deepEqual(
|
||||
concatTracerConfigs(
|
||||
{
|
||||
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
|
||||
'python': {spec: spec, env: {'b': 'b', 'c': 'c'}},
|
||||
},
|
||||
config).env,
|
||||
{
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'c': 'c',
|
||||
});
|
||||
|
||||
// Throws if env vars have same name but different values
|
||||
const e = t.throws(() =>
|
||||
concatTracerConfigs(
|
||||
{
|
||||
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
|
||||
'python': {spec: spec, env: {'b': 'c'}},
|
||||
},
|
||||
config));
|
||||
t.deepEqual(e.message, 'Incompatible values in environment parameter b: b and c');
|
||||
});
|
||||
});
|
||||
|
||||
test('concatTracerConfigs - cpp spec lines come last if present', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const spec1 = path.join(tmpDir, 'spec1');
|
||||
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
|
||||
const tc1 = {
|
||||
spec: spec1,
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
}
|
||||
};
|
||||
|
||||
const spec2 = path.join(tmpDir, 'spec2');
|
||||
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
|
||||
const tc2 = {
|
||||
spec: spec2,
|
||||
env: {
|
||||
'c': 'c',
|
||||
}
|
||||
};
|
||||
|
||||
const result = concatTracerConfigs({ 'cpp': tc1, 'python': tc2 }, config);
|
||||
t.deepEqual(result, {
|
||||
spec: path.join(tmpDir, 'compound-spec'),
|
||||
env: {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'c': 'c',
|
||||
}
|
||||
});
|
||||
t.true(fs.existsSync(result.spec));
|
||||
t.deepEqual(
|
||||
fs.readFileSync(result.spec, 'utf8'),
|
||||
path.join(tmpDir, 'compound-build-tracer.log') + '\n3\nghi\nabc\ndef');
|
||||
});
|
||||
});
|
||||
|
||||
test('concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT is updated to point to compound spec', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const spec = path.join(tmpDir, 'spec');
|
||||
fs.writeFileSync(spec, 'foo.log\n0');
|
||||
|
||||
const result = concatTracerConfigs(
|
||||
{
|
||||
'javascript': {spec: spec, env: {'a': 'a', 'b': 'b'}},
|
||||
'python': {spec: spec, env: {'SEMMLE_COPY_EXECUTABLES_ROOT': 'foo'}},
|
||||
},
|
||||
config);
|
||||
|
||||
t.deepEqual(result.env, {
|
||||
'a': 'a',
|
||||
'b': 'b',
|
||||
'SEMMLE_COPY_EXECUTABLES_ROOT': path.join(tmpDir, 'compound-temp')
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('concatTracerConfigs - compound environment file is created correctly', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const spec1 = path.join(tmpDir, 'spec1');
|
||||
fs.writeFileSync(spec1, 'foo.log\n2\nabc\ndef');
|
||||
const tc1 = {
|
||||
spec: spec1,
|
||||
env: {
|
||||
'a': 'a',
|
||||
}
|
||||
};
|
||||
|
||||
const spec2 = path.join(tmpDir, 'spec2');
|
||||
fs.writeFileSync(spec2, 'foo.log\n1\nghi');
|
||||
const tc2 = {
|
||||
spec: spec2,
|
||||
env: {
|
||||
'foo': 'bar_baz',
|
||||
}
|
||||
};
|
||||
|
||||
const result = concatTracerConfigs({ 'javascript': tc1, 'python': tc2 }, config);
|
||||
const envPath = result.spec + '.environment';
|
||||
t.true(fs.existsSync(envPath));
|
||||
|
||||
const buffer: Buffer = fs.readFileSync(envPath);
|
||||
// Contents is binary data
|
||||
t.deepEqual(buffer.length, 28);
|
||||
t.deepEqual(buffer.readInt32LE(0), 2); // number of env vars
|
||||
t.deepEqual(buffer.readInt32LE(4), 4); // length of env var definition
|
||||
t.deepEqual(buffer.toString('utf8', 8, 12), 'a=a\0'); // [key]=[value]\0
|
||||
t.deepEqual(buffer.readInt32LE(12), 12); // length of env var definition
|
||||
t.deepEqual(buffer.toString('utf8', 16, 28), 'foo=bar_baz\0'); // [key]=[value]\0
|
||||
});
|
||||
});
|
||||
|
||||
test('getCombinedTracerConfig - return undefined when no languages are traced languages', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
// No traced languages
|
||||
config.languages = [Language.javascript, Language.python];
|
||||
|
||||
const codeQL = setCodeQL({
|
||||
getTracerEnv: async function() {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': 'abc',
|
||||
'foo': 'bar'
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
t.deepEqual(await getCombinedTracerConfig(config, codeQL), undefined);
|
||||
});
|
||||
});
|
||||
|
||||
test('getCombinedTracerConfig - valid spec file', async t => {
|
||||
await util.withTmpDir(async tmpDir => {
|
||||
const config = getTestConfig(tmpDir);
|
||||
|
||||
const spec = path.join(tmpDir, 'spec');
|
||||
fs.writeFileSync(spec, 'foo.log\n2\nabc\ndef');
|
||||
|
||||
const codeQL = setCodeQL({
|
||||
getTracerEnv: async function() {
|
||||
return {
|
||||
'ODASA_TRACER_CONFIGURATION': spec,
|
||||
'foo': 'bar',
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const result = await getCombinedTracerConfig(config, codeQL);
|
||||
t.deepEqual(result, {
|
||||
spec: path.join(tmpDir, 'compound-spec'),
|
||||
env: {
|
||||
'foo': 'bar',
|
||||
'ODASA_TRACER_CONFIGURATION': result!.spec,
|
||||
'LD_PRELOAD': path.join(path.dirname(codeQL.getPath()), 'tools', 'linux64', '${LIB}trace.so'),
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
159
src/tracer-config.ts
Normal file
159
src/tracer-config.ts
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
import { CodeQL } from './codeql';
|
||||
import * as configUtils from './config-utils';
|
||||
import { isTracedLanguage, Language } from './languages';
|
||||
import * as util from './util';
|
||||
|
||||
export type TracerConfig = {
|
||||
spec: string;
|
||||
env: { [key: string]: string };
|
||||
};
|
||||
|
||||
const CRITICAL_TRACER_VARS = new Set(
|
||||
['SEMMLE_PRELOAD_libtrace',
|
||||
, 'SEMMLE_RUNNER',
|
||||
, 'SEMMLE_COPY_EXECUTABLES_ROOT',
|
||||
, 'SEMMLE_DEPTRACE_SOCKET',
|
||||
, 'SEMMLE_JAVA_TOOL_OPTIONS'
|
||||
]);
|
||||
|
||||
export async function getTracerConfigForLanguage(
|
||||
codeql: CodeQL,
|
||||
config: configUtils.Config,
|
||||
language: Language): Promise<TracerConfig> {
|
||||
|
||||
const env = await codeql.getTracerEnv(util.getCodeQLDatabasePath(config.tempDir, language));
|
||||
|
||||
const spec = env['ODASA_TRACER_CONFIGURATION'];
|
||||
const info: TracerConfig = { spec, env: {} };
|
||||
|
||||
// Extract critical tracer variables from the environment
|
||||
for (let entry of Object.entries(env)) {
|
||||
const key = entry[0];
|
||||
const value = entry[1];
|
||||
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
|
||||
if (key === 'ODASA_TRACER_CONFIGURATION') {
|
||||
continue;
|
||||
}
|
||||
// skip undefined values
|
||||
if (typeof value === 'undefined') {
|
||||
continue;
|
||||
}
|
||||
// Keep variables that do not exist in current environment. In addition always keep
|
||||
// critical and CODEQL_ variables
|
||||
if (typeof process.env[key] === 'undefined' || CRITICAL_TRACER_VARS.has(key) || key.startsWith('CODEQL_')) {
|
||||
info.env[key] = value;
|
||||
}
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
export function concatTracerConfigs(
|
||||
tracerConfigs: { [lang: string]: TracerConfig },
|
||||
config: configUtils.Config): TracerConfig {
|
||||
|
||||
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
|
||||
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
|
||||
|
||||
// Merge the environments
|
||||
const env: { [key: string]: string; } = {};
|
||||
let copyExecutables = false;
|
||||
let envSize = 0;
|
||||
for (const v of Object.values(tracerConfigs)) {
|
||||
for (let e of Object.entries(v.env)) {
|
||||
const name = e[0];
|
||||
const value = e[1];
|
||||
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
|
||||
if (name === 'SEMMLE_COPY_EXECUTABLES_ROOT') {
|
||||
copyExecutables = true;
|
||||
} else if (name in env) {
|
||||
if (env[name] !== value) {
|
||||
throw Error('Incompatible values in environment parameter ' +
|
||||
name + ': ' + env[name] + ' and ' + value);
|
||||
}
|
||||
} else {
|
||||
env[name] = value;
|
||||
envSize += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Concatenate spec files into a new spec file
|
||||
let languages = Object.keys(tracerConfigs);
|
||||
const cppIndex = languages.indexOf('cpp');
|
||||
// Make sure cpp is the last language, if it's present since it must be concatenated last
|
||||
if (cppIndex !== -1) {
|
||||
let lastLang = languages[languages.length - 1];
|
||||
languages[languages.length - 1] = languages[cppIndex];
|
||||
languages[cppIndex] = lastLang;
|
||||
}
|
||||
|
||||
let totalLines: string[] = [];
|
||||
let totalCount = 0;
|
||||
for (let lang of languages) {
|
||||
const lines = fs.readFileSync(tracerConfigs[lang].spec, 'utf8').split(/\r?\n/);
|
||||
const count = parseInt(lines[1], 10);
|
||||
totalCount += count;
|
||||
totalLines.push(...lines.slice(2));
|
||||
}
|
||||
|
||||
const newLogFilePath = path.resolve(config.tempDir, 'compound-build-tracer.log');
|
||||
const spec = path.resolve(config.tempDir, 'compound-spec');
|
||||
const compoundTempFolder = path.resolve(config.tempDir, 'compound-temp');
|
||||
const newSpecContent = [newLogFilePath, totalCount.toString(10), ...totalLines];
|
||||
|
||||
if (copyExecutables) {
|
||||
env['SEMMLE_COPY_EXECUTABLES_ROOT'] = compoundTempFolder;
|
||||
envSize += 1;
|
||||
}
|
||||
|
||||
fs.writeFileSync(spec, newSpecContent.join('\n'));
|
||||
|
||||
// Prepare the content of the compound environment file
|
||||
let buffer = Buffer.alloc(4);
|
||||
buffer.writeInt32LE(envSize, 0);
|
||||
for (let e of Object.entries(env)) {
|
||||
const key = e[0];
|
||||
const value = e[1];
|
||||
const lineBuffer = new Buffer(key + '=' + value + '\0', 'utf8');
|
||||
const sizeBuffer = Buffer.alloc(4);
|
||||
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
|
||||
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
|
||||
}
|
||||
// Write the compound environment
|
||||
const envPath = spec + '.environment';
|
||||
fs.writeFileSync(envPath, buffer);
|
||||
|
||||
return { env, spec };
|
||||
}
|
||||
|
||||
export async function getCombinedTracerConfig(
|
||||
config: configUtils.Config,
|
||||
codeql: CodeQL): Promise<TracerConfig | undefined> {
|
||||
|
||||
// Abort if there are no traced languages as there's nothing to do
|
||||
const tracedLanguages = config.languages.filter(isTracedLanguage);
|
||||
if (tracedLanguages.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Get all the tracer configs and combine them together
|
||||
const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
|
||||
for (const language of tracedLanguages) {
|
||||
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(codeql, config, language);
|
||||
}
|
||||
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
|
||||
|
||||
// Add a couple more variables
|
||||
mainTracerConfig.env['ODASA_TRACER_CONFIGURATION'] = mainTracerConfig.spec;
|
||||
const codeQLDir = path.dirname(codeql.getPath());
|
||||
if (process.platform === 'darwin') {
|
||||
mainTracerConfig.env['DYLD_INSERT_LIBRARIES'] = path.join(codeQLDir, 'tools', 'osx64', 'libtrace.dylib');
|
||||
} else if (process.platform !== 'win32') {
|
||||
mainTracerConfig.env['LD_PRELOAD'] = path.join(codeQLDir, 'tools', 'linux64', '${LIB}trace.so');
|
||||
}
|
||||
|
||||
return mainTracerConfig;
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ import * as os from 'os';
|
|||
import * as path from 'path';
|
||||
|
||||
import * as api from './api-client';
|
||||
import { Language } from './languages';
|
||||
import * as sharedEnv from './shared-environment';
|
||||
|
||||
/**
|
||||
|
|
@ -433,3 +434,10 @@ export function getThreadsFlag(): string {
|
|||
export function getCodeQLDatabasesDir(tempDir: string) {
|
||||
return path.resolve(tempDir, 'codeql_databases');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path where the CodeQL database for the given language lives.
|
||||
*/
|
||||
export function getCodeQLDatabasePath(tempDir: string, language: Language) {
|
||||
return path.resolve(getCodeQLDatabasesDir(tempDir), language);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue