robojerk/codeql-action

Delete legacy tracing

Browse source
This commit is contained in:
Henry Mercer 2023-03-28 18:38:33 +01:00
parent 4772c1d99f
commit d8fe76e161
27 changed files with 122 additions and 1528 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/analyze-action.ts
View file

@ -16,7 +16,7 @@ import {
} from "./analyze";
import { getApiDetails, getGitHubVersion } from "./api-client";
import { runAutobuild } from "./autobuild";
import { enrichEnvironment, getCodeQL } from "./codeql";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { Features } from "./feature-flags";
@ -207,8 +207,6 @@ async function run() {
);
}
await enrichEnvironment(await getCodeQL(config.codeQLCmd));
const apiDetails = getApiDetails();
const outputDir = actionsUtil.getRequiredInput("output");
const threads = util.getThreadsFlag(

13
src/analyze.ts
View file

@ -8,12 +8,11 @@ import * as yaml from "js-yaml";
import { DatabaseCreationTimings } from "./actions-util";
import * as analysisPaths from "./analysis-paths";
import { CodeQL, CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { CodeQL, getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { FeatureEnablement } from "./feature-flags";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
import { endTracingForCluster } from "./tracer-config";
import * as util from "./util";
@ -493,19 +492,13 @@ export async function runFinalize(
logger
);
const codeql = await getCodeQL(config.codeQLCmd);
// WARNING: This does not _really_ end tracing, as the tracer will restore its
// critical environment variables and it'll still be active for all processes
// launched from this build step.
// However, it will stop tracing for all steps past the codeql-action/analyze
// step.
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Delete variables as specified by the end-tracing script
await endTracingForCluster(config);
} else {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
}
// Delete variables as specified by the end-tracing script
await endTracingForCluster(config);
return timings;
}

136
src/codeql.ts
View file

@ -1,7 +1,6 @@
import * as fs from "fs";
import * as path from "path";
import * as core from "@actions/core";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as yaml from "js-yaml";
@ -18,7 +17,6 @@ import { ToolsSource } from "./init";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { EnvVar } from "./shared-environment";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import {
getTrapCachingExtractorConfigArgs,
@ -76,19 +74,6 @@ export interface CodeQL {
* Print version information about CodeQL.
*/
printVersion(): Promise<void>;
/**
* Run 'codeql database trace-command' on 'tracer-env.js' and parse
* the result to get environment variables set by CodeQL.
*/
getTracerEnv(databasePath: string): Promise<{ [key: string]: string }>;
/**
* Run 'codeql database init'.
*/
databaseInit(
databasePath: string,
language: Language,
sourceRoot: string
): Promise<void>;
/**
* Run 'codeql database init --db-cluster'.
*/
@ -279,23 +264,6 @@ const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
const CODEQL_VERSION_FILE_BASELINE_INFORMATION = "2.11.3";
/**
* This variable controls using the new style of tracing from the CodeQL
* CLI. In particular, with versions above this we will use both indirect
* tracing, and multi-language tracing together with database clusters.
*
* Note that there were bugs in both of these features that were fixed in
* release 2.7.0 of the CodeQL CLI, therefore this flag is only enabled for
* versions above that.
*/
export const CODEQL_VERSION_NEW_TRACING = "2.7.0";
/**
* Versions 2.7.3+ of the CodeQL CLI support build tracing with glibc 2.34 on Linux. Versions before
* this cannot perform build tracing when running on the Actions `ubuntu-22.04` runner image.
*/
export const CODEQL_VERSION_TRACING_GLIBC_2_34 = "2.7.3";
/**
* Versions 2.9.0+ of the CodeQL CLI run machine learning models from a temporary directory, which
* resolves an issue on Windows where TensorFlow models are not correctly loaded due to the path of
@ -418,8 +386,6 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
() => new Promise((resolve) => resolve("1.0.0"))
),
printVersion: resolveFunction(partialCodeql, "printVersion"),
getTracerEnv: resolveFunction(partialCodeql, "getTracerEnv"),
databaseInit: resolveFunction(partialCodeql, "databaseInit"),
databaseInitCluster: resolveFunction(partialCodeql, "databaseInitCluster"),
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(
@ -506,94 +472,6 @@ export async function getCodeQLForCmd(
async printVersion() {
await runTool(cmd, ["version", "--format=json"]);
},
async getTracerEnv(databasePath: string) {
// Write tracer-env.js to a temp location.
// BEWARE: The name and location of this file is recognized by `codeql database
// trace-command` in order to enable special support for concatenable tracer
// configurations. Consequently the name must not be changed.
// (This warning can be removed once a different way to recognize the
// action/runner has been implemented in `codeql database trace-command`
// _and_ is present in the latest supported CLI release.)
const tracerEnvJs = path.resolve(
databasePath,
"working",
"tracer-env.js"
);
fs.mkdirSync(path.dirname(tracerEnvJs), { recursive: true });
fs.writeFileSync(
tracerEnvJs,
`
const fs = require('fs');
const env = {};
for (let entry of Object.entries(process.env)) {
const key = entry[0];
const value = entry[1];
if (typeof value !== 'undefined' && key !== '_' && !key.startsWith('JAVA_MAIN_CLASS_')) {
env[key] = value;
}
}
process.stdout.write(process.argv[2]);
fs.writeFileSync(process.argv[2], JSON.stringify(env), 'utf-8');`
);
// BEWARE: The name and location of this file is recognized by `codeql database
// trace-command` in order to enable special support for concatenable tracer
// configurations. Consequently the name must not be changed.
// (This warning can be removed once a different way to recognize the
// action/runner has been implemented in `codeql database trace-command`
// _and_ is present in the latest supported CLI release.)
const envFile = path.resolve(databasePath, "working", "env.tmp");
try {
await runTool(cmd, [
"database",
"trace-command",
databasePath,
...getExtraOptionsFromEnv(["database", "trace-command"]),
process.execPath,
tracerEnvJs,
envFile,
]);
} catch (e) {
if (
e instanceof CommandInvocationError &&
e.output.includes(
"undefined symbol: __libc_dlopen_mode, version GLIBC_PRIVATE"
) &&
process.platform === "linux" &&
!(await util.codeQlVersionAbove(
this,
CODEQL_VERSION_TRACING_GLIBC_2_34
))
) {
throw new util.UserError(
"The CodeQL CLI is incompatible with the version of glibc on your system. " +
`Please upgrade to CodeQL CLI version ${CODEQL_VERSION_TRACING_GLIBC_2_34} or ` +
"later. If you cannot upgrade to a newer version of the CodeQL CLI, you can " +
`alternatively run your workflow on another runner image such as "ubuntu-20.04" ` +
"that has glibc 2.33 or earlier installed."
);
} else {
throw e;
}
}
return JSON.parse(fs.readFileSync(envFile, "utf-8"));
},
async databaseInit(
databasePath: string,
language: Language,
sourceRoot: string
) {
await runTool(cmd, [
"database",
"init",
databasePath,
`--language=${language}`,
`--source-root=${sourceRoot}`,
...getExtraOptionsFromEnv(["database", "init"]),
]);
},
async databaseInitCluster(
config: Config,
sourceRoot: string,
@ -1301,17 +1179,3 @@ async function getCodeScanningConfigExportArguments(
}
return [];
}
/**
* Enrich the environment variables with further flags that we cannot
* know the value of until we know what version of CodeQL we're running.
*/
export async function enrichEnvironment(codeql: CodeQL) {
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "false");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "false");
} else {
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "true");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "true");
}
}

37
src/debug-artifacts.ts
View file

@ -8,13 +8,12 @@ import del from "del";
import { getRequiredInput } from "./actions-util";
import { dbIsFinalized } from "./analyze";
import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { getCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import {
bundleDb,
codeQlVersionAbove,
doesDirectoryExist,
getCodeQLDatabasePath,
listFolder,
@ -72,8 +71,6 @@ export async function uploadSarifDebugArtifact(
}
export async function uploadLogsDebugArtifact(config: Config) {
const codeql = await getCodeQL(config.codeQLCmd);
let toUpload: string[] = [];
for (const language of config.languages) {
const databaseDirectory = getCodeQLDatabasePath(config, language);
@ -83,36 +80,20 @@ export async function uploadLogsDebugArtifact(config: Config) {
}
}
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Multilanguage tracing: there are additional logs in the root of the cluster
const multiLanguageTracingLogsDirectory = path.resolve(
config.dbLocation,
"log"
);
if (doesDirectoryExist(multiLanguageTracingLogsDirectory)) {
toUpload = toUpload.concat(listFolder(multiLanguageTracingLogsDirectory));
}
// Multilanguage tracing: there are additional logs in the root of the cluster
const multiLanguageTracingLogsDirectory = path.resolve(
config.dbLocation,
"log"
);
if (doesDirectoryExist(multiLanguageTracingLogsDirectory)) {
toUpload = toUpload.concat(listFolder(multiLanguageTracingLogsDirectory));
}
await uploadDebugArtifacts(
toUpload,
config.dbLocation,
config.debugArtifactName
);
// Before multi-language tracing, we wrote a compound-build-tracer.log in the temp dir
if (!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))) {
const compoundBuildTracerLogDirectory = path.resolve(
config.tempDir,
"compound-build-tracer.log"
);
if (doesDirectoryExist(compoundBuildTracerLogDirectory)) {
await uploadDebugArtifacts(
[compoundBuildTracerLogDirectory],
config.tempDir,
config.debugArtifactName
);
}
}
}
/**

22
src/init-action.ts
View file

@ -14,17 +14,12 @@ import {
StatusReportBase,
} from "./actions-util";
import { getGitHubVersion } from "./api-client";
import {
CodeQL,
CODEQL_VERSION_NEW_TRACING,
enrichEnvironment,
} from "./codeql";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Feature, Features } from "./feature-flags";
import {
initCodeQL,
initConfig,
injectWindowsTracer,
installPythonDeps,
runInit,
ToolsSource,
@ -36,7 +31,6 @@ import { getTotalCacheSize } from "./trap-caching";
import {
checkForTimeout,
checkGitHubVersionInRange,
codeQlVersionAbove,
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
getMemoryFlagValue,
@ -250,7 +244,6 @@ async function run() {
toolsDownloadDurationMs = initCodeQLResult.toolsDownloadDurationMs;
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
await enrichEnvironment(codeql);
config = await initConfig(
getOptionalInput("languages"),
@ -349,19 +342,6 @@ async function run() {
for (const [key, value] of Object.entries(tracerConfig.env)) {
core.exportVariable(key, value);
}
if (
process.platform === "win32" &&
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))
) {
await injectWindowsTracer(
"Runner.Worker.exe",
undefined,
config,
codeql,
tracerConfig
);
}
}
core.setOutput("codeql-path", config.codeQLCmd);

177
src/init.ts
View file

@ -6,14 +6,13 @@ import * as safeWhich from "@chrisgavin/safe-which";
import * as analysisPaths from "./analysis-paths";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING, setupCodeQL } from "./codeql";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
import { codeQlVersionAbove } from "./util";
export enum ToolsSource {
Unknown = "UNKNOWN",
@ -110,53 +109,42 @@ export async function runInit(
fs.mkdirSync(config.dbLocation, { recursive: true });
try {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// When parsing the codeql config in the CLI, we have not yet created the qlconfig file.
// So, create it now.
// If we are parsing the config file in the Action, then the qlconfig file was already created
// before the `pack download` command was invoked. It is not required for the init command.
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (await util.useCodeScanningConfigInCli(codeql, features)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeql,
config.tempDir,
logger
));
}
await configUtils.wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
async () =>
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
features,
qlconfigFile,
logger
)
);
} else {
for (const language of config.languages) {
// Init language database
await codeql.databaseInit(
util.getCodeQLDatabasePath(config, language),
language,
sourceRoot
);
}
// When parsing the codeql config in the CLI, we have not yet created the qlconfig file.
// So, create it now.
// If we are parsing the config file in the Action, then the qlconfig file was already created
// before the `pack download` command was invoked. It is not required for the init command.
let registriesAuthTokens: string | undefined;
let qlconfigFile: string | undefined;
if (await util.useCodeScanningConfigInCli(codeql, features)) {
({ registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
codeql,
config.tempDir,
logger
));
}
await configUtils.wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
async () =>
await codeql.databaseInitCluster(
config,
sourceRoot,
processName,
features,
qlconfigFile,
logger
)
);
} catch (e) {
throw processError(e);
}
return await getCombinedTracerConfig(config, codeql);
return await getCombinedTracerConfig(config);
}
/**
@ -195,105 +183,6 @@ function processError(e: any): Error {
return e;
}
// Runs a powershell script to inject the tracer into a parent process
// so it can tracer future processes, hopefully including the build process.
// If processName is given then injects into the nearest parent process with
// this name, otherwise uses the processLevel-th parent if defined, otherwise
// defaults to the 3rd parent as a rough guess.
export async function injectWindowsTracer(
processName: string | undefined,
processLevel: number | undefined,
config: configUtils.Config,
codeql: CodeQL,
tracerConfig: TracerConfig
) {
let script: string;
if (processName !== undefined) {
script = `
Param(
[Parameter(Position=0)]
[String]
$tracer
)
$id = $PID
while ($true) {
$p = Get-CimInstance -Class Win32_Process -Filter "ProcessId = $id"
Write-Host "Found process: $p"
if ($p -eq $null) {
throw "Could not determine ${processName} process"
}
if ($p[0].Name -eq "${processName}") {
Break
} else {
$id = $p[0].ParentProcessId
}
}
Write-Host "Final process: $p"
Invoke-Expression "&$tracer --inject=$id"`;
} else {
// If the level is not defined then guess at the 3rd parent process.
// This won't be correct in every setting but it should be enough in most settings,
// and overestimating is likely better in this situation so we definitely trace
// what we want, though this does run the risk of interfering with future CI jobs.
// Note that the default of 3 doesn't work on github actions, so we include a
// special case in the script that checks for Runner.Worker.exe so we can still work
// on actions if the runner is invoked there.
processLevel = processLevel || 3;
script = `
Param(
[Parameter(Position=0)]
[String]
$tracer
)
$id = $PID
for ($i = 0; $i -le ${processLevel}; $i++) {
$p = Get-CimInstance -Class Win32_Process -Filter "ProcessId = $id"
Write-Host "Parent process \${i}: $p"
if ($p -eq $null) {
throw "Process tree ended before reaching required level"
}
# Special case just in case the runner is used on actions
if ($p[0].Name -eq "Runner.Worker.exe") {
Write-Host "Found Runner.Worker.exe process which means we are running on GitHub Actions"
Write-Host "Aborting search early and using process: $p"
Break
} elseif ($p[0].Name -eq "Agent.Worker.exe") {
Write-Host "Found Agent.Worker.exe process which means we are running on Azure Pipelines"
Write-Host "Aborting search early and using process: $p"
Break
} else {
$id = $p[0].ParentProcessId
}
}
Write-Host "Final process: $p"
Invoke-Expression "&$tracer --inject=$id"`;
}
const injectTracerPath = path.join(config.tempDir, "inject-tracer.ps1");
fs.writeFileSync(injectTracerPath, script);
await new toolrunner.ToolRunner(
await safeWhich.safeWhich("powershell"),
[
"-ExecutionPolicy",
"Bypass",
"-file",
injectTracerPath,
path.resolve(
path.dirname(codeql.getPath()),
"tools",
"win64",
"tracer.exe"
),
],
{ env: { ODASA_TRACER_CONFIGURATION: tracerConfig.spec } }
).exec();
}
export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
logger.startGroup("Setup Python dependencies");

351
src/tracer-config.test.ts
View file

@ -3,15 +3,10 @@ import * as path from "path";
import test from "ava";
import { setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import { setupTests } from "./testing-utils";
import {
concatTracerConfigs,
getCombinedTracerConfig,
getTracerConfigForLanguage,
} from "./tracer-config";
import { getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
setupTests(test);
@ -37,309 +32,19 @@ function getTestConfig(tmpDir: string): configUtils.Config {
};
}
// A very minimal setup
test("getTracerConfigForLanguage - minimal setup", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const codeQL = setCodeQL({
async getTracerEnv() {
return {
ODASA_TRACER_CONFIGURATION: "abc",
foo: "bar",
};
},
});
const result = await getTracerConfigForLanguage(
codeQL,
config,
Language.javascript
);
t.deepEqual(result, { spec: "abc", env: { foo: "bar" } });
});
});
// Existing vars should not be overwritten, unless they are critical or prefixed with CODEQL_
test("getTracerConfigForLanguage - existing / critical vars", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
// Set up some variables in the environment
process.env["foo"] = "abc";
process.env["SEMMLE_PRELOAD_libtrace"] = "abc";
process.env["SEMMLE_RUNNER"] = "abc";
process.env["SEMMLE_COPY_EXECUTABLES_ROOT"] = "abc";
process.env["SEMMLE_DEPTRACE_SOCKET"] = "abc";
process.env["SEMMLE_JAVA_TOOL_OPTIONS"] = "abc";
process.env["CODEQL_VAR"] = "abc";
// Now CodeQL returns all these variables, and one more, with different values
const codeQL = setCodeQL({
async getTracerEnv() {
return {
ODASA_TRACER_CONFIGURATION: "abc",
foo: "bar",
baz: "qux",
SEMMLE_PRELOAD_libtrace: "SEMMLE_PRELOAD_libtrace",
SEMMLE_RUNNER: "SEMMLE_RUNNER",
SEMMLE_COPY_EXECUTABLES_ROOT: "SEMMLE_COPY_EXECUTABLES_ROOT",
SEMMLE_DEPTRACE_SOCKET: "SEMMLE_DEPTRACE_SOCKET",
SEMMLE_JAVA_TOOL_OPTIONS: "SEMMLE_JAVA_TOOL_OPTIONS",
CODEQL_VAR: "CODEQL_VAR",
};
},
});
const result = await getTracerConfigForLanguage(
codeQL,
config,
Language.javascript
);
t.deepEqual(result, {
spec: "abc",
env: {
// Should contain all variables except 'foo', because that already existed in the
// environment with a different value, and is not deemed a "critical" variable.
baz: "qux",
SEMMLE_PRELOAD_libtrace: "SEMMLE_PRELOAD_libtrace",
SEMMLE_RUNNER: "SEMMLE_RUNNER",
SEMMLE_COPY_EXECUTABLES_ROOT: "SEMMLE_COPY_EXECUTABLES_ROOT",
SEMMLE_DEPTRACE_SOCKET: "SEMMLE_DEPTRACE_SOCKET",
SEMMLE_JAVA_TOOL_OPTIONS: "SEMMLE_JAVA_TOOL_OPTIONS",
CODEQL_VAR: "CODEQL_VAR",
},
});
});
});
test("concatTracerConfigs - minimal configs correctly combined", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, "spec1");
fs.writeFileSync(spec1, "foo.log\n2\nabc\ndef");
const tc1 = {
spec: spec1,
env: {
a: "a",
b: "b",
},
};
const spec2 = path.join(tmpDir, "spec2");
fs.writeFileSync(spec2, "foo.log\n1\nghi");
const tc2 = {
spec: spec2,
env: {
c: "c",
},
};
const result = concatTracerConfigs(
{ javascript: tc1, python: tc2 },
config
);
t.deepEqual(result, {
spec: path.join(tmpDir, "compound-spec"),
env: {
a: "a",
b: "b",
c: "c",
},
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, "utf8"),
`${path.join(tmpDir, "compound-build-tracer.log")}\n3\nabc\ndef\nghi`
);
});
});
test("concatTracerConfigs - conflicting env vars", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, "spec");
fs.writeFileSync(spec, "foo.log\n0");
// Ok if env vars have the same name and the same value
t.deepEqual(
concatTracerConfigs(
{
javascript: { spec, env: { a: "a", b: "b" } },
python: { spec, env: { b: "b", c: "c" } },
},
config
).env,
{
a: "a",
b: "b",
c: "c",
}
);
// Throws if env vars have same name but different values
const e = t.throws(() =>
concatTracerConfigs(
{
javascript: { spec, env: { a: "a", b: "b" } },
python: { spec, env: { b: "c" } },
},
config
)
);
// If e is undefined, then the previous assertion will fail.
if (e !== undefined) {
t.deepEqual(
e.message,
"Incompatible values in environment parameter b: b and c"
);
}
});
});
test("concatTracerConfigs - cpp spec lines come last if present", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, "spec1");
fs.writeFileSync(spec1, "foo.log\n2\nabc\ndef");
const tc1 = {
spec: spec1,
env: {
a: "a",
b: "b",
},
};
const spec2 = path.join(tmpDir, "spec2");
fs.writeFileSync(spec2, "foo.log\n1\nghi");
const tc2 = {
spec: spec2,
env: {
c: "c",
},
};
const result = concatTracerConfigs({ cpp: tc1, python: tc2 }, config);
t.deepEqual(result, {
spec: path.join(tmpDir, "compound-spec"),
env: {
a: "a",
b: "b",
c: "c",
},
});
t.true(fs.existsSync(result.spec));
t.deepEqual(
fs.readFileSync(result.spec, "utf8"),
`${path.join(tmpDir, "compound-build-tracer.log")}\n3\nghi\nabc\ndef`
);
});
});
test("concatTracerConfigs - SEMMLE_COPY_EXECUTABLES_ROOT is updated to point to compound spec", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, "spec");
fs.writeFileSync(spec, "foo.log\n0");
const result = concatTracerConfigs(
{
javascript: { spec, env: { a: "a", b: "b" } },
python: { spec, env: { SEMMLE_COPY_EXECUTABLES_ROOT: "foo" } },
},
config
);
t.deepEqual(result.env, {
a: "a",
b: "b",
SEMMLE_COPY_EXECUTABLES_ROOT: path.join(tmpDir, "compound-temp"),
});
});
});
test("concatTracerConfigs - compound environment file is created correctly", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec1 = path.join(tmpDir, "spec1");
fs.writeFileSync(spec1, "foo.log\n2\nabc\ndef");
const tc1 = {
spec: spec1,
env: {
a: "a",
},
};
const spec2 = path.join(tmpDir, "spec2");
fs.writeFileSync(spec2, "foo.log\n1\nghi");
const tc2 = {
spec: spec2,
env: {
foo: "bar_baz",
},
};
const result = concatTracerConfigs(
{ javascript: tc1, python: tc2 },
config,
true
);
// Check binary contents for the Unix file
const envPath = `${result.spec}.environment`;
t.true(fs.existsSync(envPath));
const buffer: Buffer = fs.readFileSync(envPath);
t.deepEqual(buffer.length, 28);
t.deepEqual(buffer.readInt32LE(0), 2); // number of env vars
t.deepEqual(buffer.readInt32LE(4), 4); // length of env var definition
t.deepEqual(buffer.toString("utf8", 8, 12), "a=a\0"); // [key]=[value]\0
t.deepEqual(buffer.readInt32LE(12), 12); // length of env var definition
t.deepEqual(buffer.toString("utf8", 16, 28), "foo=bar_baz\0"); // [key]=[value]\0
// Check binary contents for the Windows file
const envPathWindows = `${result.spec}.win32env`;
t.true(fs.existsSync(envPathWindows));
const bufferWindows: Buffer = fs.readFileSync(envPathWindows);
t.deepEqual(bufferWindows.length, 38);
t.deepEqual(bufferWindows.readInt32LE(0), 4 + 12 + 1); // number of tchars to represent the environment
t.deepEqual(bufferWindows.toString("utf16le", 4, 12), "a=a\0"); // [key]=[value]\0
t.deepEqual(bufferWindows.toString("utf16le", 12, 36), "foo=bar_baz\0"); // [key]=[value]\0
t.deepEqual(bufferWindows.toString("utf16le", 36, 38), "\0"); // trailing null character
});
});
test("getCombinedTracerConfig - return undefined when no languages are traced languages", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
// No traced languages
config.languages = [Language.javascript, Language.python];
const codeQL = setCodeQL({
async getTracerEnv() {
return {
ODASA_TRACER_CONFIGURATION: "abc",
CODEQL_DIST: "/",
foo: "bar",
};
},
});
t.deepEqual(await getCombinedTracerConfig(config, codeQL), undefined);
t.deepEqual(await getCombinedTracerConfig(config), undefined);
});
});
test("getCombinedTracerConfig - valid spec file", async (t) => {
test("getCombinedTracerConfig - with start-tracing.json environment file", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
const spec = path.join(tmpDir, "spec");
fs.writeFileSync(spec, "foo.log\n2\nabc\ndef");
const bundlePath = path.join(tmpDir, "bundle");
const codeqlPlatform =
process.platform === "win32"
@ -347,43 +52,28 @@ test("getCombinedTracerConfig - valid spec file", async (t) => {
: process.platform === "darwin"
? "osx64"
: "linux64";
const codeQL = setCodeQL({
async getTracerEnv() {
return {
ODASA_TRACER_CONFIGURATION: spec,
CODEQL_DIST: bundlePath,
CODEQL_PLATFORM: codeqlPlatform,
foo: "bar",
};
},
});
const result = await getCombinedTracerConfig(config, codeQL);
t.notDeepEqual(result, undefined);
const expectedEnv = {
const startTracingEnv = {
foo: "bar",
CODEQL_DIST: bundlePath,
CODEQL_PLATFORM: codeqlPlatform,
ODASA_TRACER_CONFIGURATION: result!.spec,
};
if (process.platform === "darwin") {
expectedEnv["DYLD_INSERT_LIBRARIES"] = path.join(
path.dirname(codeQL.getPath()),
"tools",
"osx64",
"libtrace.dylib"
);
} else if (process.platform !== "win32") {
expectedEnv["LD_PRELOAD"] = path.join(
path.dirname(codeQL.getPath()),
"tools",
"linux64",
"${LIB}trace.so"
);
}
const tracingEnvironmentDir = path.join(
config.dbLocation,
"temp",
"tracingEnvironment"
);
fs.mkdirSync(tracingEnvironmentDir, { recursive: true });
const startTracingJson = path.join(
tracingEnvironmentDir,
"start-tracing.json"
);
fs.writeFileSync(startTracingJson, JSON.stringify(startTracingEnv));
const result = await getCombinedTracerConfig(config);
t.notDeepEqual(result, undefined);
const expectedEnv = startTracingEnv;
if (process.platform === "win32") {
expectedEnv["CODEQL_RUNNER"] = path.join(
@ -403,7 +93,6 @@ test("getCombinedTracerConfig - valid spec file", async (t) => {
}
t.deepEqual(result, {
spec: path.join(tmpDir, "compound-spec"),
env: expectedEnv,
});
});

203
src/tracer-config.ts
View file

@ -1,25 +1,13 @@
import * as fs from "fs";
import * as path from "path";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { Language, isTracedLanguage } from "./languages";
import * as util from "./util";
import { codeQlVersionAbove } from "./util";
import { isTracedLanguage } from "./languages";
export type TracerConfig = {
spec: string;
env: { [key: string]: string };
};
const CRITICAL_TRACER_VARS = new Set([
"SEMMLE_PRELOAD_libtrace",
"SEMMLE_RUNNER",
"SEMMLE_COPY_EXECUTABLES_ROOT",
"SEMMLE_DEPTRACE_SOCKET",
"SEMMLE_JAVA_TOOL_OPTIONS",
]);
export async function endTracingForCluster(
config: configUtils.Config
): Promise<void> {
@ -66,164 +54,12 @@ export async function getTracerConfigForCluster(
)
);
return {
spec: tracingEnvVariables["ODASA_TRACER_CONFIGURATION"],
env: tracingEnvVariables,
};
}
export async function getTracerConfigForLanguage(
codeql: CodeQL,
config: configUtils.Config,
language: Language
): Promise<TracerConfig> {
const env = await codeql.getTracerEnv(
util.getCodeQLDatabasePath(config, language)
);
const spec = env["ODASA_TRACER_CONFIGURATION"];
const info: TracerConfig = { spec, env: {} };
// Extract critical tracer variables from the environment
for (const entry of Object.entries(env)) {
const key = entry[0];
const value = entry[1];
// skip ODASA_TRACER_CONFIGURATION as it is handled separately
if (key === "ODASA_TRACER_CONFIGURATION") {
continue;
}
// skip undefined values
if (typeof value === "undefined") {
continue;
}
// Keep variables that do not exist in current environment. In addition always keep
// critical and CODEQL_ variables
if (
typeof process.env[key] === "undefined" ||
CRITICAL_TRACER_VARS.has(key) ||
key.startsWith("CODEQL_")
) {
info.env[key] = value;
}
}
return info;
}
export function concatTracerConfigs(
tracerConfigs: { [lang: string]: TracerConfig },
config: configUtils.Config,
writeBothEnvironments = false
): TracerConfig {
// A tracer config is a map containing additional environment variables and a tracer 'spec' file.
// A tracer 'spec' file has the following format [log_file, number_of_blocks, blocks_text]
// Merge the environments
const env: { [key: string]: string } = {};
let copyExecutables = false;
let envSize = 0;
for (const v of Object.values(tracerConfigs)) {
for (const e of Object.entries(v.env)) {
const name = e[0];
const value = e[1];
// skip SEMMLE_COPY_EXECUTABLES_ROOT as it is handled separately
if (name === "SEMMLE_COPY_EXECUTABLES_ROOT") {
copyExecutables = true;
} else if (name in env) {
if (env[name] !== value) {
throw Error(
`Incompatible values in environment parameter ${name}: ${env[name]} and ${value}`
);
}
} else {
env[name] = value;
envSize += 1;
}
}
}
// Concatenate spec files into a new spec file
const languages = Object.keys(tracerConfigs);
const cppIndex = languages.indexOf("cpp");
// Make sure cpp is the last language, if it's present since it must be concatenated last
if (cppIndex !== -1) {
const lastLang = languages[languages.length - 1];
languages[languages.length - 1] = languages[cppIndex];
languages[cppIndex] = lastLang;
}
const totalLines: string[] = [];
let totalCount = 0;
for (const lang of languages) {
const lines = fs
.readFileSync(tracerConfigs[lang].spec, "utf8")
.split(/\r?\n/);
const count = parseInt(lines[1], 10);
totalCount += count;
totalLines.push(...lines.slice(2));
}
const newLogFilePath = path.resolve(
config.tempDir,
"compound-build-tracer.log"
);
const spec = path.resolve(config.tempDir, "compound-spec");
const compoundTempFolder = path.resolve(config.tempDir, "compound-temp");
const newSpecContent = [
newLogFilePath,
totalCount.toString(10),
...totalLines,
];
if (copyExecutables) {
env["SEMMLE_COPY_EXECUTABLES_ROOT"] = compoundTempFolder;
envSize += 1;
}
fs.writeFileSync(spec, newSpecContent.join("\n"));
if (writeBothEnvironments || process.platform !== "win32") {
// Prepare the content of the compound environment file on Unix
let buffer = Buffer.alloc(4);
buffer.writeInt32LE(envSize, 0);
for (const e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const lineBuffer = Buffer.from(`${key}=${value}\0`, "utf8");
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(lineBuffer.length, 0);
buffer = Buffer.concat([buffer, sizeBuffer, lineBuffer]);
}
// Write the compound environment for Unix
const envPath = `${spec}.environment`;
fs.writeFileSync(envPath, buffer);
}
if (writeBothEnvironments || process.platform === "win32") {
// Prepare the content of the compound environment file on Windows
let bufferWindows = Buffer.alloc(0);
let length = 0;
for (const e of Object.entries(env)) {
const key = e[0];
const value = e[1];
const string = `${key}=${value}\0`;
length += string.length;
const lineBuffer = Buffer.from(string, "utf16le");
bufferWindows = Buffer.concat([bufferWindows, lineBuffer]);
}
const sizeBuffer = Buffer.alloc(4);
sizeBuffer.writeInt32LE(length + 1, 0); // Add one for trailing null character marking end
const trailingNull = Buffer.from(`\0`, "utf16le");
bufferWindows = Buffer.concat([sizeBuffer, bufferWindows, trailingNull]);
// Write the compound environment for Windows
const envPathWindows = `${spec}.win32env`;
fs.writeFileSync(envPathWindows, bufferWindows);
}
return { env, spec };
}
export async function getCombinedTracerConfig(
config: configUtils.Config,
codeql: CodeQL
config: configUtils.Config
): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter((l) => isTracedLanguage(l));
@ -231,40 +67,7 @@ export async function getCombinedTracerConfig(
return undefined;
}
let mainTracerConfig: TracerConfig;
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
mainTracerConfig = await getTracerConfigForCluster(config);
} else {
// Get all the tracer configs and combine them together
const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
for (const language of tracedLanguages) {
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(
codeql,
config,
language
);
}
mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
// Add a couple more variables
mainTracerConfig.env["ODASA_TRACER_CONFIGURATION"] = mainTracerConfig.spec;
const codeQLDir = path.dirname(codeql.getPath());
if (process.platform === "darwin") {
mainTracerConfig.env["DYLD_INSERT_LIBRARIES"] = path.join(
codeQLDir,
"tools",
"osx64",
"libtrace.dylib"
);
} else if (process.platform !== "win32") {
mainTracerConfig.env["LD_PRELOAD"] = path.join(
codeQLDir,
"tools",
"linux64",
"${LIB}trace.so"
);
}
}
const mainTracerConfig = await getTracerConfigForCluster(config);
// On macos it's necessary to prefix the build command with the runner executable
// on order to trace when System Integrity Protection is enabled.

4
src/util.ts
View file

@ -438,9 +438,11 @@ export function assertNever(value: never): never {
* knowing what version of CodeQL we're running.
*/
export function initializeEnvironment(version: string) {
core.exportVariable(EnvVar.VERSION, version);
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "false");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "false");
core.exportVariable(EnvVar.FEATURE_SARIF_COMBINE, "true");
core.exportVariable(EnvVar.FEATURE_WILL_UPLOAD, "true");
core.exportVariable(EnvVar.VERSION, version);
}
/**