robojerk/codeql-action

Make use of multi-language and indirect tracing

Browse source
This commit is contained in:
Edoardo Pirovano 2021-09-15 14:49:20 +01:00
parent e40e887968
commit 1f4460b9fb
No known key found for this signature in database
GPG key ID: 047556B5D93FFE28
22 changed files with 275 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/analyze-action.ts
View file

@ -11,6 +11,7 @@ import {
runQueries,
runFinalize,
} from "./analyze";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { getActionsLogger } from "./logging";
@ -79,6 +80,10 @@ async function run() {
"Config file could not be found at expected location. Has the 'init' action been called?"
);
}
await util.enrichEnvironment(
util.Mode.actions,
await getCodeQL(config.codeQLCmd)
);
const apiDetails = {
auth: actionsUtil.getRequiredInput("token"),

40
src/codeql.ts
View file

@ -12,7 +12,7 @@ import * as api from "./api-client";
import { PackWithVersion } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { Language } from "./languages";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as toolcache from "./toolcache";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
@ -75,6 +75,16 @@ export interface CodeQL {
language: Language,
sourceRoot: string
): Promise<void>;
/**
* Run 'codeql database init --db-cluster'.
*/
databaseInitCluster(
databasePath: string,
languages: Language[],
sourceRoot: string,
processName: string | undefined,
processLevel: number | undefined
): Promise<void>;
/**
* Runs the autobuilder for the given language.
*/
@ -198,6 +208,7 @@ const CODEQL_VERSION_DIAGNOSTICS = "2.5.6";
const CODEQL_VERSION_METRICS = "2.5.5";
const CODEQL_VERSION_GROUP_RULES = "2.5.5";
const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
export const CODEQL_VERSION_NEW_TRACING = "2.6.0"; // Use multi-language (>= 2.5.6) and indirect (>= 2.6.0) tracing.
function getCodeQLBundleName(): string {
let platform: string;
@ -528,6 +539,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
printVersion: resolveFunction(partialCodeql, "printVersion"),
getTracerEnv: resolveFunction(partialCodeql, "getTracerEnv"),
databaseInit: resolveFunction(partialCodeql, "databaseInit"),
databaseInitCluster: resolveFunction(partialCodeql, "databaseInitCluster"),
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(
partialCodeql,
@ -643,6 +655,32 @@ async function getCodeQLForCmd(
...getExtraOptionsFromEnv(["database", "init"]),
]);
},
async databaseInitCluster(
databasePath: string,
languages: Language[],
sourceRoot: string,
processName: string | undefined,
processLevel: number | undefined
) {
const extraArgs = languages.map((language) => `--language=${language}`);
if (languages.filter(isTracedLanguage).length > 0) {
extraArgs.push("--begin-tracing");
if (processName !== undefined) {
extraArgs.push(`--trace-process-name=${processName}`);
} else {
extraArgs.push(`--trace-process-level=${processLevel || 3}`);
}
}
await runTool(cmd, [
"database",
"init",
"--db-cluster",
databasePath,
`--source-root=${sourceRoot}`,
...extraArgs,
...getExtraOptionsFromEnv(["database", "init"]),
]);
},
async runAutobuild(language: Language) {
const cmdName =
process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";

18
src/init-action.ts
View file

@ -12,7 +12,7 @@ import {
StatusReportBase,
validateWorkflow,
} from "./actions-util";
import { CodeQL } from "./codeql";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import {
initCodeQL,
@ -30,6 +30,8 @@ import {
Mode,
checkGitHubVersionInRange,
getGitHubVersion,
codeQlVersionAbove,
enrichEnvironment,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -151,6 +153,7 @@ async function run() {
);
codeql = initCodeQLResult.codeql;
toolsVersion = initCodeQLResult.toolsVersion;
await enrichEnvironment(Mode.actions, codeql);
config = await initConfig(
getOptionalInput("languages"),
@ -210,13 +213,22 @@ async function run() {
getOptionalInput("source-root") || ""
);
const tracerConfig = await runInit(codeql, config, sourceRoot);
const tracerConfig = await runInit(
codeql,
config,
sourceRoot,
"Runner.Worker.exe",
undefined
);
if (tracerConfig !== undefined) {
for (const [key, value] of Object.entries(tracerConfig.env)) {
core.exportVariable(key, value);
}
if (process.platform === "win32") {
if (
process.platform === "win32" &&
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))
) {
await injectWindowsTracer(
"Runner.Worker.exe",
undefined,

31
src/init.ts
View file

@ -6,12 +6,13 @@ import * as safeWhich from "@chrisgavin/safe-which";
import * as analysisPaths from "./analysis-paths";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, setupCodeQL } from "./codeql";
import { CodeQL, CODEQL_VERSION_NEW_TRACING, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
import { codeQlVersionAbove } from "./util";
export async function initCodeQL(
codeqlURL: string | undefined,
@ -75,18 +76,30 @@ export async function initConfig(
export async function runInit(
codeql: CodeQL,
config: configUtils.Config,
sourceRoot: string
sourceRoot: string,
processName: string | undefined,
processLevel: number | undefined
): Promise<TracerConfig | undefined> {
fs.mkdirSync(config.dbLocation, { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (const language of config.languages) {
// Init language database
await codeql.databaseInit(
util.getCodeQLDatabasePath(config, language),
language,
sourceRoot
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Init a database cluster
await codeql.databaseInitCluster(
config.dbLocation,
config.languages,
sourceRoot,
processName,
processLevel
);
} else {
for (const language of config.languages) {
// Init language database
await codeql.databaseInit(
util.getCodeQLDatabasePath(config, language),
language,
sourceRoot
);
}
}
return await getCombinedTracerConfig(config, codeql);

20
src/runner.ts
View file

@ -6,7 +6,7 @@ import { Command } from "commander";
import { runFinalize, runQueries } from "./analyze";
import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
import { CodeQL, getCodeQL } from "./codeql";
import { CodeQL, CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
import { Language, parseLanguage } from "./languages";
@ -23,6 +23,8 @@ import {
getGitHubAuth,
initializeEnvironment,
Mode,
codeQlVersionAbove,
enrichEnvironment,
} from "./util";
// eslint-disable-next-line import/no-commonjs
@ -208,6 +210,7 @@ program
)
).codeql;
}
await enrichEnvironment(Mode.runner, codeql);
const workspacePath = checkoutPath;
const config = await initConfig(
cmd.languages,
@ -226,12 +229,21 @@ program
);
const sourceRoot = checkoutPath;
const tracerConfig = await runInit(codeql, config, sourceRoot);
const tracerConfig = await runInit(
codeql,
config,
sourceRoot,
parseTraceProcessName(),
parseTraceProcessLevel()
);
if (tracerConfig === undefined) {
return;
}
if (process.platform === "win32") {
if (
process.platform === "win32" &&
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))
) {
await injectWindowsTracer(
parseTraceProcessName(),
parseTraceProcessLevel(),
@ -317,6 +329,7 @@ program
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
await enrichEnvironment(Mode.runner, await getCodeQL(config.codeQLCmd));
importTracerEnvironment(config);
let language: Language | undefined = undefined;
if (cmd.language !== undefined) {
@ -419,6 +432,7 @@ program
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
await enrichEnvironment(Mode.runner, await getCodeQL(config.codeQLCmd));
const auth = await getGitHubAuth(
logger,

44
src/tracer-config.ts
View file

@ -1,10 +1,11 @@
import * as fs from "fs";
import * as path from "path";
import { CodeQL } from "./codeql";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { Language, isTracedLanguage } from "./languages";
import * as util from "./util";
import { codeQlVersionAbove } from "./util";
export type TracerConfig = {
spec: string;
@ -19,6 +20,24 @@ const CRITICAL_TRACER_VARS = new Set([
"SEMMLE_JAVA_TOOL_OPTIONS",
]);
export async function getTracerConfigForCluster(
config: configUtils.Config
): Promise<TracerConfig> {
const tracingEnvVariables = JSON.parse(
fs.readFileSync(
path.resolve(
config.dbLocation,
"temp/tracingEnvironment/start-tracing.json"
),
"utf8"
)
);
return {
spec: tracingEnvVariables["ODASA_TRACER_CONFIGURATION"],
env: tracingEnvVariables,
};
}
export async function getTracerConfigForLanguage(
codeql: CodeQL,
config: configUtils.Config,
@ -179,16 +198,21 @@ export async function getCombinedTracerConfig(
return undefined;
}
// Get all the tracer configs and combine them together
const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
for (const language of tracedLanguages) {
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(
codeql,
config,
language
);
let mainTracerConfig: TracerConfig;
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
mainTracerConfig = await getTracerConfigForCluster(config);
} else {
// Get all the tracer configs and combine them together
const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
for (const language of tracedLanguages) {
tracedLanguageConfigs[language] = await getTracerConfigForLanguage(
codeql,
config,
language
);
}
mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
}
const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
// Add a couple more variables
mainTracerConfig.env["ODASA_TRACER_CONFIGURATION"] = mainTracerConfig.spec;

46
src/util.ts
View file

@ -8,7 +8,7 @@ import * as semver from "semver";
import { getApiClient, GitHubApiDetails } from "./api-client";
import * as apiCompatibility from "./api-compatibility.json";
import { CodeQL } from "./codeql";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
@ -437,21 +437,37 @@ enum EnvVar {
FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
}
export function initializeEnvironment(mode: Mode, version: string) {
const exportVar = (name: string, value: string) => {
if (mode === Mode.actions) {
core.exportVariable(name, value);
} else {
process.env[name] = value;
}
};
const exportVar = (mode: Mode, name: string, value: string) => {
if (mode === Mode.actions) {
core.exportVariable(name, value);
} else {
process.env[name] = value;
}
};
exportVar(EnvVar.RUN_MODE, mode);
exportVar(EnvVar.VERSION, version);
exportVar(EnvVar.FEATURE_SARIF_COMBINE, "true");
exportVar(EnvVar.FEATURE_WILL_UPLOAD, "true");
exportVar(EnvVar.FEATURE_MULTI_LANGUAGE, "true");
exportVar(EnvVar.FEATURE_SANDWICH, "true");
/**
* Set some initial environment variables that we can set even without
* knowing what version of CodeQL we're running.
*/
export function initializeEnvironment(mode: Mode, version: string) {
exportVar(mode, EnvVar.RUN_MODE, mode);
exportVar(mode, EnvVar.VERSION, version);
exportVar(mode, EnvVar.FEATURE_SARIF_COMBINE, "true");
exportVar(mode, EnvVar.FEATURE_WILL_UPLOAD, "true");
}
/**
* Enrich the environment variables with further flags that we cannot
* know the value of until we know what version of CodeQL we're running.
*/
export async function enrichEnvironment(mode: Mode, codeql: CodeQL) {
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
exportVar(mode, EnvVar.FEATURE_MULTI_LANGUAGE, "false");
exportVar(mode, EnvVar.FEATURE_SANDWICH, "false");
} else {
exportVar(mode, EnvVar.FEATURE_MULTI_LANGUAGE, "true");
exportVar(mode, EnvVar.FEATURE_SANDWICH, "true");
}
}
export function getMode(): Mode {