robojerk/codeql-action

Merge branch 'main' into henrymercer/remove-support-codeql-2.12.5

Browse source
This commit is contained in:
Henry Mercer 2024-04-09 18:57:26 +01:00
commit 987236cc0f
71 changed files with 1478 additions and 2361 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/analyze-action.ts
View file

@ -253,7 +253,6 @@ async function run() {
memory,
config,
logger,
features,
);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

61
src/analyze.ts
View file

@ -2,7 +2,6 @@ import * as fs from "fs";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import { safeWhich } from "@chrisgavin/safe-which";
import del from "del";
import * as yaml from "js-yaml";
@ -17,11 +16,7 @@ import * as configUtils from "./config-utils";
import { BuildMode } from "./config-utils";
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import { EnvVar } from "./environment";
import {
FeatureEnablement,
Feature,
isPythonDependencyInstallationDisabled,
} from "./feature-flags";
import { FeatureEnablement, Feature } from "./feature-flags";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { DatabaseCreationTimings, EventReport } from "./status-report";
@ -121,59 +116,24 @@ export interface QueriesStatusReport {
event_reports?: EventReport[];
}
async function setupPythonExtractor(
logger: Logger,
features: FeatureEnablement,
codeql: CodeQL,
) {
async function setupPythonExtractor(logger: Logger) {
const codeqlPython = process.env["CODEQL_PYTHON"];
if (codeqlPython === undefined || codeqlPython.length === 0) {
// If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
return;
}
if (await isPythonDependencyInstallationDisabled(codeql, features)) {
logger.warning(
"We recommend that you remove the CODEQL_PYTHON environment variable from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
"\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.",
);
return;
}
const scriptsFolder = path.resolve(__dirname, "../python-setup");
let output = "";
const options = {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
};
await new toolrunner.ToolRunner(
codeqlPython,
[path.join(scriptsFolder, "find_site_packages.py")],
options,
).exec();
logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`);
process.env["LGTM_INDEX_IMPORT_PATH"] = output;
output = "";
await new toolrunner.ToolRunner(
codeqlPython,
["-c", "import sys; print(sys.version_info[0])"],
options,
).exec();
logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
logger.warning(
"The CODEQL_PYTHON environment variable is no longer supported. Please remove it from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
"\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.",
);
return;
}
export async function runExtraction(
codeql: CodeQL,
config: configUtils.Config,
logger: Logger,
features: FeatureEnablement,
) {
for (const language of config.languages) {
if (dbIsFinalized(config, language, logger)) {
@ -186,7 +146,7 @@ export async function runExtraction(
if (shouldExtractLanguage(config, language)) {
logger.startGroup(`Extracting ${language}`);
if (language === Language.python) {
await setupPythonExtractor(logger, features, codeql);
await setupPythonExtractor(logger);
}
if (
config.buildMode &&
@ -262,12 +222,11 @@ async function finalizeDatabaseCreation(
threadsFlag: string,
memoryFlag: string,
logger: Logger,
features: FeatureEnablement,
): Promise<DatabaseCreationTimings> {
const codeql = await getCodeQL(config.codeQLCmd);
const extractionStart = performance.now();
await runExtraction(codeql, config, logger, features);
await runExtraction(codeql, config, logger);
const extractionTime = performance.now() - extractionStart;
const trapImportStart = performance.now();
@ -443,7 +402,6 @@ export async function runFinalize(
memoryFlag: string,
config: configUtils.Config,
logger: Logger,
features: FeatureEnablement,
): Promise<DatabaseCreationTimings> {
try {
await del(outputDir, { force: true });
@ -459,7 +417,6 @@ export async function runFinalize(
threadsFlag,
memoryFlag,
logger,
features,
);
// WARNING: This does not _really_ end tracing, as the tracer will restore its

37
src/feature-flags.ts
View file

@ -49,8 +49,6 @@ export enum Feature {
CppTrapCachingEnabled = "cpp_trap_caching_enabled",
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
DisablePythonDependencyInstallationEnabled = "disable_python_dependency_installation_enabled",
PythonDefaultIsToSkipDependencyInstallationEnabled = "python_default_is_to_skip_dependency_installation_enabled",
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
}
@ -95,25 +93,6 @@ export const featureConfig: Record<
minimumVersion: undefined,
defaultValue: false,
},
[Feature.DisablePythonDependencyInstallationEnabled]: {
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
// Although the python extractor only started supporting not extracting installed
// dependencies in 2.13.1, the init-action can still benefit from not installing
// dependencies no matter what codeql version we are using, so therefore the
// minimumVersion is set to 'undefined'. This means that with an old CodeQL version,
// packages available with current python3 installation might get extracted.
minimumVersion: undefined,
defaultValue: false,
},
[Feature.PythonDefaultIsToSkipDependencyInstallationEnabled]: {
// we can reuse the same environment variable as above. If someone has set it to
// `true` in their workflow this means dependencies are not installed, setting it to
// `false` means dependencies _will_ be installed. The same semantics are applied
// here!
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
minimumVersion: "2.16.0",
defaultValue: true,
},
};
/**
@ -458,19 +437,3 @@ class GitHubFeatureFlags {
}
}
}
export async function isPythonDependencyInstallationDisabled(
codeql: CodeQL,
features: FeatureEnablement,
): Promise<boolean> {
return (
(await features.getValue(
Feature.DisablePythonDependencyInstallationEnabled,
codeql,
)) ||
(await features.getValue(
Feature.PythonDefaultIsToSkipDependencyInstallationEnabled,
codeql,
))
);
}

71
src/init-action.ts
View file

@ -16,18 +16,8 @@ import { getGitHubVersion } from "./api-client";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import {
Feature,
Features,
isPythonDependencyInstallationDisabled,
} from "./feature-flags";
import {
checkInstallPython311,
initCodeQL,
initConfig,
installPythonDeps,
runInit,
} from "./init";
import { Feature, Features } from "./feature-flags";
import { checkInstallPython311, initCodeQL, initConfig, runInit } from "./init";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@ -298,24 +288,6 @@ async function run() {
);
await checkInstallPython311(config.languages, codeql);
if (
config.languages.includes(Language.python) &&
getRequiredInput("setup-python-dependencies") === "true"
) {
if (await isPythonDependencyInstallationDisabled(codeql, features)) {
logger.info("Skipping python dependency installation");
} else {
try {
await installPythonDeps(codeql, logger);
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
logger.warning(
`${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`,
);
}
}
}
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
core.setFailed(error.message);
@ -467,18 +439,43 @@ async function run() {
}
}
// Disable Python dependency extraction if feature flag set
if (await isPythonDependencyInstallationDisabled(codeql, features)) {
// From 2.16.0 the default for the python extractor is to not perform any
// dependency extraction. For versions before that, you needed to set this flag to
// enable this behavior (supported since 2.13.1).
if (await codeQlVersionAbove(codeql, "2.17.1")) {
// disabled by default, no warning
} else if (await codeQlVersionAbove(codeql, "2.16.0")) {
// disabled by default, prints warning if environment variable is not set
core.exportVariable(
"CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION",
"true",
);
} else if (await codeQlVersionAbove(codeql, "2.13.1")) {
core.exportVariable(
"CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION",
"true",
);
} else {
// From 2.16.0 the default for the python extractor is to not perform any library
// extraction, so we need to set this flag to enable it.
core.exportVariable(
"CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0",
"true",
logger.warning(
`CodeQL Action versions 3.25.0 and later, and versions 2.25.0 and later no longer install Python dependencies. We recommend upgrading to at least CodeQL Bundle 2.16.0 to avoid any potential problems due to this (you are currently using ${
(await codeql.getVersion()).version
}). Alternatively, we recommend downgrading the CodeQL Action to version 3.24.10 (for customers using GitHub.com or GitHub Enterprise Server v3.12 or later) or 2.24.10 (for customers using GitHub Enterprise Server v3.11 or earlier).`,
);
}
if (getOptionalInput("setup-python-dependencies") !== undefined) {
logger.warning(
"The setup-python-dependencies input is deprecated and no longer has any effect. We recommend removing any references from your workflows. See https://github.blog/changelog/2024-01-23-codeql-2-16-python-dependency-installation-disabled-new-queries-and-bug-fixes/ for more information.",
);
}
if (
process.env["CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION"] !==
undefined
) {
logger.warning(
"The CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION environment variable is deprecated and no longer has any effect. We recommend removing any references from your workflows. See https://github.blog/changelog/2024-01-23-codeql-2-16-python-dependency-installation-disabled-new-queries-and-bug-fixes/ for more information.",
);
}

43
src/init.ts
View file

@ -138,46 +138,3 @@ export async function checkInstallPython311(
]).exec();
}
}
export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
logger.startGroup("Setup Python dependencies");
const scriptsFolder = path.resolve(__dirname, "../python-setup");
try {
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("powershell"), [
path.join(scriptsFolder, "install_tools.ps1"),
]).exec();
} else {
await new toolrunner.ToolRunner(
path.join(scriptsFolder, "install_tools.sh"),
).exec();
}
const script = "auto_install_packages.py";
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("py"), [
"-3",
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
} else {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("python3"), [
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
}
} catch (e) {
logger.endGroup();
logger.warning(
`An error occurred while trying to automatically install Python dependencies: ${e}\n` +
"Please make sure any necessary dependencies are installed before calling the codeql-action/analyze " +
"step, and add a 'setup-python-dependencies: false' argument to this step to disable our automatic " +
"dependency installation and avoid this warning.",
);
return;
}
logger.endGroup();
}