robojerk/codeql-action

Merge pull request #2224 from github/RasmusWL/remove-python-dep-inst

Browse source 
Delete python dependency installation code
This commit is contained in:
Rasmus Wriedt Larsen 2024-04-09 14:07:45 +02:00 • committed by GitHub
commit 21eac7ce4e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 95 additions and 1468 deletions
Download patch file Download diff file Expand all files Collapse all files

174
.github/workflows/python-deps.yml vendored
View file

@ -1,174 +0,0 @@
name: Test Python Package Installation
on:
push:
branches: [main, releases/v*]
pull_request:
# Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
# by other workflows.
types: [opened, synchronize, reopened, ready_for_review]
paths:
# Changes to this workflow.
- '.github/workflows/python-deps.yml'
# Changes to the Python package installation scripts and their tests.
- 'python-setup/**'
# Changes to the default CodeQL bundle version.
- '**/defaults.json'
schedule:
# Weekly on Monday.
- cron: '0 0 * * 1'
workflow_dispatch:
jobs:
test-setup-python-scripts:
timeout-minutes: 45
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, ubuntu-22.04, macos-latest]
python_deps_type: [pipenv, poetry, requirements, setup_py]
python_version: [3]
env:
PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }}
PYTHON_VERSION: ${{ matrix.python_version }}
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v4
- name: Initialize CodeQL
uses: ./init
id: init
with:
tools: latest
languages: python
setup-python-dependencies: false
- name: Test Auto Package Installation
run: |
set -x
$GITHUB_WORKSPACE/python-setup/install_tools.sh
cd $GITHUB_WORKSPACE/python-setup/tests/${PYTHON_DEPS_TYPE}/requests-${PYTHON_VERSION}
case ${{ matrix.os }} in
ubuntu-20.04*) basePath="/opt";;
ubuntu-22.04*) basePath="/opt";;
macos-latest*) basePath="/Users/runner";;
esac
echo ${basePath}
$GITHUB_WORKSPACE/python-setup/auto_install_packages.py "$(dirname ${{steps.init.outputs.codeql-path}})"
- name: Setup for extractor
run: |
echo $CODEQL_PYTHON
# only run if $CODEQL_PYTHON is set
if [ ! -z $CODEQL_PYTHON ]; then
$GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON;
fi
- name: Verify packages installed
run: |
$GITHUB_WORKSPACE/python-setup/tests/check_requests.sh ${PYTHON_VERSION} 2.31.0
# This one shouldn't fail, but also won't install packages
test-setup-python-scripts-non-standard-location:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, ubuntu-22.04, macos-latest]
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v4
- name: Initialize CodeQL
uses: ./init
id: init
with:
tools: latest
languages: python
setup-python-dependencies: false
- name: Test Auto Package Installation
run: |
set -x
$GITHUB_WORKSPACE/python-setup/install_tools.sh
cd $GITHUB_WORKSPACE/python-setup/tests/requirements/non-standard-location
case ${{ matrix.os }} in
ubuntu-20.04*) basePath="/opt";;
ubuntu-22.04*) basePath="/opt";;
macos-latest*) basePath="/Users/runner";;
esac
echo ${basePath}
$GITHUB_WORKSPACE/python-setup/auto_install_packages.py "$(dirname ${{steps.init.outputs.codeql-path}})"
- name: Setup for extractor
run: |
echo $CODEQL_PYTHON
# only run if $CODEQL_PYTHON is set
if [ ! -z $CODEQL_PYTHON ]; then
$GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON;
fi
- name: Verify packages installed
run: |
test -z $LGTM_INDEX_IMPORT_PATH
test-setup-python-scripts-windows:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
python_deps_type: [pipenv, poetry, requirements, setup_py]
python_version: [3]
env:
CODEQL_ACTION_TEST_MODE: true
PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }}
PYTHON_VERSION: ${{ matrix.python_version }}
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
- name: Initialize CodeQL
id: init
uses: ./init
with:
tools: latest
languages: python
setup-python-dependencies: false
- name: Test Auto Package Installation
env:
CODEQL_PATH: ${{ steps.init.outputs.codeql-path }}
run: |
$cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\install_tools.ps1"
powershell -File $cmd
cd $Env:GITHUB_WORKSPACE\\python-setup/tests/$Env:PYTHON_DEPS_TYPE/requests-$Env:PYTHON_VERSION
$codeql_dist = (get-item $Env:CODEQL_PATH).Directory.FullName
py -3 $Env:GITHUB_WORKSPACE\\python-setup\\auto_install_packages.py $codeql_dist
- name: Setup for extractor
run: |
echo $Env:CODEQL_PYTHON
py -3 $Env:GITHUB_WORKSPACE\\python-setup\\tests\\from_python_exe.py $Env:CODEQL_PYTHON
- name: Verify packages installed
run: |
$cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\tests\\check_requests.ps1"
powershell -File $cmd $Env:PYTHON_VERSION 2.31.0

8
CHANGELOG.md
View file

@ -6,6 +6,14 @@ Note that the only difference between `v2` and `v3` of the CodeQL Action is the
## [UNRELEASED]
- The deprecated feature for extracting dependencies for a Python analysis has been removed. [#2224](https://github.com/github/codeql-action/pull/2224)
As a result, the following inputs and environment variables are now ignored:
- The `setup-python-dependencies` input to the `init` Action
- The `CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION` environment variable
We recommend removing any references to these from your workflows. For more information, see the release notes for CodeQL Action v3.23.0 and v2.23.0.
- Automatically overwrite an existing database if found on the filesystem. [#2229](https://github.com/github/codeql-action/pull/2229)
## 3.24.10 - 05 Apr 2024

2
CODEOWNERS
View file

@ -1,3 +1 @@
**/* @github/codeql-action-reviewers
/python-setup/ @github/codeql-python @github/codeql-action-reviewers

7
init/action.yml
View file

@ -21,7 +21,7 @@ inputs:
The build mode that will be used to analyze the language. This input is only available when
analyzing a single CodeQL language per job, for example using a matrix.
Available build modes will differ based on the language being analyzed. One of:
Available build modes will differ based on the language being analyzed. One of:
- `none`: The database will be created without building the source code.
Available for all interpreted languages and some compiled languages.
@ -89,9 +89,8 @@ inputs:
description: A token for fetching external config files and queries if they reside in a private repository in the same GitHub instance that is running this action.
required: false
setup-python-dependencies:
description: Try to auto-install your python dependencies
required: true
default: 'true'
description: DEPRECATED. This option is ignored since CodeQL Action no longer installs Python dependencies as of versions 3.25.0 and 2.25.0.
required: false
source-root:
description: Path of the root source code directory, relative to $GITHUB_WORKSPACE.
required: false

2
lib/analyze-action.js generated
View file

@ -168,7 +168,7 @@ async function run() {
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"], logger);
await (0, analyze_1.warnIfGoInstalledAfterInit)(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, features);
dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger, features);
}

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

39
lib/analyze.js generated
View file

@ -30,7 +30,6 @@ exports.runCleanup = exports.warnIfGoInstalledAfterInit = exports.runFinalize =
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const safe_which_1 = require("@chrisgavin/safe-which");
const del_1 = __importDefault(require("del"));
const yaml = __importStar(require("js-yaml"));
@ -53,35 +52,17 @@ class CodeQLAnalysisError extends Error {
}
}
exports.CodeQLAnalysisError = CodeQLAnalysisError;
async function setupPythonExtractor(logger, features, codeql) {
async function setupPythonExtractor(logger) {
const codeqlPython = process.env["CODEQL_PYTHON"];
if (codeqlPython === undefined || codeqlPython.length === 0) {
// If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
return;
}
if (await (0, feature_flags_1.isPythonDependencyInstallationDisabled)(codeql, features)) {
logger.warning("We recommend that you remove the CODEQL_PYTHON environment variable from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
"\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.");
return;
}
const scriptsFolder = path.resolve(__dirname, "../python-setup");
let output = "";
const options = {
listeners: {
stdout: (data) => {
output += data.toString();
},
},
};
await new toolrunner.ToolRunner(codeqlPython, [path.join(scriptsFolder, "find_site_packages.py")], options).exec();
logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`);
process.env["LGTM_INDEX_IMPORT_PATH"] = output;
output = "";
await new toolrunner.ToolRunner(codeqlPython, ["-c", "import sys; print(sys.version_info[0])"], options).exec();
logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
logger.warning("The CODEQL_PYTHON environment variable is no longer supported. Please remove it from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
"\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.");
return;
}
async function runExtraction(codeql, config, logger, features) {
async function runExtraction(codeql, config, logger) {
for (const language of config.languages) {
if (dbIsFinalized(config, language, logger)) {
logger.debug(`Database for ${language} has already been finalized, skipping extraction.`);
@ -90,7 +71,7 @@ async function runExtraction(codeql, config, logger, features) {
if (shouldExtractLanguage(config, language)) {
logger.startGroup(`Extracting ${language}`);
if (language === languages_1.Language.python) {
await setupPythonExtractor(logger, features, codeql);
await setupPythonExtractor(logger);
}
if (config.buildMode &&
(await codeql.supportsFeature(tools_features_1.ToolsFeature.TraceCommandUseBuildMode))) {
@ -143,10 +124,10 @@ function dbIsFinalized(config, language, logger) {
}
}
exports.dbIsFinalized = dbIsFinalized;
async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, features) {
async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger) {
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
const extractionStart = perf_hooks_1.performance.now();
await runExtraction(codeql, config, logger, features);
await runExtraction(codeql, config, logger);
const extractionTime = perf_hooks_1.performance.now() - extractionStart;
const trapImportStart = perf_hooks_1.performance.now();
for (const language of config.languages) {
@ -251,7 +232,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
}
}
exports.runQueries = runQueries;
async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, features) {
async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
try {
await (0, del_1.default)(outputDir, { force: true });
}
@ -261,7 +242,7 @@ async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, f
}
}
await fs.promises.mkdir(outputDir, { recursive: true });
const timings = await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, features);
const timings = await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
// WARNING: This does not _really_ end tracing, as the tracer will restore its
// critical environment variables and it'll still be active for all processes
// launched from this build step.

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

28
lib/feature-flags.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.isPythonDependencyInstallationDisabled = exports.Features = exports.FEATURE_FLAGS_FILE_NAME = exports.featureConfig = exports.Feature = exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = exports.CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED = void 0;
exports.Features = exports.FEATURE_FLAGS_FILE_NAME = exports.featureConfig = exports.Feature = exports.CODEQL_VERSION_FINE_GRAINED_PARALLELISM = exports.CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const semver = __importStar(require("semver"));
@ -53,8 +53,6 @@ var Feature;
Feature["CppTrapCachingEnabled"] = "cpp_trap_caching_enabled";
Feature["DisableJavaBuildlessEnabled"] = "disable_java_buildless_enabled";
Feature["DisableKotlinAnalysisEnabled"] = "disable_kotlin_analysis_enabled";
Feature["DisablePythonDependencyInstallationEnabled"] = "disable_python_dependency_installation_enabled";
Feature["PythonDefaultIsToSkipDependencyInstallationEnabled"] = "python_default_is_to_skip_dependency_installation_enabled";
Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
})(Feature || (exports.Feature = Feature = {}));
@ -95,25 +93,6 @@ exports.featureConfig = {
minimumVersion: undefined,
defaultValue: false,
},
[Feature.DisablePythonDependencyInstallationEnabled]: {
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
// Although the python extractor only started supporting not extracting installed
// dependencies in 2.13.1, the init-action can still benefit from not installing
// dependencies no matter what codeql version we are using, so therefore the
// minimumVersion is set to 'undefined'. This means that with an old CodeQL version,
// packages available with current python3 installation might get extracted.
minimumVersion: undefined,
defaultValue: false,
},
[Feature.PythonDefaultIsToSkipDependencyInstallationEnabled]: {
// we can reuse the same environment variable as above. If someone has set it to
// `true` in their workflow this means dependencies are not installed, setting it to
// `false` means dependencies _will_ be installed. The same semantics are applied
// here!
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
minimumVersion: "2.16.0",
defaultValue: true,
},
};
exports.FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
/**
@ -343,9 +322,4 @@ class GitHubFeatureFlags {
}
}
}
async function isPythonDependencyInstallationDisabled(codeql, features) {
return ((await features.getValue(Feature.DisablePythonDependencyInstallationEnabled, codeql)) ||
(await features.getValue(Feature.PythonDefaultIsToSkipDependencyInstallationEnabled, codeql)));
}
exports.isPythonDependencyInstallationDisabled = isPythonDependencyInstallationDisabled;
//# sourceMappingURL=feature-flags.js.map

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

39
lib/init-action.js generated
View file

@ -169,21 +169,6 @@ async function run() {
logger,
}, codeql);
await (0, init_1.checkInstallPython311)(config.languages, codeql);
if (config.languages.includes(languages_1.Language.python) &&
(0, actions_util_1.getRequiredInput)("setup-python-dependencies") === "true") {
if (await (0, feature_flags_1.isPythonDependencyInstallationDisabled)(codeql, features)) {
logger.info("Skipping python dependency installation");
}
else {
try {
await (0, init_1.installPythonDeps)(codeql, logger);
}
catch (unwrappedError) {
const error = (0, util_1.wrapError)(unwrappedError);
logger.warning(`${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`);
}
}
}
}
catch (unwrappedError) {
const error = (0, util_1.wrapError)(unwrappedError);
@ -285,14 +270,28 @@ async function run() {
core.exportVariable(envVar, "false");
}
}
// Disable Python dependency extraction if feature flag set
if (await (0, feature_flags_1.isPythonDependencyInstallationDisabled)(codeql, features)) {
// From 2.16.0 the default for the python extractor is to not perform any
// dependency extraction. For versions before that, you needed to set this flag to
// enable this behavior (supported since 2.13.1).
if (await (0, util_1.codeQlVersionAbove)(codeql, "2.17.1")) {
// disabled by default, no warning
}
else if (await (0, util_1.codeQlVersionAbove)(codeql, "2.16.0")) {
// disabled by default, prints warning if environment variable is not set
core.exportVariable("CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION", "true");
}
else if (await (0, util_1.codeQlVersionAbove)(codeql, "2.13.1")) {
core.exportVariable("CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION", "true");
}
else {
// From 2.16.0 the default for the python extractor is to not perform any library
// extraction, so we need to set this flag to enable it.
core.exportVariable("CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0", "true");
logger.warning(`CodeQL Action versions 3.25.0 and later, and versions 2.25.0 and later no longer install Python dependencies. We recommend upgrading to at least CodeQL Bundle 2.16.0 to avoid any potential problems due to this (you are currently using ${(await codeql.getVersion()).version}). Alternatively, we recommend downgrading the CodeQL Action to version 3.24.10 (for customers using GitHub.com or GitHub Enterprise Server v3.12 or later) or 2.24.10 (for customers using GitHub Enterprise Server v3.11 or earlier).`);
}
if ((0, actions_util_1.getOptionalInput)("setup-python-dependencies") !== undefined) {
logger.warning("The setup-python-dependencies input is deprecated and no longer has any effect. We recommend removing any references from your workflows. See https://github.blog/changelog/2024-01-23-codeql-2-16-python-dependency-installation-disabled-new-queries-and-bug-fixes/ for more information.");
}
if (process.env["CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION"] !==
undefined) {
logger.warning("The CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION environment variable is deprecated and no longer has any effect. We recommend removing any references from your workflows. See https://github.blog/changelog/2024-01-23-codeql-2-16-python-dependency-installation-disabled-new-queries-and-bug-fixes/ for more information.");
}
const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", (0, actions_util_1.getOptionalInput)("registries"), apiDetails, logger);

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

42
lib/init.js generated
View file

@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.installPythonDeps = exports.checkInstallPython311 = exports.printPathFiltersWarning = exports.runInit = exports.initConfig = exports.initCodeQL = void 0;
exports.checkInstallPython311 = exports.printPathFiltersWarning = exports.runInit = exports.initConfig = exports.initCodeQL = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@ -89,44 +89,4 @@ async function checkInstallPython311(languages, codeql) {
}
}
exports.checkInstallPython311 = checkInstallPython311;
async function installPythonDeps(codeql, logger) {
logger.startGroup("Setup Python dependencies");
const scriptsFolder = path.resolve(__dirname, "../python-setup");
try {
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("powershell"), [
path.join(scriptsFolder, "install_tools.ps1"),
]).exec();
}
else {
await new toolrunner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec();
}
const script = "auto_install_packages.py";
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("py"), [
"-3",
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
}
else {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("python3"), [
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
}
}
catch (e) {
logger.endGroup();
logger.warning(`An error occurred while trying to automatically install Python dependencies: ${e}\n` +
"Please make sure any necessary dependencies are installed before calling the codeql-action/analyze " +
"step, and add a 'setup-python-dependencies: false' argument to this step to disable our automatic " +
"dependency installation and avoid this warning.");
return;
}
logger.endGroup();
}
exports.installPythonDeps = installPythonDeps;
//# sourceMappingURL=init.js.map

2
lib/init.js.map
View file

@ -1 +1 @@
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAGpD,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAG1D,qDAAgD;AAChD,mDAAwE;AAGjE,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,MAAc;IAOd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,EAAE,uBAAuB,EAAE,WAAW,EAAE,YAAY,EAAE,GAClE,MAAM,IAAA,oBAAW,EACf,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,MAAM,EACN,IAAI,CACL,CAAC;IACJ,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AACxE,CAAC;AA3BD,gCA2BC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAhBD,gCAgBC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAlCD,0BAkCC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAfD,0DAeC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAlBD,sDAkBC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;gBACvE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;aAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;QACX,CAAC;QACD,MAAM,MAAM,GAAG,0BAA0B,CAAC;QAC1C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBAC/D,IAAI;gBACJ,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE;gBACpE,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,gFAAgF,CAAC,IAAI;YACnF,qGAAqG;YACrG,oGAAoG;YACpG,iDAAiD,CACpD,CAAC;QACF,OAAO;IACT,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAzCD,8CAyCC"}
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAGpD,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAA0D;AAG1D,qDAAgD;AAChD,mDAAwE;AAGjE,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,MAAc;IAOd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,EAAE,uBAAuB,EAAE,WAAW,EAAE,YAAY,EAAE,GAClE,MAAM,IAAA,oBAAW,EACf,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,MAAM,EACN,IAAI,CACL,CAAC;IACJ,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AACxE,CAAC;AA3BD,gCA2BC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC,EACpC,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpD,IACE,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAC5B,6BAAY,CAAC,kCAAkC,CAChD,CAAC,EACF,CAAC;QACD,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAhBD,gCAgBC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAlCD,0BAkCC;AAED,SAAgB,uBAAuB,CACrC,MAA0B,EAC1B,MAAc;IAEd,qEAAqE;IACrE,sEAAsE;IACtE,IACE,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM;QACrC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACnD,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,6BAAiB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,mGAAmG,CACpG,CAAC;IACJ,CAAC;AACH,CAAC;AAfD,0DAeC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,oBAAQ,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;YACvE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAlBD,sDAkBC"}

2
node_modules/.package-lock.json generated vendored
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "3.24.11",
"version": "3.25.0",
"lockfileVersion": 3,
"requires": true,
"packages": {

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "codeql",
"version": "3.24.11",
"version": "3.25.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "codeql",
"version": "3.24.11",
"version": "3.25.0",
"license": "MIT",
"dependencies": {
"@actions/artifact": "^1.1.2",

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "3.24.11",
"version": "3.25.0",
"private": true,
"description": "CodeQL action",
"scripts": {

217
python-setup/auto_install_packages.py
View file

@ -1,217 +0,0 @@
#!/usr/bin/env python3
import sys
import os
import subprocess
from tempfile import mkdtemp
from typing import Optional
import shutil
import extractor_version
def _check_call(command, extra_env={}):
print('+ {}'.format(' '.join(command)), flush=True)
env = os.environ.copy()
env.update(extra_env)
subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env)
sys.stdout.flush()
sys.stderr.flush()
def _check_output(command, extra_env={}):
print('+ {}'.format(' '.join(command)), flush=True)
env = os.environ.copy()
env.update(extra_env)
out = subprocess.check_output(command, stdin=subprocess.DEVNULL, env=env)
print(out, flush=True)
sys.stderr.flush()
return out
def install_packages_with_poetry():
extra_poetry_env = {
# To handle poetry 1.2, which started to use keyring interaction MUCH more, we need
# add a workaround. See
# https://github.com/python-poetry/poetry/issues/2692#issuecomment-1235683370
"PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring",
# Projects that specify `virtualenvs.in-project = true` in their poetry.toml
# would get the venv created inside the repo directory, which would cause CodeQL
# to consider it as user-written code. We don't want this to happen. see
# https://python-poetry.org/docs/configuration/#virtualenvsin-project
"POETRY_VIRTUALENVS_IN_PROJECT": "False",
}
command = [sys.executable, '-m', 'poetry']
if sys.platform.startswith('win32'):
# In windows the default path were the deps are installed gets wiped out between steps,
# so we have to set it up to a folder that will be kept
os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
try:
_check_call(command + ['install', '--no-root'], extra_env=extra_poetry_env)
except subprocess.CalledProcessError:
sys.exit('package installation with poetry failed, see error above')
# poetry is super annoying with `poetry run`, since it will put lots of output on
# STDOUT if the current global python interpreter is not matching the one in the
# virtualenv for the package, which was the case for using poetry for Python 2 when
# default system interpreter was Python 3 :/
poetry_out = _check_output(command + ['run', 'which', 'python'], extra_env=extra_poetry_env)
python_executable_path = poetry_out.decode('utf-8').splitlines()[-1]
if sys.platform.startswith('win32'):
# Poetry produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter.
# We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE)
python_executable_path = python_executable_path[2:]
return python_executable_path
def install_packages_with_pipenv(has_lockfile):
command = [sys.executable, '-m', 'pipenv']
if sys.platform.startswith('win32'):
# In windows the default path were the deps are installed gets wiped out between steps,
# so we have to set it up to a folder that will be kept
os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
lock_args = ['--ignore-pipfile'] if has_lockfile else ['--skip-lock']
try:
_check_call(command + ['install'] + lock_args)
except subprocess.CalledProcessError:
sys.exit('package installation with pipenv failed, see error above')
pipenv_out = _check_output(command + ['run', 'which', 'python'])
python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1]
if sys.platform.startswith('win32'):
# Pipenv produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter.
# We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE)
python_executable_path = python_executable_path[2:]
return python_executable_path
def _create_venv(version: int):
# create temporary directory ... that just lives "forever"
venv_path = os.path.join(os.environ['RUNNER_WORKSPACE'], 'codeql-action-python-autoinstall')
print ("Creating venv in " + venv_path, flush = True)
# virtualenv is a bit nicer for setting up virtual environment, since it will provide
# up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't
if sys.platform.startswith('win32'):
if version == 2:
_check_call(['py', '-2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['py', '-3', '-m', 'virtualenv', venv_path])
else:
if version == 2:
_check_call(['python2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['python3', '-m', 'virtualenv', venv_path])
return venv_path
def install_requirements_txt_packages(version: int):
venv_path = _create_venv(version)
venv_pip = os.path.join(venv_path, 'bin', 'pip')
venv_python = os.path.join(venv_path, 'bin', 'python')
if sys.platform.startswith('win32'):
venv_pip = os.path.join(venv_path, 'Scripts', 'pip')
venv_python = os.path.join(venv_path, 'Scripts', 'python')
try:
_check_call([venv_pip, 'install', '-r', 'requirements.txt'])
except subprocess.CalledProcessError:
sys.exit('package installation with `pip install -r requirements.txt` failed, see error above')
return venv_python
def install_with_setup_py(version: int):
venv_path = _create_venv(version)
venv_pip = os.path.join(venv_path, 'bin', 'pip')
venv_python = os.path.join(venv_path, 'bin', 'python')
if sys.platform.startswith('win32'):
venv_pip = os.path.join(venv_path, 'Scripts', 'pip')
venv_python = os.path.join(venv_path, 'Scripts', 'python')
try:
# We have to choose between `python setup.py develop` and `pip install -e .`.
# Modern projects use `pip install -e .` and I wasn't able to see any downsides
# to doing so. However, `python setup.py develop` has some downsides -- from
# https://stackoverflow.com/a/19048754 :
# > Note that it is highly recommended to use pip install . (install) and pip
# > install -e . (developer install) to install packages, as invoking setup.py
# > directly will do the wrong things for many dependencies, such as pull
# > prereleases and incompatible package versions, or make the package hard to
# > uninstall with pip.
_check_call([venv_pip, 'install', '-e', '.'])
except subprocess.CalledProcessError:
sys.exit('package installation with `pip install -e .` failed, see error above')
return venv_python
def install_packages(codeql_base_dir) -> Optional[str]:
if os.path.exists('poetry.lock'):
print('Found poetry.lock, will install packages with poetry', flush=True)
return install_packages_with_poetry()
if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'):
if os.path.exists('Pipfile.lock'):
print('Found Pipfile.lock, will install packages with Pipenv', flush=True)
return install_packages_with_pipenv(has_lockfile=True)
else:
print('Found Pipfile, will install packages with Pipenv', flush=True)
return install_packages_with_pipenv(has_lockfile=False)
# get_extractor_version returns the Python version the extractor thinks this repo is using
version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False)
sys.stdout.flush()
sys.stderr.flush()
if version == 2 and not sys.platform.startswith('win32'):
# On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better
# error message than a traceback + `No such file or directory: 'python2'`
if shutil.which("python2") is None:
sys.exit(
"Python package installation failed: we detected this code as Python 2, but the 'python2' executable was not available. "
"To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, "
"for example by running 'sudo apt install python2' (Ubuntu 22.04). "
"If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new"
)
if os.path.exists('requirements.txt'):
print('Found requirements.txt, will install packages with pip', flush=True)
return install_requirements_txt_packages(version)
if os.path.exists('setup.py'):
print('Found setup.py, will install package with pip in editable mode', flush=True)
return install_with_setup_py(version)
print("was not able to install packages automatically", flush=True)
return None
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit('Must provide base directory for codeql tool as only argument')
codeql_base_dir = sys.argv[1]
python_executable_path = install_packages(codeql_base_dir)
if python_executable_path is not None:
# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
env_file = open(os.environ["GITHUB_ENV"], mode="at")
print("Setting CODEQL_PYTHON={}".format(python_executable_path))
print("CODEQL_PYTHON={}".format(python_executable_path), file=env_file)

52
python-setup/extractor_version.py
View file

@ -1,52 +0,0 @@
#!/usr/bin/env python
# A quick hack to get package installation for Code Scanning to work,
# since it needs to know which version we're going to analyze the project as.
# This file needs to be placed next to `python_tracer.py`, so in
# `<codeql-path>/python/tools/`
from __future__ import print_function, division
import os
import sys
from contextlib import contextmanager
@contextmanager
def suppress_stdout_stderr():
# taken from
# https://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/
with open(os.devnull, "w") as devnull:
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = devnull
sys.stderr = devnull
try:
yield
finally:
sys.stdout = old_stdout
sys.stderr = old_stderr
def get_extractor_version(codeql_base_dir: str, quiet: bool = True) -> int:
extractor_dir = os.path.join(codeql_base_dir, 'python', 'tools')
sys.path = [extractor_dir] + sys.path
from python_tracer import getzipfilename
zippath = os.path.join(extractor_dir, getzipfilename())
sys.path = [zippath] + sys.path
import buildtools.discover
if quiet:
with suppress_stdout_stderr():
return buildtools.discover.get_version()
else:
return buildtools.discover.get_version()
if __name__ == "__main__":
codeql_base_dir = sys.argv[1]
version = get_extractor_version(codeql_base_dir)
print('{!r}'.format(version))

34
python-setup/find_site_packages.py
View file

@ -1,34 +0,0 @@
"""
Print the path to the site-packages directory for the current Python environment.
"""
from __future__ import print_function
try:
import pip
import os
print(os.path.dirname(os.path.dirname(pip.__file__)))
except ImportError:
import sys
print("DEBUG: could not import pip", file=sys.stderr)
# if you use poetry with `virtualenvs.options.no-pip = true` you might end up with a
# virtualenv without pip, so the above trick doesn't actually work. See
# https://python-poetry.org/docs/configuration/#virtualenvsoptionsno-pip
#
# A possible option is to install `pip` into the virtualenv created by poetry
# (`poetry add pip`), but it turns out that doesn't always work :( for the test
# poetry/requests-3, I was not allowed to install pip! So I did not pursue this
# option further.
#
# Instead, testing `site.getsitepackages()` contains has the right path, whereas
# `site.getusersitepackages()` is about the system python (very confusing).
#
# We can't use the environment variable POETRY_VIRTUALENVS_OPTIONS_NO_PIP because it
# does not work, see https://github.com/python-poetry/poetry/issues/5906
import site
if sys.platform.startswith("win32"):
# On windows, the last entry of `site.getsitepackages()` has the right path
print(site.getsitepackages()[-1])
else:
# on unix, the first entry of `site.getsitepackages()` has the right path
print(site.getsitepackages()[0])

12
python-setup/install_tools.ps1
View file

@ -1,12 +0,0 @@
#! /usr/bin/pwsh
py -2 -m pip install --user --upgrade pip setuptools wheel
py -3 -m pip install --user --upgrade pip setuptools wheel
# virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
# pip/setuptools/wheel which basic `python3 -m venv venv` won't
py -2 -m pip install --user 'virtualenv!=20.12.0'
py -3 -m pip install --user virtualenv
py -3 -m pip install --user "poetry>=1.1"
py -3 -m pip install --user pipenv

50
python-setup/install_tools.sh
View file

@ -1,50 +0,0 @@
#!/bin/sh
set -x
set -e
# The binaries for packages installed with `pip install --user` are not available on PATH
# by default, so we fix up PATH to suppress warnings by pip. This also needs to be done by
# any script that needs to access poetry/pipenv.
#
# Using `::add-path::` from the actions toolkit is not enough, since that only affects
# subsequent actions in the current job, and not the current action.
export PATH="$HOME/.local/bin:$PATH"
# Setup Python 3 dependency installation tools.
python3 -m pip install --user --upgrade pip setuptools wheel
# virtualenv is a bit nicer for setting up virtual environment, since it will provide
# up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't.
#
# version 20.16.5 (Python 3 only) had some problems when used together with newer
# versions of setuptools (60+) and would not always put binaries under `<venv-path>/bin`
# -- see https://github.com/github/codeql-action/issues/1249 for more details.
python3 -m pip install --user --upgrade 'virtualenv>20.16.5'
# We install poetry with pip instead of the recommended way, since the recommended way
# caused some problem since `poetry run` gives output like:
#
# /root/.poetry/lib/poetry/_vendor/py2.7/subprocess32.py:149: RuntimeWarning: The _posixsubprocess module is not being used. Child process reliability may suffer if your program uses threads.
# "program uses threads.", RuntimeWarning)
# LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3
python3 -m pip install --user "poetry>=1.1"
python3 -m pip install --user pipenv
if command -v python2 >/dev/null 2>&1; then
# Setup Python 2 dependency installation tools. The Ubuntu 20.04 GHA environment
# does not come with a Python 2 pip, but if it is already installed, don't try to
# install it again (since that causes problems).
#
# This might seem like a hypothetical situation, but it happens all the time in our
# internal testing where we run the action twice in a row.
if ! python2 -m pip --version; then
echo "Will install pip for python2"
curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
fi
python2 -m pip install --user --upgrade pip setuptools wheel
python2 -m pip install --user 'virtualenv!=20.12.0'
fi

27
python-setup/tests/check_requests.ps1
View file

@ -1,27 +0,0 @@
#! /usr/bin/pwsh
$EXPECTED_PYTHON_VERSION=$args[0]
$EXPECTED_REQUESTS_VERSION=$args[1]
$FOUND_PYTHON_VERSION="$Env:LGTM_PYTHON_SETUP_VERSION"
$FOUND_PYTHONPATH="$Env:LGTM_INDEX_IMPORT_PATH"
write-host "FOUND_PYTHON_VERSION=$FOUND_PYTHON_VERSION FOUND_PYTHONPATH=$FOUND_PYTHONPATH "
if ($FOUND_PYTHON_VERSION -ne $EXPECTED_PYTHON_VERSION) {
write-host "Script told us to use Python $FOUND_PYTHON_VERSION, but expected $EXPECTED_PYTHON_VERSION"
exit 1
} else {
write-host "Script told us to use Python $FOUND_PYTHON_VERSION, which was expected"
}
$env:PYTHONPATH=$FOUND_PYTHONPATH
$INSTALLED_REQUESTS_VERSION = (py -3 -c "import requests; print(requests.__version__)")
if ($INSTALLED_REQUESTS_VERSION -ne $EXPECTED_REQUESTS_VERSION) {
write-host "Using $FOUND_PYTHONPATH as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS_VERSION"
exit 1
} else {
write-host "Using $FOUND_PYTHONPATH as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected"
}

31
python-setup/tests/check_requests.sh
View file

@ -1,31 +0,0 @@
#!/bin/bash
set -e
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
EXPECTED_PYTHON_VERSION=$1
EXPECTED_REQUESTS_VERSION=$2
FOUND_PYTHON_VERSION="$LGTM_PYTHON_SETUP_VERSION"
FOUND_PYTHONPATH="$LGTM_INDEX_IMPORT_PATH"
echo "FOUND_PYTHON_VERSION=${FOUND_PYTHON_VERSION} FOUND_PYTHONPATH=${FOUND_PYTHONPATH} "
if [[ $FOUND_PYTHON_VERSION != $EXPECTED_PYTHON_VERSION ]]; then
echo "Script told us to use Python ${FOUND_PYTHON_VERSION}, but expected ${EXPECTED_PYTHON_VERSION}"
exit 1
else
echo "Script told us to use Python ${FOUND_PYTHON_VERSION}, which was expected"
fi
PYTHON_EXE="python${EXPECTED_PYTHON_VERSION}"
INSTALLED_REQUESTS_VERSION=$(PYTHONPATH="${FOUND_PYTHONPATH}" "${PYTHON_EXE}" -c 'import requests; print(requests.__version__)')
if [[ "$INSTALLED_REQUESTS_VERSION" != "$EXPECTED_REQUESTS_VERSION" ]]; then
echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS_VERSION"
exit 1
else
echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected"
fi

34
python-setup/tests/from_python_exe.py
View file

@ -1,34 +0,0 @@
#!/usr/bin/env python3
import sys
import os
import subprocess
from typing import Tuple
def get_details(path_to_python_exe: str) -> Tuple[str, str]:
import_path = subprocess.check_output(
[
path_to_python_exe,
os.path.join(os.path.dirname(__file__), "..", "find_site_packages.py")
],
stdin=subprocess.DEVNULL,
)
version = subprocess.check_output(
[path_to_python_exe, "-c", "import sys; print(sys.version_info[0])"],
stdin=subprocess.DEVNULL,
)
return version.decode("utf-8").strip(), import_path.decode("utf-8").strip()
if __name__ == "__main__":
version, import_path = get_details(sys.argv[1])
# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
env_file = open(os.environ["GITHUB_ENV"], mode="at")
print("Setting LGTM_PYTHON_SETUP_VERSION={}".format(version))
print("LGTM_PYTHON_SETUP_VERSION={}".format(version), file=env_file)
print("Setting LGTM_INDEX_IMPORT_PATH={}".format(import_path))
print("LGTM_INDEX_IMPORT_PATH={}".format(import_path), file=env_file)

12
python-setup/tests/pipenv/python-3.8/Pipfile
View file

@ -1,12 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
requests = "*"
[requires]
python_version = "3.8"

151
python-setup/tests/pipenv/python-3.8/Pipfile.lock generated
View file

@ -1,151 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "acbc8c4e7f2f98f1059b2a93d581ef43f4aa0c9741e64e6253adff8e35fbd99e"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.8"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"certifi": {
"hashes": [
"sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
"sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
],
"markers": "python_version >= '3.6'",
"version": "==2023.7.22"
},
"charset-normalizer": {
"hashes": [
"sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843",
"sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786",
"sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e",
"sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8",
"sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4",
"sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa",
"sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d",
"sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82",
"sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7",
"sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895",
"sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d",
"sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a",
"sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382",
"sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678",
"sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b",
"sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e",
"sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741",
"sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4",
"sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596",
"sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9",
"sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69",
"sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c",
"sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77",
"sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13",
"sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459",
"sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e",
"sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7",
"sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908",
"sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a",
"sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f",
"sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8",
"sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482",
"sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d",
"sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d",
"sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545",
"sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34",
"sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86",
"sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6",
"sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe",
"sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e",
"sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc",
"sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7",
"sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd",
"sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c",
"sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557",
"sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a",
"sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89",
"sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078",
"sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e",
"sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4",
"sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403",
"sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0",
"sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89",
"sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115",
"sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9",
"sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05",
"sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a",
"sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec",
"sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56",
"sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38",
"sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479",
"sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c",
"sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e",
"sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd",
"sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186",
"sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455",
"sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c",
"sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65",
"sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78",
"sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287",
"sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df",
"sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43",
"sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1",
"sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7",
"sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989",
"sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a",
"sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63",
"sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884",
"sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649",
"sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810",
"sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828",
"sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4",
"sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2",
"sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd",
"sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5",
"sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe",
"sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293",
"sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e",
"sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e",
"sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"
],
"markers": "python_full_version >= '3.7.0'",
"version": "==3.3.0"
},
"idna": {
"hashes": [
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
],
"markers": "python_version >= '3.5'",
"version": "==3.4"
},
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
"sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
"version": "==2.31.0"
},
"urllib3": {
"hashes": [
"sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
"sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
"version": "==2.0.7"
}
},
"develop": {}
}

11
python-setup/tests/pipenv/requests-3/Pipfile
View file

@ -1,11 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
requests = "*"
[requires]

149
python-setup/tests/pipenv/requests-3/Pipfile.lock generated
View file

@ -1,149 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "70e8bf6bc774f5ca177467cab4e67d4264d0536857993326abc13ff43063bec0"
},
"pipfile-spec": 6,
"requires": {},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"certifi": {
"hashes": [
"sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
"sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
],
"markers": "python_version >= '3.6'",
"version": "==2023.7.22"
},
"charset-normalizer": {
"hashes": [
"sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843",
"sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786",
"sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e",
"sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8",
"sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4",
"sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa",
"sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d",
"sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82",
"sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7",
"sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895",
"sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d",
"sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a",
"sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382",
"sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678",
"sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b",
"sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e",
"sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741",
"sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4",
"sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596",
"sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9",
"sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69",
"sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c",
"sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77",
"sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13",
"sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459",
"sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e",
"sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7",
"sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908",
"sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a",
"sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f",
"sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8",
"sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482",
"sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d",
"sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d",
"sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545",
"sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34",
"sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86",
"sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6",
"sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe",
"sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e",
"sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc",
"sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7",
"sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd",
"sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c",
"sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557",
"sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a",
"sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89",
"sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078",
"sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e",
"sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4",
"sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403",
"sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0",
"sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89",
"sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115",
"sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9",
"sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05",
"sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a",
"sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec",
"sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56",
"sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38",
"sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479",
"sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c",
"sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e",
"sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd",
"sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186",
"sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455",
"sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c",
"sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65",
"sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78",
"sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287",
"sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df",
"sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43",
"sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1",
"sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7",
"sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989",
"sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a",
"sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63",
"sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884",
"sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649",
"sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810",
"sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828",
"sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4",
"sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2",
"sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd",
"sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5",
"sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe",
"sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293",
"sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e",
"sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e",
"sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"
],
"markers": "python_full_version >= '3.7.0'",
"version": "==3.3.0"
},
"idna": {
"hashes": [
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
],
"markers": "python_version >= '3.5'",
"version": "==3.4"
},
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
"sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
"version": "==2.31.0"
},
"urllib3": {
"hashes": [
"sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
"sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
"version": "==2.0.7"
}
},
"develop": {}
}

79
python-setup/tests/poetry/python-3.8/poetry.lock generated
View file

@ -1,79 +0,0 @@
# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
[[package]]
name = "certifi"
version = "2023.7.22"
description = "Python package for providing Mozilla's CA Bundle."
optional = false
python-versions = ">=3.6"
files = [
{file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"},
{file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"},
]
[[package]]
name = "charset-normalizer"
version = "2.0.7"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
optional = false
python-versions = ">=3.5.0"
files = [
{file = "charset-normalizer-2.0.7.tar.gz", hash = "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0"},
{file = "charset_normalizer-2.0.7-py3-none-any.whl", hash = "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b"},
]
[package.extras]
unicode-backport = ["unicodedata2"]
[[package]]
name = "idna"
version = "3.3"
description = "Internationalized Domain Names in Applications (IDNA)"
optional = false
python-versions = ">=3.5"
files = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
]
[[package]]
name = "requests"
version = "2.31.0"
description = "Python HTTP for Humans."
optional = false
python-versions = ">=3.7"
files = [
{file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
{file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
]
[package.dependencies]
certifi = ">=2017.4.17"
charset-normalizer = ">=2,<4"
idna = ">=2.5,<4"
urllib3 = ">=1.21.1,<3"
[package.extras]
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
[[package]]
name = "urllib3"
version = "1.26.18"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
files = [
{file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"},
{file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
]
[package.extras]
brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.8"
content-hash = "fabc9cabf9f18437e7b9ea3dbd1895a5a118239c17b3d097c465a290707e6bfd"

15
python-setup/tests/poetry/python-3.8/pyproject.toml
View file

@ -1,15 +0,0 @@
[tool.poetry]
name = "autoinstall-test"
version = "0.1.0"
description = ""
authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = "^3.8"
requests = "*"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"

79
python-setup/tests/poetry/requests-3/poetry.lock generated
View file

@ -1,79 +0,0 @@
# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
[[package]]
name = "certifi"
version = "2023.7.22"
description = "Python package for providing Mozilla's CA Bundle."
optional = false
python-versions = ">=3.6"
files = [
{file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"},
{file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"},
]
[[package]]
name = "charset-normalizer"
version = "2.0.7"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
optional = false
python-versions = ">=3.5.0"
files = [
{file = "charset-normalizer-2.0.7.tar.gz", hash = "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0"},
{file = "charset_normalizer-2.0.7-py3-none-any.whl", hash = "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b"},
]
[package.extras]
unicode-backport = ["unicodedata2"]
[[package]]
name = "idna"
version = "3.3"
description = "Internationalized Domain Names in Applications (IDNA)"
optional = false
python-versions = ">=3.5"
files = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
]
[[package]]
name = "requests"
version = "2.31.0"
description = "Python HTTP for Humans."
optional = false
python-versions = ">=3.7"
files = [
{file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
{file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
]
[package.dependencies]
certifi = ">=2017.4.17"
charset-normalizer = ">=2,<4"
idna = ">=2.5,<4"
urllib3 = ">=1.21.1,<3"
[package.extras]
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
[[package]]
name = "urllib3"
version = "1.26.18"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
files = [
{file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"},
{file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
]
[package.extras]
brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.7"
content-hash = "05ba07023dd383fd84c8e4945f1eae1aac30917ad1e4c10fb03e8235fcf3c248"

5
python-setup/tests/poetry/requests-3/poetry.toml
View file

@ -1,5 +0,0 @@
[virtualenvs]
in-project = true
[virtualenvs.options]
no-pip = true

15
python-setup/tests/poetry/requests-3/pyproject.toml
View file

@ -1,15 +0,0 @@
[tool.poetry]
name = "autoinstall-test"
version = "0.1.0"
description = ""
authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = "^3.7"
requests = "*"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"

1
python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt
View file

@ -1 +0,0 @@
requests==2.31.0

1
python-setup/tests/requirements/non-standard-location/wat.py
View file

@ -1 +0,0 @@
print('hello')

1
python-setup/tests/requirements/requests-3/requirements.txt
View file

@ -1 +0,0 @@
requests==2.31.0

3
python-setup/tests/requirements/requests-3/setup.py
View file

@ -1,3 +0,0 @@
# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 3 for sure
# Programming Language :: Python :: 3.7

12
python-setup/tests/setup_py/requests-3/setup.py
View file

@ -1,12 +0,0 @@
from setuptools import setup
# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure
# Programming Language :: Python :: 3.7
setup(
name="example-setup.py",
install_requires=["requests==2.31.0"],
python_requires='>=3.7',
)

1
src/analyze-action.ts
View file

@ -253,7 +253,6 @@ async function run() {
memory,
config,
logger,
features,
);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

61
src/analyze.ts
View file

@ -2,7 +2,6 @@ import * as fs from "fs";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import { safeWhich } from "@chrisgavin/safe-which";
import del from "del";
import * as yaml from "js-yaml";
@ -17,11 +16,7 @@ import * as configUtils from "./config-utils";
import { BuildMode } from "./config-utils";
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import { EnvVar } from "./environment";
import {
FeatureEnablement,
Feature,
isPythonDependencyInstallationDisabled,
} from "./feature-flags";
import { FeatureEnablement, Feature } from "./feature-flags";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { DatabaseCreationTimings, EventReport } from "./status-report";
@ -121,59 +116,24 @@ export interface QueriesStatusReport {
event_reports?: EventReport[];
}
async function setupPythonExtractor(
logger: Logger,
features: FeatureEnablement,
codeql: CodeQL,
) {
async function setupPythonExtractor(logger: Logger) {
const codeqlPython = process.env["CODEQL_PYTHON"];
if (codeqlPython === undefined || codeqlPython.length === 0) {
// If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
return;
}
if (await isPythonDependencyInstallationDisabled(codeql, features)) {
logger.warning(
"We recommend that you remove the CODEQL_PYTHON environment variable from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
"\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.",
);
return;
}
const scriptsFolder = path.resolve(__dirname, "../python-setup");
let output = "";
const options = {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
};
await new toolrunner.ToolRunner(
codeqlPython,
[path.join(scriptsFolder, "find_site_packages.py")],
options,
).exec();
logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`);
process.env["LGTM_INDEX_IMPORT_PATH"] = output;
output = "";
await new toolrunner.ToolRunner(
codeqlPython,
["-c", "import sys; print(sys.version_info[0])"],
options,
).exec();
logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
logger.warning(
"The CODEQL_PYTHON environment variable is no longer supported. Please remove it from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
"\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.",
);
return;
}
export async function runExtraction(
codeql: CodeQL,
config: configUtils.Config,
logger: Logger,
features: FeatureEnablement,
) {
for (const language of config.languages) {
if (dbIsFinalized(config, language, logger)) {
@ -186,7 +146,7 @@ export async function runExtraction(
if (shouldExtractLanguage(config, language)) {
logger.startGroup(`Extracting ${language}`);
if (language === Language.python) {
await setupPythonExtractor(logger, features, codeql);
await setupPythonExtractor(logger);
}
if (
config.buildMode &&
@ -262,12 +222,11 @@ async function finalizeDatabaseCreation(
threadsFlag: string,
memoryFlag: string,
logger: Logger,
features: FeatureEnablement,
): Promise<DatabaseCreationTimings> {
const codeql = await getCodeQL(config.codeQLCmd);
const extractionStart = performance.now();
await runExtraction(codeql, config, logger, features);
await runExtraction(codeql, config, logger);
const extractionTime = performance.now() - extractionStart;
const trapImportStart = performance.now();
@ -443,7 +402,6 @@ export async function runFinalize(
memoryFlag: string,
config: configUtils.Config,
logger: Logger,
features: FeatureEnablement,
): Promise<DatabaseCreationTimings> {
try {
await del(outputDir, { force: true });
@ -459,7 +417,6 @@ export async function runFinalize(
threadsFlag,
memoryFlag,
logger,
features,
);
// WARNING: This does not _really_ end tracing, as the tracer will restore its

37
src/feature-flags.ts
View file

@ -49,8 +49,6 @@ export enum Feature {
CppTrapCachingEnabled = "cpp_trap_caching_enabled",
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
DisablePythonDependencyInstallationEnabled = "disable_python_dependency_installation_enabled",
PythonDefaultIsToSkipDependencyInstallationEnabled = "python_default_is_to_skip_dependency_installation_enabled",
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
}
@ -95,25 +93,6 @@ export const featureConfig: Record<
minimumVersion: undefined,
defaultValue: false,
},
[Feature.DisablePythonDependencyInstallationEnabled]: {
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
// Although the python extractor only started supporting not extracting installed
// dependencies in 2.13.1, the init-action can still benefit from not installing
// dependencies no matter what codeql version we are using, so therefore the
// minimumVersion is set to 'undefined'. This means that with an old CodeQL version,
// packages available with current python3 installation might get extracted.
minimumVersion: undefined,
defaultValue: false,
},
[Feature.PythonDefaultIsToSkipDependencyInstallationEnabled]: {
// we can reuse the same environment variable as above. If someone has set it to
// `true` in their workflow this means dependencies are not installed, setting it to
// `false` means dependencies _will_ be installed. The same semantics are applied
// here!
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
minimumVersion: "2.16.0",
defaultValue: true,
},
};
/**
@ -458,19 +437,3 @@ class GitHubFeatureFlags {
}
}
}
export async function isPythonDependencyInstallationDisabled(
codeql: CodeQL,
features: FeatureEnablement,
): Promise<boolean> {
return (
(await features.getValue(
Feature.DisablePythonDependencyInstallationEnabled,
codeql,
)) ||
(await features.getValue(
Feature.PythonDefaultIsToSkipDependencyInstallationEnabled,
codeql,
))
);
}

71
src/init-action.ts
View file

@ -16,18 +16,8 @@ import { getGitHubVersion } from "./api-client";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import {
Feature,
Features,
isPythonDependencyInstallationDisabled,
} from "./feature-flags";
import {
checkInstallPython311,
initCodeQL,
initConfig,
installPythonDeps,
runInit,
} from "./init";
import { Feature, Features } from "./feature-flags";
import { checkInstallPython311, initCodeQL, initConfig, runInit } from "./init";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@ -298,24 +288,6 @@ async function run() {
);
await checkInstallPython311(config.languages, codeql);
if (
config.languages.includes(Language.python) &&
getRequiredInput("setup-python-dependencies") === "true"
) {
if (await isPythonDependencyInstallationDisabled(codeql, features)) {
logger.info("Skipping python dependency installation");
} else {
try {
await installPythonDeps(codeql, logger);
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
logger.warning(
`${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`,
);
}
}
}
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
core.setFailed(error.message);
@ -467,18 +439,43 @@ async function run() {
}
}
// Disable Python dependency extraction if feature flag set
if (await isPythonDependencyInstallationDisabled(codeql, features)) {
// From 2.16.0 the default for the python extractor is to not perform any
// dependency extraction. For versions before that, you needed to set this flag to
// enable this behavior (supported since 2.13.1).
if (await codeQlVersionAbove(codeql, "2.17.1")) {
// disabled by default, no warning
} else if (await codeQlVersionAbove(codeql, "2.16.0")) {
// disabled by default, prints warning if environment variable is not set
core.exportVariable(
"CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION",
"true",
);
} else if (await codeQlVersionAbove(codeql, "2.13.1")) {
core.exportVariable(
"CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION",
"true",
);
} else {
// From 2.16.0 the default for the python extractor is to not perform any library
// extraction, so we need to set this flag to enable it.
core.exportVariable(
"CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0",
"true",
logger.warning(
`CodeQL Action versions 3.25.0 and later, and versions 2.25.0 and later no longer install Python dependencies. We recommend upgrading to at least CodeQL Bundle 2.16.0 to avoid any potential problems due to this (you are currently using ${
(await codeql.getVersion()).version
}). Alternatively, we recommend downgrading the CodeQL Action to version 3.24.10 (for customers using GitHub.com or GitHub Enterprise Server v3.12 or later) or 2.24.10 (for customers using GitHub Enterprise Server v3.11 or earlier).`,
);
}
if (getOptionalInput("setup-python-dependencies") !== undefined) {
logger.warning(
"The setup-python-dependencies input is deprecated and no longer has any effect. We recommend removing any references from your workflows. See https://github.blog/changelog/2024-01-23-codeql-2-16-python-dependency-installation-disabled-new-queries-and-bug-fixes/ for more information.",
);
}
if (
process.env["CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION"] !==
undefined
) {
logger.warning(
"The CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION environment variable is deprecated and no longer has any effect. We recommend removing any references from your workflows. See https://github.blog/changelog/2024-01-23-codeql-2-16-python-dependency-installation-disabled-new-queries-and-bug-fixes/ for more information.",
);
}

43
src/init.ts
View file

@ -138,46 +138,3 @@ export async function checkInstallPython311(
]).exec();
}
}
export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
logger.startGroup("Setup Python dependencies");
const scriptsFolder = path.resolve(__dirname, "../python-setup");
try {
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("powershell"), [
path.join(scriptsFolder, "install_tools.ps1"),
]).exec();
} else {
await new toolrunner.ToolRunner(
path.join(scriptsFolder, "install_tools.sh"),
).exec();
}
const script = "auto_install_packages.py";
if (process.platform === "win32") {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("py"), [
"-3",
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
} else {
await new toolrunner.ToolRunner(await safeWhich.safeWhich("python3"), [
"-B",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
}
} catch (e) {
logger.endGroup();
logger.warning(
`An error occurred while trying to automatically install Python dependencies: ${e}\n` +
"Please make sure any necessary dependencies are installed before calling the codeql-action/analyze " +
"step, and add a 'setup-python-dependencies: false' argument to this step to disable our automatic " +
"dependency installation and avoid this warning.",
);
return;
}
logger.endGroup();
}