robojerk/codeql-action

Merge branch 'main' into robertbrignull/runner_workflow

Browse source
This commit is contained in:
Robin Neatherway 2020-10-20 11:59:11 +01:00 • committed by GitHub
commit b6e9407b12
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 211 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

10
.github/workflows/codeql.yml vendored
View file

@ -14,16 +14,6 @@ jobs:
steps:
- uses: actions/checkout@v2
with:
# Must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head of the pull request.
fetch-depth: 2
# If this run was triggered by a pull request event then checkout
# the head of the pull request instead of the merge commit.
- run: git checkout HEAD^2
if: ${{ github.event_name == 'pull_request' }}
- uses: ./init
with:
languages: javascript

2
.github/workflows/python-deps.yml → .github/workflows/python-deps-linux.yml vendored
View file

@ -1,4 +1,4 @@
name: Test Python Package Installation
name: Test Python Package Installation on Linux
on:
push:

61
.github/workflows/python-deps-windows.yml vendored Normal file
View file

@ -0,0 +1,61 @@
name: Test Python Package Installation on Windows
on:
push:
branches: [main, v1]
pull_request:
jobs:
test-setup-python-scripts:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- test_dir: python-setup/tests/pipenv/requests-2
python_version: 2
- test_dir: python-setup/tests/pipenv/requests-3
python_version: 3
- test_dir: python-setup/tests/poetry/requests-2
python_version: 2
- test_dir: python-setup/tests/poetry/requests-3
python_version: 3
- test_dir: python-setup/tests/requirements/requests-2
python_version: 2
- test_dir: python-setup/tests/requirements/requests-3
python_version: 3
- test_dir: python-setup/tests/setup_py/requests-2
python_version: 2
- test_dir: python-setup/tests/setup_py/requests-3
python_version: 3
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: python
- name: Test Auto Package Installation
run: |
$cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\install_tools.ps1"
powershell -File $cmd
cd $Env:GITHUB_WORKSPACE\\${{ matrix.test_dir }}
py -3 $Env:GITHUB_WORKSPACE\\python-setup\\auto_install_packages.py C:\\hostedtoolcache\\windows\\CodeQL\\0.0.0-20200826\\x64\\codeql
- name: Setup for extractor
run: |
echo $Env:CODEQL_PYTHON
py -3 $Env:GITHUB_WORKSPACE\\python-setup\\tests\\from_python_exe.py $Env:CODEQL_PYTHON
- name: Verify packages installed
run: |
$cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\tests\\check_requests_123.ps1"
powershell -File $cmd ${{ matrix.python_version }}

2
README.md
View file

@ -31,7 +31,7 @@ on:
# │ │ │ │ │
# │ │ │ │ │
# * * * * *
- cron: '0 0 * * 0'
- cron: '30 1 * * 0'
jobs:
CodeQL-Build:

2
lib/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20200826"
"bundleVersion": "codeql-bundle-20201008"
}

29
lib/init.js generated
View file

@ -123,8 +123,8 @@ async function injectWindowsTracer(processName, processLevel, config, codeql, tr
exports.injectWindowsTracer = injectWindowsTracer;
async function installPythonDeps(codeql, logger) {
logger.startGroup("Setup Python dependencies");
if (process.platform !== "linux") {
logger.info("Currently, auto-installing python dependancies is only supported on linux");
if (process.platform === "darwin") {
logger.info("Currently, auto-installing python dependencies is not supported on MacOS");
logger.endGroup();
return;
}
@ -132,22 +132,43 @@ async function installPythonDeps(codeql, logger) {
// Setup tools on the Github hosted runners
if (process.env["ImageOS"] !== undefined) {
try {
await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec();
if (process.platform === "win32") {
await new toolrunnner.ToolRunner("powershell", [
path.join(scriptsFolder, "install_tools.ps1"),
]).exec();
}
else {
await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec();
}
}
catch (e) {
// This script tries to install some needed tools in the runner. It should not fail, but if it does
// we just abort the process without failing the action
logger.endGroup();
logger.warning("Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process.");
return;
}
}
// Install dependencies
try {
await new toolrunnner.ToolRunner(path.join(scriptsFolder, "auto_install_packages.py"), [path.dirname(codeql.getPath())]).exec();
const script = "auto_install_packages.py";
if (process.platform === "win32") {
await new toolrunnner.ToolRunner("py", [
"-3",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
}
else {
await new toolrunnner.ToolRunner(path.join(scriptsFolder, script), [
path.dirname(codeql.getPath()),
]).exec();
}
}
catch (e) {
logger.endGroup();
logger.warning("We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process.");
return;
}
logger.endGroup();
}

2
lib/init.js.map
View file

@ -1 +1 @@
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,MAAM,CAAC,IAAI,CACT,2EAA2E,CAC5E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA1CD,8CA0CC"}
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,YAAY,EAAE;oBAC7C,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;iBAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;aACX;iBAAM;gBACL,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;aACV;SACF;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;YACF,OAAO;SACR;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,MAAM,GAAG,0BAA0B,CAAC;QAC1C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE;gBACrC,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;aAAM;YACL,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,EAAE;gBACjE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;KACF;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;QACF,OAAO;KACR;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA1DD,8CA0DC"}

62
python-setup/auto_install_packages.py
View file

@ -23,8 +23,13 @@ def _check_output(command):
def install_packages_with_poetry():
command = [sys.executable, '-m', 'poetry']
if sys.platform.startswith('win32'):
# In windows the default path were the deps are installed gets wiped out between steps,
# so we have to set it up to a folder that will be kept
os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
try:
_check_call(['poetry', 'install', '--no-root'])
_check_call(command + ['install', '--no-root'])
except subprocess.CalledProcessError:
sys.exit('package installation with poetry failed, see error above')
@ -33,44 +38,69 @@ def install_packages_with_poetry():
# virtualenv for the package, which was the case for using poetry for Python 2 when
# default system interpreter was Python 3 :/
poetry_out = _check_output(['poetry', 'run', 'which', 'python'])
poetry_out = _check_output(command + ['run', 'which', 'python'])
python_executable_path = poetry_out.decode('utf-8').splitlines()[-1]
if sys.platform.startswith('win32'):
# Poetry produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter.
# We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE)
python_executable_path = python_executable_path[2:]
return python_executable_path
def install_packages_with_pipenv():
command = [sys.executable, '-m', 'pipenv']
if sys.platform.startswith('win32'):
# In windows the default path were the deps are installed gets wiped out between steps,
# so we have to set it up to a folder that will be kept
os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
try:
_check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile'])
_check_call(command + ['install', '--keep-outdated', '--ignore-pipfile'])
except subprocess.CalledProcessError:
sys.exit('package installation with pipenv failed, see error above')
pipenv_out = _check_output(['pipenv', 'run', 'which', 'python'])
pipenv_out = _check_output(command + ['run', 'which', 'python'])
python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1]
if sys.platform.startswith('win32'):
# Pipenv produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter.
# We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE)
python_executable_path = python_executable_path[2:]
return python_executable_path
def _create_venv(version: int):
# create temporary directory ... that just lives "forever"
venv_path = mkdtemp(prefix='codeql-action-python-autoinstall-')
venv_path = os.path.join(os.environ['RUNNER_WORKSPACE'], 'codeql-action-python-autoinstall')
print ("Creating venv in " + venv_path, flush = True)
# virtualenv is a bit nicer for setting up virtual environment, since it will provide
# up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't
if version == 2:
_check_call(['python2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['python3', '-m', 'virtualenv', venv_path])
if sys.platform.startswith('win32'):
if version == 2:
_check_call(['py', '-2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['py', '-3', '-m', 'virtualenv', venv_path])
else:
if version == 2:
_check_call(['python2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['python3', '-m', 'virtualenv', venv_path])
return venv_path
def install_requirements_txt_packages(version: int):
venv_path = _create_venv(version)
venv_pip = os.path.join(venv_path, 'bin', 'pip')
venv_python = os.path.join(venv_path, 'bin', 'python')
if sys.platform.startswith('win32'):
venv_pip = os.path.join(venv_path, 'Scripts', 'pip')
venv_python = os.path.join(venv_path, 'Scripts', 'python')
try:
_check_call([venv_pip, 'install', '-r', 'requirements.txt'])
except subprocess.CalledProcessError:
@ -81,9 +111,14 @@ def install_requirements_txt_packages(version: int):
def install_with_setup_py(version: int):
venv_path = _create_venv(version)
venv_pip = os.path.join(venv_path, 'bin', 'pip')
venv_python = os.path.join(venv_path, 'bin', 'python')
if sys.platform.startswith('win32'):
venv_pip = os.path.join(venv_path, 'Scripts', 'pip')
venv_python = os.path.join(venv_path, 'Scripts', 'python')
try:
# We have to choose between `python setup.py develop` and `pip install -e .`.
# Modern projects use `pip install -e .` and I wasn't able to see any downsides
@ -135,12 +170,11 @@ if __name__ == "__main__":
codeql_base_dir = sys.argv[1]
# The binaries for packages installed with `pip install --user` are not available on
# PATH by default, so we need to manually add them.
os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH']
python_executable_path = install_packages(codeql_base_dir)
if python_executable_path is not None:
# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
env_file = open(os.environ["GITHUB_ENV"], mode="at")
print("Setting CODEQL_PYTHON={}".format(python_executable_path))
print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path))
print("CODEQL_PYTHON={}".format(python_executable_path), file=env_file)

13
python-setup/install_tools.ps1 Normal file
View file

@ -0,0 +1,13 @@
#! /usr/bin/pwsh
py -2 -m pip install --user --upgrade pip setuptools wheel
py -3 -m pip install --user --upgrade pip setuptools wheel
# virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
# pip/setuptools/wheel which basic `python3 -m venv venv` won't
py -2 -m pip install --user virtualenv
py -3 -m pip install --user virtualenv
# poetry 1.0.10 has error (https://github.com/python-poetry/poetry/issues/2711)
py -3 -m pip install --user poetry!=1.0.10
py -3 -m pip install --user pipenv

28
python-setup/tests/check_requests_123.ps1 Normal file
View file

@ -0,0 +1,28 @@
#! /usr/bin/pwsh
$EXPECTED_VERSION=$args[0]
$FOUND_VERSION="$Env:LGTM_PYTHON_SETUP_VERSION"
$FOUND_PYTHONPATH="$Env:LGTM_INDEX_IMPORT_PATH"
write-host "FOUND_VERSION=$FOUND_VERSION FOUND_PYTHONPATH=$FOUND_PYTHONPATH "
if ($FOUND_VERSION -ne $EXPECTED_VERSION) {
write-host "Script told us to use Python $FOUND_VERSION, but expected $EXPECTED_VERSION"
exit 1
} else {
write-host "Script told us to use Python $FOUND_VERSION, which was expected"
}
$env:PYTHONPATH=$FOUND_PYTHONPATH
$INSTALLED_REQUESTS_VERSION = (py -3 -c "import requests; print(requests.__version__)")
$EXPECTED_REQUESTS="1.2.3"
if ($INSTALLED_REQUESTS_VERSION -ne $EXPECTED_REQUESTS) {
write-host "Using $FOUND_PYTHONPATH as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS"
exit 1
} else {
write-host "Using $FOUND_PYTHONPATH as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected"
}

8
python-setup/tests/from_python_exe.py
View file

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import sys
import os
import subprocess
from typing import Tuple
@ -24,8 +25,11 @@ def get_details(path_to_python_exe: str) -> Tuple[str, str]:
if __name__ == "__main__":
version, import_path = get_details(sys.argv[1])
# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
env_file = open(os.environ["GITHUB_ENV"], mode="at")
print("Setting LGTM_PYTHON_SETUP_VERSION={}".format(version))
print("::set-env name=LGTM_PYTHON_SETUP_VERSION::{}".format(version))
print("LGTM_PYTHON_SETUP_VERSION={}".format(version), file=env_file)
print("Setting LGTM_INDEX_IMPORT_PATH={}".format(import_path))
print("::set-env name=LGTM_INDEX_IMPORT_PATH::{}".format(import_path))
print("LGTM_INDEX_IMPORT_PATH={}".format(import_path), file=env_file)

2
src/defaults.json
View file

@ -1,3 +1,3 @@
{
"bundleVersion": "codeql-bundle-20200826"
"bundleVersion": "codeql-bundle-20201008"
}

34
src/init.ts
View file

@ -186,9 +186,9 @@ export async function injectWindowsTracer(
export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
logger.startGroup("Setup Python dependencies");
if (process.platform !== "linux") {
if (process.platform === "darwin") {
logger.info(
"Currently, auto-installing python dependancies is only supported on linux"
"Currently, auto-installing python dependencies is not supported on MacOS"
);
logger.endGroup();
return;
@ -199,9 +199,15 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
// Setup tools on the Github hosted runners
if (process.env["ImageOS"] !== undefined) {
try {
await new toolrunnner.ToolRunner(
path.join(scriptsFolder, "install_tools.sh")
).exec();
if (process.platform === "win32") {
await new toolrunnner.ToolRunner("powershell", [
path.join(scriptsFolder, "install_tools.ps1"),
]).exec();
} else {
await new toolrunnner.ToolRunner(
path.join(scriptsFolder, "install_tools.sh")
).exec();
}
} catch (e) {
// This script tries to install some needed tools in the runner. It should not fail, but if it does
// we just abort the process without failing the action
@ -209,20 +215,30 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
logger.warning(
"Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process."
);
return;
}
}
// Install dependencies
try {
await new toolrunnner.ToolRunner(
path.join(scriptsFolder, "auto_install_packages.py"),
[path.dirname(codeql.getPath())]
).exec();
const script = "auto_install_packages.py";
if (process.platform === "win32") {
await new toolrunnner.ToolRunner("py", [
"-3",
path.join(scriptsFolder, script),
path.dirname(codeql.getPath()),
]).exec();
} else {
await new toolrunnner.ToolRunner(path.join(scriptsFolder, script), [
path.dirname(codeql.getPath()),
]).exec();
}
} catch (e) {
logger.endGroup();
logger.warning(
"We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process."
);
return;
}
logger.endGroup();
}