From ff8cd76def872e253b5d31485e43ffc1f7ed45bc Mon Sep 17 00:00:00 2001 From: David Rheinsberg Date: Tue, 21 Apr 2020 13:23:45 +0200 Subject: [PATCH] test: provide Makefile convenience targets Add convenience targets to `Makefile` which can run common sets of tests. For now, add a target for pylint, module-unittests, pipeline-runtime-tests, as well as all tests. Currently, it is quite cumbersome to run a reasonable test-setup locally. Pylint invokation is rather complex, the unittests and runtime tests in ./test are mixed, and not all tests in ./test can necessarily be run from a development system. This commit prepares for a simpler setup: * Add `make test-pylint` to run pylint as it is run by CI. * Add `make test-module` to run all module-unittests. This is meant to be fast (preferably close to instant) and easy to run during development to do a short check whether there are obvious typos or other errors in local changes. If we can keep these tests to machine-local requirements, if we avoid any sleeps or heavy computations, then this will remain a convenient test-suite to run locally without having to wait for 30min. In other words: We should be able to keep this under 10s (and for the long term under 1min) easily. * Add `make test-runtime` to run all osbuild pipeline executions. This is not meant to be fast, but thorough. This will require external sources (preferably limited to a suitable container image with everything embedded). This will very likely not be run during development, but rather by the CI. * Add `make test-all` to run all tests. Very handy for shy people when the chance of embarrassing copy-paste mistakes is too high to push publicly. Additionally to these new targets, this PR introduces 2 new directories in ./test: ./test/mod/ and ./test/run/ These are meant as equivalent to `test-module` and `test-runtime`. The reason is that preferably we stick to the auto-discovery of `unittest` to enumerate tests, rather than enrolling our own or having to enumerate them explicitly somewhere. However, we need some way to tell `unittest` which test belongs into which group. The easiest setup is likely to just use sub-directories. Note that `test-all` picks all tests independently of where they are put, even if they are in further different sub-modules under ./test. For now, no tests are moved into the new directories. I expect this to take a bit, since there are several out-standing PRs that modify ./test. I intend to do the final move once we agreed on this and we synchronized our test-modifications. --- .github/workflows/runtime-tests.yml | 23 ++++++++++++++ .github/workflows/tests.yml | 22 ++++++++------ Makefile | 47 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runtime-tests.yml b/.github/workflows/runtime-tests.yml index 2f37df5f..adc62b67 100644 --- a/.github/workflows/runtime-tests.yml +++ b/.github/workflows/runtime-tests.yml @@ -11,6 +11,29 @@ env: PYTHONUNBUFFERED: 1 jobs: + runtime_tests: + name: "Runtime Pipeline Execution Tests" + runs-on: ubuntu-latest + steps: + - name: "Clone Repository" + uses: actions/checkout@v2 + - name: "Install Dependencies" + run: | + sudo apt-get update + sudo apt-get -y install \ + nbd-client \ + qemu-utils \ + rpm \ + systemd-container \ + tar \ + yum + - name: "Set up Python" + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: "Run Pipeline Tests" + run: sudo env "PATH=$PATH" make test-runtime + noop_pipeline_tests: name: "Noop-Pipeline Tests" runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4ca693ff..ce07a516 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,22 +6,24 @@ jobs: pylint: name: "pylint" runs-on: ubuntu-latest - container: - image: docker.io/library/python:3.7 + container: docker.io/library/python:3.7 steps: - name: Install pylint run: pip install pylint==2.4.1 - - name: Clone repository uses: actions/checkout@v2 - with: - path: osbuild - - name: Run pylint - run: | - cd osbuild - find . -type f -name "*.py" | xargs pylint - pylint runners/* assemblers/* stages/* sources/* + run: make test-pylint + + module: + name: "Module Unittests" + runs-on: ubuntu-latest + container: docker.io/library/python:3.7 + steps: + - name: Clone repository + uses: actions/checkout@v2 + - name: Run Module Unittests + run: make test-module documentation: name: "📚 Documentation" diff --git a/Makefile b/Makefile index 5d99627f..924896f4 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,12 @@ BUILDDIR ?= . SRCDIR ?= . +PYLINT ?= pylint +PYTHON3 ?= python3 RST2MAN ?= rst2man +SHELL = /bin/bash + # # Automatic Variables # @@ -74,6 +78,11 @@ help: @echo @echo " help: Print this usage information." @echo " man: Generate all man-pages" + @echo + @echo " test-all: Run all tests" + @echo " test-pylint: Run pylint on all sources" + @echo " test-module: Run all module unit-tests" + @echo " test-runtime: Run all osbuild pipeline tests" $(BUILDDIR)/: mkdir -p "$@" @@ -99,6 +108,44 @@ $(MANPAGES_TROFF): $(BUILDDIR)/docs/%: $(SRCDIR)/docs/%.rst | $(BUILDDIR)/docs/ .PHONY: man man: $(MANPAGES_TROFF) +# +# Test Suite +# +# We use the python `unittest` module for all tests. All the test-sources are +# located in the `./test/` top-level directory, with `./test/mod/` for module +# unittests and `./test/run/` for osbuild pipeline runtime tests. +# + +.PHONY: test-units +test-module: + @$(PYTHON3) -m unittest \ + discover \ + --start=$(SRCDIR)/test/mod \ + --top-level-directory=$(SRCDIR) \ + -v + +.PHONY: test-runtime +test-runtime: + @[[ $${EUID} -eq 0 ]] || (echo "Error: Root privileges required!"; exit 1) + @$(PYTHON3) -m unittest \ + discover \ + --start=$(SRCDIR)/test/run \ + --top-level-directory=$(SRCDIR) \ + -v + +.PHONY: test-pylint +test-pylint: + @find . -type f -name "*.py" | xargs $(PYLINT) + @$(PYLINT) runners/* assemblers/* stages/* sources/* + +.PHONY: test +test-all: test-pylint + @$(PYTHON3) -m unittest \ + discover \ + --start=$(SRCDIR)/test \ + --top-level-directory=$(SRCDIR) \ + -v + # # Building packages #