PR#3992: Remove rpm-py-installer, update test docs and update Dockerfiles

Merges #3992
https://pagure.io/koji/pull-request/3992

Fixes: #3868
https://pagure.io/koji/issue/3868
optional dependency rpm-py-installer seems to be deprecated by upstream, causing `make test` to fail

devtools/containers/Dockerfile.centos7

@@ -5,29 +5,16 @@ RUN \
     dnf \
     dnf-plugins-core \
     gcc \
-    git \
-    make \
     krb5-devel \
     libffi-devel \
-    openssl-devel \
-    pyOpenSSL \
-    python-cheetah \
     python-devel \
     python-librepo \
     python-requests \
-    redhat-rpm-config \
+    rpm-build  && \
-    rpm-build \
-    rpm-python \
-    yum-utils && \
   yum install -y epel-release && \
   yum install -y \
-    python-flake8 \
-    python-multilib \
     python-pip \
-    python-psycopg2 \
+    python-psycopg2 && \
-    python-qpid-proton \
-    python-requests-kerberos \
-    yumdownloader && \
   yum clean all && \
   pip install -U 'pip==9.0.1' && \
   pip install -U tox

devtools/containers/Dockerfile.centos8

@@ -4,25 +4,11 @@ RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False epel-release && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
   python3-librepo \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
   dnf clean all

devtools/containers/Dockerfile.centos9

@@ -4,25 +4,11 @@ RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False epel-release && \
   dnf install -y --nodocs --setopt=install_weak_deps=False --enablerepo crb \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
   python3-librepo \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
   dnf clean all

devtools/containers/Dockerfile.f34

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:34
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
+  dnf clean all
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\

devtools/containers/Dockerfile.f35

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:35
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
+  dnf clean all
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\

devtools/containers/Dockerfile.f36

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:36
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
+  dnf clean all
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\

devtools/containers/Dockerfile.f37

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:37
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
+  dnf clean all
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\

devtools/containers/Dockerfile.f38

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:38
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
+  dnf clean all
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\

devtools/containers/Dockerfile.f39

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:39
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
+  dnf clean all
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\

devtools/containers/Dockerfile.rawhide

@@ -2,25 +2,11 @@ FROM registry.fedoraproject.org/fedora:rawhide
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
   python3-devel \
-  python3-pip \
+  python3-librepo \
-  python3-pytest \
-  python3-rpm \
   python3-tox \
-  redhat-rpm-config \
+  rpm-build && \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
   dnf clean all

devtools/containers/README.md

@@ -6,21 +6,31 @@ provided:
 
 * [`./Dockerfile.centos7`](./Dockerfile.centos7) CentOS 7, for testing with python2.7
 * [`./Dockerfile.centos8`](./Dockerfile.centos8) CentOS 8, for testing with python3.6
-* [`./Dockerfile.f32`](./Dockerfile.f32) Fedora 32, for testing with python3.8
+* [`./Dockerfile.centos9`](./Dockerfile.centos8) CentOS 9, for testing with python3.9
-* [`./Dockerfile.f33`](./Dockerfile.f33) Fedora 33, for testing with python3.9
+* [`./Dockerfile.f34`](...) Fedora 34, for testing with python3.9
+* [`./Dockerfile.f35`](./Dockerfile.f32) Fedora 35, for testing with python3.10
+* [`./Dockerfile.f36`](./Dockerfile.f32) Fedora 36, for testing with python3.10
+* [`./Dockerfile.f37`](./Dockerfile.f32) Fedora 37, for testing with python3.11
+* [`./Dockerfile.f38`](./Dockerfile.f32) Fedora 38, for testing with python3.11
+* [`./Dockerfile.f39`](./Dockerfile.f32) Fedora 39, for testing with python3.12
 * [`./Dockerfile.rawhide`](./Dockerfile.rawhie) Fedora Rawhide, for testing with python3.?
 
 To use them, taking fedora as an example:
 
-    docker build -t koji_test_fedora:32 --no-cache -f Dockerfile.f32
+    docker build -t koji_test_fedora:39 --no-cache -f Dockerfile.f39
-    docker run --rm -v $PWD:/koji --name koji_test koji_test_fedora:32 bash -c "cd /koji && tox -e flake8,py3"
+    docker run --rm -v $PWD:/koji --name koji_test koji_test_fedora:39 bash -c "cd /koji && tox -e flake8,py3,bandit"
 
-Or CentOS as an example:
+Or CentOS with py3 as an example:
 
     docker build -t koji_test_centos:8 --no-cache -f Dockerfile.centos8
-    docker run --rm -v $PWD:/koji --name koji_test koji_test_centos:8 bash -c "cd /koji && tox -e py2"
+    docker run --rm -v $PWD:/koji --name koji_test koji_test_centos:8 bash -c "cd /koji && tox -e flake8,py3,bandit"
+
+Or CentOS with py2 as an example:
+
+    docker build -t koji_test_centos:7 --no-cache -f Dockerfile.centos8
+    docker run --rm -v $PWD:/koji --name koji_test koji_test_centos:7 bash -c "cd /koji && tox -e py2"
 
 When running with Podman and SELinux enabled, use the "--security-opt
 label=disable" option:
 
-    podman run --rm -v $PWD:/koji --security-opt label=disable --name koji_test koji_test_fedora:32 bash -c "cd /koji && ls -l /koji && tox -e flake8,py3"
+    podman run --rm -v $PWD:/koji --security-opt label=disable --name koji_test koji_test_fedora:39 bash -c "cd /koji && ls -l /koji && tox -e flake8,py3,bandit"

docs/source/writing_koji_code.rst

@@ -706,21 +706,47 @@ Unit Tests
 Koji comes with a small test suite, that you should always run when making
 changes to the code. To do so, just run ``make test`` in your terminal.
 
-You will need to install the following packages to actually run the tests.
+You can run the tests from your local and you will need to install the following packages to actually run the tests.
 
+For py3 instances:
+ * ``gcc``
  * ``glibc-langpack-en``
- * ``make``
+ * ``krb5-devel``
- * ``python3-cheetah``
+ * ``python3-devel``
- * ``python3-coverage``
+ * ``python3-librepo``
- * ``python3-dateutil``
+ * ``python3-tox``
- * ``python3-mock``
+ * ``rpm-build``
- * ``python3-multilib``
+ * ``Other packages are installed by test-requirements.txt.``
- * ``python3-pytest``
+
- * ``python3-psycopg2``
+For py2 instances:
- * ``python3-qpid-proton``
+ * ``dnf``
- * ``python3-requests``
+ * ``dnf-plugins-core``
- * ``python3-requests-kerberos``
+ * ``gcc``
- * ``python3-requests-mock``
+ * ``krb5-devel``
+ * ``libffi-devel``
+ * ``python-devel``
+ * ``python-librepo``
+ * ``python-requests``
+ * ``rpm-build``
+ * ``epel-release``
+ * ``python-pip``
+ * ``python-psycopg2``
+ * ``tox``
+ * ``pip==9.0.1``
+ * ``Other packages are installed by test-requirements-py2.txt.``
+
+You can use containers for the tests. Dockerfiles are placed in `devtools/containers` directory.
+Supported Dockerfiles:
+ * ``Dockerfile.centos7``
+ * ``Dockerfile.centos8``
+ * ``Dockerfile.centos9``
+ * ``Dockerfile.f34``
+ * ``Dockerfile.f35``
+ * ``Dockerfile.f36``
+ * ``Dockerfile.f37``
+ * ``Dockerfile.f38``
+ * ``Dockerfile.f39``
+ * ``Dockerfile.rawhide``
 
 Please note that it is currently not supported to use *virtualenv* when hacking
 on Koji.

requirements.txt

@@ -5,4 +5,3 @@ psycopg2-binary;python_version >= '3.0'
 python-multilib
 python-qpid-proton<0.38.0;python_version < '3.0'
 python-qpid-proton;python_version >= '3.0'
-rpm-py-installer

setup.py

@@ -17,7 +17,6 @@ def get_install_requires():
         'requests-gssapi',
         'six',
         # 'libcomps',
-        # 'rpm-py-installer', # it is optional feature
         # 'rpm',
     ]
     return requires

test-requirements-py2.txt

@@ -5,3 +5,6 @@ mock<=2.0.0
 requests-mock<1.11.0
 coverage
 pytest
+Cheetah
+python-multilib
+python-qpid-proton<0.38.0

test-requirements.txt

@@ -5,3 +5,7 @@ mock<=2.0.0
 requests-mock
 coverage
 pytest
+Cheetah3
+psycopg2-binary
+python-multilib
+python-qpid-proton

tox.ini

@@ -21,11 +21,9 @@ deps =
     -r{toxinidir}/requirements.txt
 # We need to access python-rpm, at least, and potentially more on EL6
 sitepackages = true
-# Tell the rpm-py-installer package to download binaries instead of building
-# from source
 setenv =
-    RPM_PY_INSTALL_BIN=true
     COLUMNS=80
+# Expected that python-rpm is installed on the host
 # If rpm's python bindings are missing, don't continue
 # Also, because coverage might be installed system-wide and it serves as our
 # entry point, let's make sure it's installed in the virtualenv.